36 	buf->chan->cb->buf_unmapped(buf, vma->vm_file);  in relay_file_mmap_close()
107 	if (length != (unsigned long)buf->chan->alloc_size)  in relay_mmap_buf()
113 	buf->chan->cb->buf_mapped(buf, filp);  in relay_mmap_buf()
165 static struct rchan_buf *relay_create_buf(struct rchan *chan)  in relay_create_buf()  argument
169 	if (chan->n_subbufs > UINT_MAX / sizeof(size_t *))  in relay_create_buf()
175 	buf->padding = kmalloc(chan->n_subbufs * sizeof(size_t *), GFP_KERNEL);  in relay_create_buf()
179 	buf->start = relay_alloc_buf(buf, &chan->alloc_size);  in relay_create_buf()
183 	buf->chan = chan;  in relay_create_buf()
184 	kref_get(&buf->chan->kref);  in relay_create_buf()
201 	struct rchan *chan = container_of(kref, struct rchan, kref);  in relay_destroy_channel()  local
202 	kfree(chan);  in relay_destroy_channel()
211 	struct rchan *chan = buf->chan;  in relay_destroy_buf()  local
220 	chan->buf[buf->cpu] = NULL;  in relay_destroy_buf()
223 	kref_put(&chan->kref, relay_destroy_channel);  in relay_destroy_buf()
260 	return (ready >= buf->chan->n_subbufs) ? 1 : 0;  in relay_buf_full()
369 	for (i = 0; i < buf->chan->n_subbufs; i++)  in __relay_reset()
372 	buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0);  in __relay_reset()
386 void relay_reset(struct rchan *chan)  in relay_reset()  argument
390 	if (!chan)  in relay_reset()
393 	if (chan->is_global && chan->buf[0]) {  in relay_reset()
394 		__relay_reset(chan->buf[0], 0);  in relay_reset()
400 		if (chan->buf[i])  in relay_reset()
401 			__relay_reset(chan->buf[i], 0);  in relay_reset()
413 static struct dentry *relay_create_buf_file(struct rchan *chan,  in relay_create_buf_file()  argument
423 	snprintf(tmpname, NAME_MAX, "%s%d", chan->base_filename, cpu);  in relay_create_buf_file()
426 	dentry = chan->cb->create_buf_file(tmpname, chan->parent,  in relay_create_buf_file()
428 					   &chan->is_global);  in relay_create_buf_file()
440 static struct rchan_buf *relay_open_buf(struct rchan *chan, unsigned int cpu)  in relay_open_buf()  argument
445  	if (chan->is_global)  in relay_open_buf()
446 		return chan->buf[0];  in relay_open_buf()
448 	buf = relay_create_buf(chan);  in relay_open_buf()
452 	if (chan->has_base_filename) {  in relay_open_buf()
453 		dentry = relay_create_buf_file(chan, buf, cpu);  in relay_open_buf()
462  	if(chan->is_global) {  in relay_open_buf()
463  		chan->buf[0] = buf;  in relay_open_buf()
486 	buf->chan->cb->remove_buf_file(buf->dentry);  in relay_close_buf()
490 static void setup_callbacks(struct rchan *chan,  in setup_callbacks()  argument
494 		chan->cb = &default_channel_callbacks;  in setup_callbacks()
508 	chan->cb = cb;  in setup_callbacks()
524 	struct rchan *chan;  in relay_hotcpu_callback()  local
530 		list_for_each_entry(chan, &relay_channels, list) {  in relay_hotcpu_callback()
531 			if (chan->buf[hotcpu])  in relay_hotcpu_callback()
533 			chan->buf[hotcpu] = relay_open_buf(chan, hotcpu);  in relay_hotcpu_callback()
534 			if(!chan->buf[hotcpu]) {  in relay_hotcpu_callback()
577 	struct rchan *chan;  in relay_open()  local
584 	chan = kzalloc(sizeof(struct rchan), GFP_KERNEL);  in relay_open()
585 	if (!chan)  in relay_open()
588 	chan->version = RELAYFS_CHANNEL_VERSION;  in relay_open()
589 	chan->n_subbufs = n_subbufs;  in relay_open()
590 	chan->subbuf_size = subbuf_size;  in relay_open()
591 	chan->alloc_size = PAGE_ALIGN(subbuf_size * n_subbufs);  in relay_open()
592 	chan->parent = parent;  in relay_open()
593 	chan->private_data = private_data;  in relay_open()
595 		chan->has_base_filename = 1;  in relay_open()
596 		strlcpy(chan->base_filename, base_filename, NAME_MAX);  in relay_open()
598 	setup_callbacks(chan, cb);  in relay_open()
599 	kref_init(&chan->kref);  in relay_open()
603 		chan->buf[i] = relay_open_buf(chan, i);  in relay_open()
604 		if (!chan->buf[i])  in relay_open()
607 	list_add(&chan->list, &relay_channels);  in relay_open()
610 	return chan;  in relay_open()
614 		if (chan->buf[i])  in relay_open()
615 			relay_close_buf(chan->buf[i]);  in relay_open()
618 	kref_put(&chan->kref, relay_destroy_channel);  in relay_open()
648 int relay_late_setup_files(struct rchan *chan,  in relay_late_setup_files()  argument
658 	if (!chan || !base_filename)  in relay_late_setup_files()
661 	strlcpy(chan->base_filename, base_filename, NAME_MAX);  in relay_late_setup_files()
665 	if (unlikely(chan->has_base_filename)) {  in relay_late_setup_files()
669 	chan->has_base_filename = 1;  in relay_late_setup_files()
670 	chan->parent = parent;  in relay_late_setup_files()
678 		if (unlikely(!chan->buf[i])) {  in relay_late_setup_files()
684 		dentry = relay_create_buf_file(chan, chan->buf[i], i);  in relay_late_setup_files()
692 			relay_set_buf_dentry(chan->buf[i], dentry);  in relay_late_setup_files()
695 			disp.buf = chan->buf[i];  in relay_late_setup_files()
727 	if (unlikely(length > buf->chan->subbuf_size))  in relay_switch_subbuf()
730 	if (buf->offset != buf->chan->subbuf_size + 1) {  in relay_switch_subbuf()
731 		buf->prev_padding = buf->chan->subbuf_size - buf->offset;  in relay_switch_subbuf()
732 		old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
737 				buf->chan->subbuf_size -  in relay_switch_subbuf()
740 			buf->early_bytes += buf->chan->subbuf_size -  in relay_switch_subbuf()
754 	new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
755 	new = buf->start + new_subbuf * buf->chan->subbuf_size;  in relay_switch_subbuf()
757 	if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) {  in relay_switch_subbuf()
758 		buf->offset = buf->chan->subbuf_size + 1;  in relay_switch_subbuf()
764 	if (unlikely(length + buf->offset > buf->chan->subbuf_size))  in relay_switch_subbuf()
770 	buf->chan->last_toobig = length;  in relay_switch_subbuf()
788 void relay_subbufs_consumed(struct rchan *chan,  in relay_subbufs_consumed()  argument
794 	if (!chan)  in relay_subbufs_consumed()
797 	if (cpu >= NR_CPUS || !chan->buf[cpu] ||  in relay_subbufs_consumed()
798 					subbufs_consumed > chan->n_subbufs)  in relay_subbufs_consumed()
801 	buf = chan->buf[cpu];  in relay_subbufs_consumed()
815 void relay_close(struct rchan *chan)  in relay_close()  argument
819 	if (!chan)  in relay_close()
823 	if (chan->is_global && chan->buf[0])  in relay_close()
824 		relay_close_buf(chan->buf[0]);  in relay_close()
827 			if (chan->buf[i])  in relay_close()
828 				relay_close_buf(chan->buf[i]);  in relay_close()
830 	if (chan->last_toobig)  in relay_close()
833 		       chan->last_toobig, chan->subbuf_size);  in relay_close()
835 	list_del(&chan->list);  in relay_close()
836 	kref_put(&chan->kref, relay_destroy_channel);  in relay_close()
847 void relay_flush(struct rchan *chan)  in relay_flush()  argument
851 	if (!chan)  in relay_flush()
854 	if (chan->is_global && chan->buf[0]) {  in relay_flush()
855 		relay_switch_subbuf(chan->buf[0], 0);  in relay_flush()
861 		if (chan->buf[i])  in relay_flush()
862 			relay_switch_subbuf(chan->buf[i], 0);  in relay_flush()
943 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_consume()
944 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_consume()
952 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
960 		read_subbuf = read_pos / buf->chan->subbuf_size;  in relay_file_read_consume()
965 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
975 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_avail()
976 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_avail()
1022 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_subbuf_avail()
1052 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_start_pos()
1053 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_start_pos()
1081 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_end_pos()
1082 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_end_pos()
1181 	if (rbuf->bytes_consumed >= rbuf->chan->subbuf_size) {  in relay_consume_bytes()
1182 		relay_subbufs_consumed(rbuf->chan, rbuf->cpu, 1);  in relay_consume_bytes()
1183 		rbuf->bytes_consumed %= rbuf->chan->subbuf_size;  in relay_consume_bytes()
1220 	unsigned int subbuf_size = rbuf->chan->subbuf_size;  in subbuf_splice_actor()
1222 	uint32_t alloc_size = (uint32_t) rbuf->chan->alloc_size;  in subbuf_splice_actor()
1251 	subbuf_pages = rbuf->chan->alloc_size >> PAGE_SHIFT;  in subbuf_splice_actor()