1884 rb_iter_head_event(struct ring_buffer_iter *iter)  in rb_iter_head_event()  argument
1886 	return __rb_page_index(iter->head_page, iter->head);  in rb_iter_head_event()
1987 static void rb_inc_iter(struct ring_buffer_iter *iter)  in rb_inc_iter()  argument
1989 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in rb_inc_iter()
1997 	if (iter->head_page == cpu_buffer->reader_page)  in rb_inc_iter()
1998 		iter->head_page = rb_set_head_page(cpu_buffer);  in rb_inc_iter()
2000 		rb_inc_page(cpu_buffer, &iter->head_page);  in rb_inc_iter()
2002 	iter->read_stamp = iter->head_page->page->time_stamp;  in rb_inc_iter()
2003 	iter->head = 0;  in rb_inc_iter()
3402 static void rb_iter_reset(struct ring_buffer_iter *iter)  in rb_iter_reset()  argument
3404 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in rb_iter_reset()
3407 	iter->head_page = cpu_buffer->reader_page;  in rb_iter_reset()
3408 	iter->head = cpu_buffer->reader_page->read;  in rb_iter_reset()
3410 	iter->cache_reader_page = iter->head_page;  in rb_iter_reset()
3411 	iter->cache_read = cpu_buffer->read;  in rb_iter_reset()
3413 	if (iter->head)  in rb_iter_reset()
3414 		iter->read_stamp = cpu_buffer->read_stamp;  in rb_iter_reset()
3416 		iter->read_stamp = iter->head_page->page->time_stamp;  in rb_iter_reset()
3426 void ring_buffer_iter_reset(struct ring_buffer_iter *iter)  in ring_buffer_iter_reset()  argument
3431 	if (!iter)  in ring_buffer_iter_reset()
3434 	cpu_buffer = iter->cpu_buffer;  in ring_buffer_iter_reset()
3437 	rb_iter_reset(iter);  in ring_buffer_iter_reset()
3446 int ring_buffer_iter_empty(struct ring_buffer_iter *iter)  in ring_buffer_iter_empty()  argument
3450 	cpu_buffer = iter->cpu_buffer;  in ring_buffer_iter_empty()
3452 	return iter->head_page == cpu_buffer->commit_page &&  in ring_buffer_iter_empty()
3453 		iter->head == rb_commit_index(cpu_buffer);  in ring_buffer_iter_empty()
3489 rb_update_iter_read_stamp(struct ring_buffer_iter *iter,  in rb_update_iter_read_stamp()  argument
3502 		iter->read_stamp += delta;  in rb_update_iter_read_stamp()
3510 		iter->read_stamp += event->time_delta;  in rb_update_iter_read_stamp()
3671 static void rb_advance_iter(struct ring_buffer_iter *iter)  in rb_advance_iter()  argument
3677 	cpu_buffer = iter->cpu_buffer;  in rb_advance_iter()
3682 	if (iter->head >= rb_page_size(iter->head_page)) {  in rb_advance_iter()
3684 		if (iter->head_page == cpu_buffer->commit_page)  in rb_advance_iter()
3686 		rb_inc_iter(iter);  in rb_advance_iter()
3690 	event = rb_iter_head_event(iter);  in rb_advance_iter()
3699 		       (iter->head_page == cpu_buffer->commit_page) &&  in rb_advance_iter()
3700 		       (iter->head + length > rb_commit_index(cpu_buffer))))  in rb_advance_iter()
3703 	rb_update_iter_read_stamp(iter, event);  in rb_advance_iter()
3705 	iter->head += length;  in rb_advance_iter()
3708 	if ((iter->head >= rb_page_size(iter->head_page)) &&  in rb_advance_iter()
3709 	    (iter->head_page != cpu_buffer->commit_page))  in rb_advance_iter()
3710 		rb_inc_iter(iter);  in rb_advance_iter()
3785 rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)  in rb_iter_peek()  argument
3792 	cpu_buffer = iter->cpu_buffer;  in rb_iter_peek()
3800 	if (unlikely(iter->cache_read != cpu_buffer->read ||  in rb_iter_peek()
3801 		     iter->cache_reader_page != cpu_buffer->reader_page))  in rb_iter_peek()
3802 		rb_iter_reset(iter);  in rb_iter_peek()
3805 	if (ring_buffer_iter_empty(iter))  in rb_iter_peek()
3822 	if (iter->head >= rb_page_size(iter->head_page)) {  in rb_iter_peek()
3823 		rb_inc_iter(iter);  in rb_iter_peek()
3827 	event = rb_iter_head_event(iter);  in rb_iter_peek()
3832 			rb_inc_iter(iter);  in rb_iter_peek()
3835 		rb_advance_iter(iter);  in rb_iter_peek()
3840 		rb_advance_iter(iter);  in rb_iter_peek()
3845 		rb_advance_iter(iter);  in rb_iter_peek()
3850 			*ts = iter->read_stamp + event->time_delta;  in rb_iter_peek()
3928 ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)  in ring_buffer_iter_peek()  argument
3930 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in ring_buffer_iter_peek()
3936 	event = rb_iter_peek(iter, ts);  in ring_buffer_iter_peek()
4023 	struct ring_buffer_iter *iter;  in ring_buffer_read_prepare()  local
4028 	iter = kmalloc(sizeof(*iter), GFP_KERNEL);  in ring_buffer_read_prepare()
4029 	if (!iter)  in ring_buffer_read_prepare()
4034 	iter->cpu_buffer = cpu_buffer;  in ring_buffer_read_prepare()
4039 	return iter;  in ring_buffer_read_prepare()
4069 ring_buffer_read_start(struct ring_buffer_iter *iter)  in ring_buffer_read_start()  argument
4074 	if (!iter)  in ring_buffer_read_start()
4077 	cpu_buffer = iter->cpu_buffer;  in ring_buffer_read_start()
4081 	rb_iter_reset(iter);  in ring_buffer_read_start()
4095 ring_buffer_read_finish(struct ring_buffer_iter *iter)  in ring_buffer_read_finish()  argument
4097 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in ring_buffer_read_finish()
4112 	kfree(iter);  in ring_buffer_read_finish()
4124 ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)  in ring_buffer_read()  argument
4127 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in ring_buffer_read()
4132 	event = rb_iter_peek(iter, ts);  in ring_buffer_read()
4139 	rb_advance_iter(iter);  in ring_buffer_read()