68 static void trace_note(struct blk_trace *bt, pid_t pid, int action,  in trace_note()  argument
90 	if (!bt->rchan)  in trace_note()
93 	t = relay_reserve(bt->rchan, sizeof(*t) + len);  in trace_note()
98 		t->device = bt->dev;  in trace_note()
117 	struct blk_trace *bt;  in trace_note_tsk()  local
121 	list_for_each_entry(bt, &running_trace_list, running_list) {  in trace_note_tsk()
122 		trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm,  in trace_note_tsk()
128 static void trace_note_time(struct blk_trace *bt)  in trace_note_time()  argument
139 	trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words));  in trace_note_time()
143 void __trace_note_message(struct blk_trace *bt, const char *fmt, ...)  in __trace_note_message()  argument
150 	if (unlikely(bt->trace_state != Blktrace_running &&  in __trace_note_message()
158 	if (!(bt->act_mask & BLK_TC_NOTIFY))  in __trace_note_message()
162 	buf = this_cpu_ptr(bt->msg_data);  in __trace_note_message()
167 	trace_note(bt, 0, BLK_TN_MESSAGE, buf, n);  in __trace_note_message()
172 static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,  in act_log_check()  argument
175 	if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)  in act_log_check()
177 	if (sector && (sector < bt->start_lba || sector > bt->end_lba))  in act_log_check()
179 	if (bt->pid && pid != bt->pid)  in act_log_check()
201 static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,  in __blk_add_trace()  argument
214 	if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))  in __blk_add_trace()
226 	if (act_log_check(bt, what, sector, pid))  in __blk_add_trace()
253 	t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);  in __blk_add_trace()
255 		sequence = per_cpu_ptr(bt->sequence, cpu);  in __blk_add_trace()
273 		t->device = bt->dev;  in __blk_add_trace()
292 static void blk_trace_free(struct blk_trace *bt)  in blk_trace_free()  argument
294 	debugfs_remove(bt->msg_file);  in blk_trace_free()
295 	debugfs_remove(bt->dropped_file);  in blk_trace_free()
296 	relay_close(bt->rchan);  in blk_trace_free()
297 	debugfs_remove(bt->dir);  in blk_trace_free()
298 	free_percpu(bt->sequence);  in blk_trace_free()
299 	free_percpu(bt->msg_data);  in blk_trace_free()
300 	kfree(bt);  in blk_trace_free()
303 static void blk_trace_cleanup(struct blk_trace *bt)  in blk_trace_cleanup()  argument
305 	blk_trace_free(bt);  in blk_trace_cleanup()
312 	struct blk_trace *bt;  in blk_trace_remove()  local
314 	bt = xchg(&q->blk_trace, NULL);  in blk_trace_remove()
315 	if (!bt)  in blk_trace_remove()
318 	if (bt->trace_state != Blktrace_running)  in blk_trace_remove()
319 		blk_trace_cleanup(bt);  in blk_trace_remove()
328 	struct blk_trace *bt = filp->private_data;  in blk_dropped_read()  local
331 	snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));  in blk_dropped_read()
347 	struct blk_trace *bt;  in blk_msg_write()  local
362 	bt = filp->private_data;  in blk_msg_write()
363 	__trace_note_message(bt, "%s", msg);  in blk_msg_write()
383 	struct blk_trace *bt;  in blk_subbuf_start_callback()  local
388 	bt = buf->chan->private_data;  in blk_subbuf_start_callback()
389 	atomic_inc(&bt->dropped);  in blk_subbuf_start_callback()
416 static void blk_trace_setup_lba(struct blk_trace *bt,  in blk_trace_setup_lba()  argument
425 		bt->start_lba = part->start_sect;  in blk_trace_setup_lba()
426 		bt->end_lba = part->start_sect + part->nr_sects;  in blk_trace_setup_lba()
428 		bt->start_lba = 0;  in blk_trace_setup_lba()
429 		bt->end_lba = -1ULL;  in blk_trace_setup_lba()
440 	struct blk_trace *old_bt, *bt = NULL;  in do_blk_trace_setup()  local
458 	bt = kzalloc(sizeof(*bt), GFP_KERNEL);  in do_blk_trace_setup()
459 	if (!bt)  in do_blk_trace_setup()
463 	bt->sequence = alloc_percpu(unsigned long);  in do_blk_trace_setup()
464 	if (!bt->sequence)  in do_blk_trace_setup()
467 	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));  in do_blk_trace_setup()
468 	if (!bt->msg_data)  in do_blk_trace_setup()
488 	bt->dir = dir;  in do_blk_trace_setup()
489 	bt->dev = dev;  in do_blk_trace_setup()
490 	atomic_set(&bt->dropped, 0);  in do_blk_trace_setup()
491 	INIT_LIST_HEAD(&bt->running_list);  in do_blk_trace_setup()
494 	bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt,  in do_blk_trace_setup()
496 	if (!bt->dropped_file)  in do_blk_trace_setup()
499 	bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);  in do_blk_trace_setup()
500 	if (!bt->msg_file)  in do_blk_trace_setup()
503 	bt->rchan = relay_open("trace", dir, buts->buf_size,  in do_blk_trace_setup()
504 				buts->buf_nr, &blk_relay_callbacks, bt);  in do_blk_trace_setup()
505 	if (!bt->rchan)  in do_blk_trace_setup()
508 	bt->act_mask = buts->act_mask;  in do_blk_trace_setup()
509 	if (!bt->act_mask)  in do_blk_trace_setup()
510 		bt->act_mask = (u16) -1;  in do_blk_trace_setup()
512 	blk_trace_setup_lba(bt, bdev);  in do_blk_trace_setup()
516 		bt->start_lba = buts->start_lba;  in do_blk_trace_setup()
518 		bt->end_lba = buts->end_lba;  in do_blk_trace_setup()
520 	bt->pid = buts->pid;  in do_blk_trace_setup()
521 	bt->trace_state = Blktrace_setup;  in do_blk_trace_setup()
524 	old_bt = xchg(&q->blk_trace, bt);  in do_blk_trace_setup()
535 	blk_trace_free(bt);  in do_blk_trace_setup()
599 	struct blk_trace *bt = q->blk_trace;  in blk_trace_startstop()  local
601 	if (bt == NULL)  in blk_trace_startstop()
610 		if (bt->trace_state == Blktrace_setup ||  in blk_trace_startstop()
611 		    bt->trace_state == Blktrace_stopped) {  in blk_trace_startstop()
614 			bt->trace_state = Blktrace_running;  in blk_trace_startstop()
616 			list_add(&bt->running_list, &running_trace_list);  in blk_trace_startstop()
619 			trace_note_time(bt);  in blk_trace_startstop()
623 		if (bt->trace_state == Blktrace_running) {  in blk_trace_startstop()
624 			bt->trace_state = Blktrace_stopped;  in blk_trace_startstop()
626 			list_del_init(&bt->running_list);  in blk_trace_startstop()
628 			relay_flush(bt->rchan);  in blk_trace_startstop()
715 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_rq()  local
717 	if (likely(!bt))  in blk_add_trace_rq()
722 		__blk_add_trace(bt, 0, nr_bytes, rq->cmd_flags,  in blk_add_trace_rq()
726 		__blk_add_trace(bt, blk_rq_pos(rq), nr_bytes,  in blk_add_trace_rq()
778 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_bio()  local
780 	if (likely(!bt))  in blk_add_trace_bio()
786 	__blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,  in blk_add_trace_bio()
832 		struct blk_trace *bt = q->blk_trace;  in blk_add_trace_getrq()  local
834 		if (bt)  in blk_add_trace_getrq()
835 			__blk_add_trace(bt, 0, 0, rw, BLK_TA_GETRQ, 0, 0, NULL);  in blk_add_trace_getrq()
847 		struct blk_trace *bt = q->blk_trace;  in blk_add_trace_sleeprq()  local
849 		if (bt)  in blk_add_trace_sleeprq()
850 			__blk_add_trace(bt, 0, 0, rw, BLK_TA_SLEEPRQ,  in blk_add_trace_sleeprq()
857 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_plug()  local
859 	if (bt)  in blk_add_trace_plug()
860 		__blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL);  in blk_add_trace_plug()
866 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_unplug()  local
868 	if (bt) {  in blk_add_trace_unplug()
877 		__blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu);  in blk_add_trace_unplug()
885 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_split()  local
887 	if (bt) {  in blk_add_trace_split()
890 		__blk_add_trace(bt, bio->bi_iter.bi_sector,  in blk_add_trace_split()
914 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_bio_remap()  local
917 	if (likely(!bt))  in blk_add_trace_bio_remap()
924 	__blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,  in blk_add_trace_bio_remap()
947 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_rq_remap()  local
950 	if (likely(!bt))  in blk_add_trace_rq_remap()
957 	__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),  in blk_add_trace_rq_remap()
977 	struct blk_trace *bt = q->blk_trace;  in blk_add_driver_data()  local
979 	if (likely(!bt))  in blk_add_driver_data()
983 		__blk_add_trace(bt, 0, blk_rq_bytes(rq), 0,  in blk_add_driver_data()
986 		__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 0,  in blk_add_driver_data()
1471 	struct blk_trace *bt;  in blk_trace_remove_queue()  local
1473 	bt = xchg(&q->blk_trace, NULL);  in blk_trace_remove_queue()
1474 	if (bt == NULL)  in blk_trace_remove_queue()
1480 	blk_trace_free(bt);  in blk_trace_remove_queue()
1490 	struct blk_trace *old_bt, *bt = NULL;  in blk_trace_setup_queue()  local
1493 	bt = kzalloc(sizeof(*bt), GFP_KERNEL);  in blk_trace_setup_queue()
1494 	if (!bt)  in blk_trace_setup_queue()
1497 	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));  in blk_trace_setup_queue()
1498 	if (!bt->msg_data)  in blk_trace_setup_queue()
1501 	bt->dev = bdev->bd_dev;  in blk_trace_setup_queue()
1502 	bt->act_mask = (u16)-1;  in blk_trace_setup_queue()
1504 	blk_trace_setup_lba(bt, bdev);  in blk_trace_setup_queue()
1506 	old_bt = xchg(&q->blk_trace, bt);  in blk_trace_setup_queue()
1518 	blk_trace_free(bt);  in blk_trace_setup_queue()