68 static void trace_note(struct blk_trace *bt, pid_t pid, int action,  in trace_note()  argument
90 	if (!bt->rchan)  in trace_note()
93 	t = relay_reserve(bt->rchan, sizeof(*t) + len);  in trace_note()
98 		t->device = bt->dev;  in trace_note()
117 	struct blk_trace *bt;  in trace_note_tsk()  local
121 	list_for_each_entry(bt, &running_trace_list, running_list) {  in trace_note_tsk()
122 		trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm,  in trace_note_tsk()
128 static void trace_note_time(struct blk_trace *bt)  in trace_note_time()  argument
139 	trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words));  in trace_note_time()
143 void __trace_note_message(struct blk_trace *bt, const char *fmt, ...)  in __trace_note_message()  argument
150 	if (unlikely(bt->trace_state != Blktrace_running &&  in __trace_note_message()
158 	if (!(bt->act_mask & BLK_TC_NOTIFY))  in __trace_note_message()
162 	buf = this_cpu_ptr(bt->msg_data);  in __trace_note_message()
167 	trace_note(bt, 0, BLK_TN_MESSAGE, buf, n);  in __trace_note_message()
172 static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,  in act_log_check()  argument
175 	if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)  in act_log_check()
177 	if (sector && (sector < bt->start_lba || sector > bt->end_lba))  in act_log_check()
179 	if (bt->pid && pid != bt->pid)  in act_log_check()
201 static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,  in __blk_add_trace()  argument
214 	if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))  in __blk_add_trace()
226 	if (act_log_check(bt, what, sector, pid))  in __blk_add_trace()
253 	t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);  in __blk_add_trace()
255 		sequence = per_cpu_ptr(bt->sequence, cpu);  in __blk_add_trace()
273 		t->device = bt->dev;  in __blk_add_trace()
292 static void blk_trace_free(struct blk_trace *bt)  in blk_trace_free()  argument
294 	debugfs_remove(bt->msg_file);  in blk_trace_free()
295 	debugfs_remove(bt->dropped_file);  in blk_trace_free()
296 	relay_close(bt->rchan);  in blk_trace_free()
297 	debugfs_remove(bt->dir);  in blk_trace_free()
298 	free_percpu(bt->sequence);  in blk_trace_free()
299 	free_percpu(bt->msg_data);  in blk_trace_free()
300 	kfree(bt);  in blk_trace_free()
303 static void blk_trace_cleanup(struct blk_trace *bt)  in blk_trace_cleanup()  argument
305 	blk_trace_free(bt);  in blk_trace_cleanup()
312 	struct blk_trace *bt;  in blk_trace_remove()  local
314 	bt = xchg(&q->blk_trace, NULL);  in blk_trace_remove()
315 	if (!bt)  in blk_trace_remove()
318 	if (bt->trace_state != Blktrace_running)  in blk_trace_remove()
319 		blk_trace_cleanup(bt);  in blk_trace_remove()
328 	struct blk_trace *bt = filp->private_data;  in blk_dropped_read()  local
331 	snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));  in blk_dropped_read()
347 	struct blk_trace *bt;  in blk_msg_write()  local
362 	bt = filp->private_data;  in blk_msg_write()
363 	__trace_note_message(bt, "%s", msg);  in blk_msg_write()
383 	struct blk_trace *bt;  in blk_subbuf_start_callback()  local
388 	bt = buf->chan->private_data;  in blk_subbuf_start_callback()
389 	atomic_inc(&bt->dropped);  in blk_subbuf_start_callback()
416 static void blk_trace_setup_lba(struct blk_trace *bt,  in blk_trace_setup_lba()  argument
425 		bt->start_lba = part->start_sect;  in blk_trace_setup_lba()
426 		bt->end_lba = part->start_sect + part->nr_sects;  in blk_trace_setup_lba()
428 		bt->start_lba = 0;  in blk_trace_setup_lba()
429 		bt->end_lba = -1ULL;  in blk_trace_setup_lba()
440 	struct blk_trace *bt = NULL;  in do_blk_trace_setup()  local
456 	bt = kzalloc(sizeof(*bt), GFP_KERNEL);  in do_blk_trace_setup()
457 	if (!bt)  in do_blk_trace_setup()
461 	bt->sequence = alloc_percpu(unsigned long);  in do_blk_trace_setup()
462 	if (!bt->sequence)  in do_blk_trace_setup()
465 	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));  in do_blk_trace_setup()
466 	if (!bt->msg_data)  in do_blk_trace_setup()
486 	bt->dir = dir;  in do_blk_trace_setup()
487 	bt->dev = dev;  in do_blk_trace_setup()
488 	atomic_set(&bt->dropped, 0);  in do_blk_trace_setup()
489 	INIT_LIST_HEAD(&bt->running_list);  in do_blk_trace_setup()
492 	bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt,  in do_blk_trace_setup()
494 	if (!bt->dropped_file)  in do_blk_trace_setup()
497 	bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);  in do_blk_trace_setup()
498 	if (!bt->msg_file)  in do_blk_trace_setup()
501 	bt->rchan = relay_open("trace", dir, buts->buf_size,  in do_blk_trace_setup()
502 				buts->buf_nr, &blk_relay_callbacks, bt);  in do_blk_trace_setup()
503 	if (!bt->rchan)  in do_blk_trace_setup()
506 	bt->act_mask = buts->act_mask;  in do_blk_trace_setup()
507 	if (!bt->act_mask)  in do_blk_trace_setup()
508 		bt->act_mask = (u16) -1;  in do_blk_trace_setup()
510 	blk_trace_setup_lba(bt, bdev);  in do_blk_trace_setup()
514 		bt->start_lba = buts->start_lba;  in do_blk_trace_setup()
516 		bt->end_lba = buts->end_lba;  in do_blk_trace_setup()
518 	bt->pid = buts->pid;  in do_blk_trace_setup()
519 	bt->trace_state = Blktrace_setup;  in do_blk_trace_setup()
522 	if (cmpxchg(&q->blk_trace, NULL, bt))  in do_blk_trace_setup()
530 	blk_trace_free(bt);  in do_blk_trace_setup()
594 	struct blk_trace *bt = q->blk_trace;  in blk_trace_startstop()  local
596 	if (bt == NULL)  in blk_trace_startstop()
605 		if (bt->trace_state == Blktrace_setup ||  in blk_trace_startstop()
606 		    bt->trace_state == Blktrace_stopped) {  in blk_trace_startstop()
609 			bt->trace_state = Blktrace_running;  in blk_trace_startstop()
611 			list_add(&bt->running_list, &running_trace_list);  in blk_trace_startstop()
614 			trace_note_time(bt);  in blk_trace_startstop()
618 		if (bt->trace_state == Blktrace_running) {  in blk_trace_startstop()
619 			bt->trace_state = Blktrace_stopped;  in blk_trace_startstop()
621 			list_del_init(&bt->running_list);  in blk_trace_startstop()
623 			relay_flush(bt->rchan);  in blk_trace_startstop()
710 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_rq()  local
712 	if (likely(!bt))  in blk_add_trace_rq()
717 		__blk_add_trace(bt, 0, nr_bytes, rq->cmd_flags,  in blk_add_trace_rq()
721 		__blk_add_trace(bt, blk_rq_pos(rq), nr_bytes,  in blk_add_trace_rq()
773 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_bio()  local
775 	if (likely(!bt))  in blk_add_trace_bio()
778 	__blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,  in blk_add_trace_bio()
824 		struct blk_trace *bt = q->blk_trace;  in blk_add_trace_getrq()  local
826 		if (bt)  in blk_add_trace_getrq()
827 			__blk_add_trace(bt, 0, 0, rw, BLK_TA_GETRQ, 0, 0, NULL);  in blk_add_trace_getrq()
839 		struct blk_trace *bt = q->blk_trace;  in blk_add_trace_sleeprq()  local
841 		if (bt)  in blk_add_trace_sleeprq()
842 			__blk_add_trace(bt, 0, 0, rw, BLK_TA_SLEEPRQ,  in blk_add_trace_sleeprq()
849 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_plug()  local
851 	if (bt)  in blk_add_trace_plug()
852 		__blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL);  in blk_add_trace_plug()
858 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_unplug()  local
860 	if (bt) {  in blk_add_trace_unplug()
869 		__blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu);  in blk_add_trace_unplug()
877 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_split()  local
879 	if (bt) {  in blk_add_trace_split()
882 		__blk_add_trace(bt, bio->bi_iter.bi_sector,  in blk_add_trace_split()
905 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_bio_remap()  local
908 	if (likely(!bt))  in blk_add_trace_bio_remap()
915 	__blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,  in blk_add_trace_bio_remap()
938 	struct blk_trace *bt = q->blk_trace;  in blk_add_trace_rq_remap()  local
941 	if (likely(!bt))  in blk_add_trace_rq_remap()
948 	__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),  in blk_add_trace_rq_remap()
968 	struct blk_trace *bt = q->blk_trace;  in blk_add_driver_data()  local
970 	if (likely(!bt))  in blk_add_driver_data()
974 		__blk_add_trace(bt, 0, blk_rq_bytes(rq), 0,  in blk_add_driver_data()
977 		__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 0,  in blk_add_driver_data()
1463 	struct blk_trace *bt;  in blk_trace_remove_queue()  local
1465 	bt = xchg(&q->blk_trace, NULL);  in blk_trace_remove_queue()
1466 	if (bt == NULL)  in blk_trace_remove_queue()
1472 	blk_trace_free(bt);  in blk_trace_remove_queue()
1482 	struct blk_trace *bt = NULL;  in blk_trace_setup_queue()  local
1485 	bt = kzalloc(sizeof(*bt), GFP_KERNEL);  in blk_trace_setup_queue()
1486 	if (!bt)  in blk_trace_setup_queue()
1489 	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));  in blk_trace_setup_queue()
1490 	if (!bt->msg_data)  in blk_trace_setup_queue()
1493 	bt->dev = bdev->bd_dev;  in blk_trace_setup_queue()
1494 	bt->act_mask = (u16)-1;  in blk_trace_setup_queue()
1496 	blk_trace_setup_lba(bt, bdev);  in blk_trace_setup_queue()
1499 	if (cmpxchg(&q->blk_trace, NULL, bt))  in blk_trace_setup_queue()
1507 	blk_trace_free(bt);  in blk_trace_setup_queue()