292 	struct cfqg_stats stats;	/* stats for this cfqg */  member
469 static inline void cfqg_stats_mark_##name(struct cfqg_stats *stats)	\
471 	stats->flags |= (1 << CFQG_stats_##name);			\
473 static inline void cfqg_stats_clear_##name(struct cfqg_stats *stats)	\
475 	stats->flags &= ~(1 << CFQG_stats_##name);			\
477 static inline int cfqg_stats_##name(struct cfqg_stats *stats)		\
479 	return (stats->flags & (1 << CFQG_stats_##name)) != 0;		\
488 static void cfqg_stats_update_group_wait_time(struct cfqg_stats *stats)  in CFQG_FLAG_FNS()
492 	if (!cfqg_stats_waiting(stats))  in CFQG_FLAG_FNS()
496 	if (time_after64(now, stats->start_group_wait_time))  in CFQG_FLAG_FNS()
497 		blkg_stat_add(&stats->group_wait_time,  in CFQG_FLAG_FNS()
498 			      now - stats->start_group_wait_time);  in CFQG_FLAG_FNS()
499 	cfqg_stats_clear_waiting(stats);  in CFQG_FLAG_FNS()
506 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_set_start_group_wait_time()  local
508 	if (cfqg_stats_waiting(stats))  in cfqg_stats_set_start_group_wait_time()
512 	stats->start_group_wait_time = sched_clock();  in cfqg_stats_set_start_group_wait_time()
513 	cfqg_stats_mark_waiting(stats);  in cfqg_stats_set_start_group_wait_time()
517 static void cfqg_stats_end_empty_time(struct cfqg_stats *stats)  in cfqg_stats_end_empty_time()  argument
521 	if (!cfqg_stats_empty(stats))  in cfqg_stats_end_empty_time()
525 	if (time_after64(now, stats->start_empty_time))  in cfqg_stats_end_empty_time()
526 		blkg_stat_add(&stats->empty_time,  in cfqg_stats_end_empty_time()
527 			      now - stats->start_empty_time);  in cfqg_stats_end_empty_time()
528 	cfqg_stats_clear_empty(stats);  in cfqg_stats_end_empty_time()
533 	blkg_stat_add(&cfqg->stats.dequeue, 1);  in cfqg_stats_update_dequeue()
538 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_set_start_empty_time()  local
540 	if (blkg_rwstat_total(&stats->queued))  in cfqg_stats_set_start_empty_time()
548 	if (cfqg_stats_empty(stats))  in cfqg_stats_set_start_empty_time()
551 	stats->start_empty_time = sched_clock();  in cfqg_stats_set_start_empty_time()
552 	cfqg_stats_mark_empty(stats);  in cfqg_stats_set_start_empty_time()
557 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_update_idle_time()  local
559 	if (cfqg_stats_idling(stats)) {  in cfqg_stats_update_idle_time()
562 		if (time_after64(now, stats->start_idle_time))  in cfqg_stats_update_idle_time()
563 			blkg_stat_add(&stats->idle_time,  in cfqg_stats_update_idle_time()
564 				      now - stats->start_idle_time);  in cfqg_stats_update_idle_time()
565 		cfqg_stats_clear_idling(stats);  in cfqg_stats_update_idle_time()
571 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_set_start_idle_time()  local
573 	BUG_ON(cfqg_stats_idling(stats));  in cfqg_stats_set_start_idle_time()
575 	stats->start_idle_time = sched_clock();  in cfqg_stats_set_start_idle_time()
576 	cfqg_stats_mark_idling(stats);  in cfqg_stats_set_start_idle_time()
581 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_update_avg_queue_size()  local
583 	blkg_stat_add(&stats->avg_queue_size_sum,  in cfqg_stats_update_avg_queue_size()
584 		      blkg_rwstat_total(&stats->queued));  in cfqg_stats_update_avg_queue_size()
585 	blkg_stat_add(&stats->avg_queue_size_samples, 1);  in cfqg_stats_update_avg_queue_size()
586 	cfqg_stats_update_group_wait_time(stats);  in cfqg_stats_update_avg_queue_size()
592 static inline void cfqg_stats_end_empty_time(struct cfqg_stats *stats) { }  in cfqg_stats_end_empty_time()  argument
647 	blkg_rwstat_add(&cfqg->stats.queued, rw, 1);  in cfqg_stats_update_io_add()
648 	cfqg_stats_end_empty_time(&cfqg->stats);  in cfqg_stats_update_io_add()
655 	blkg_stat_add(&cfqg->stats.time, time);  in cfqg_stats_update_timeslice_used()
657 	blkg_stat_add(&cfqg->stats.unaccounted_time, unaccounted_time);  in cfqg_stats_update_timeslice_used()
663 	blkg_rwstat_add(&cfqg->stats.queued, rw, -1);  in cfqg_stats_update_io_remove()
668 	blkg_rwstat_add(&cfqg->stats.merged, rw, 1);  in cfqg_stats_update_io_merged()
674 	blkg_stat_add(&cfqg->stats.sectors, bytes >> 9);  in cfqg_stats_update_dispatch()
675 	blkg_rwstat_add(&cfqg->stats.serviced, rw, 1);  in cfqg_stats_update_dispatch()
676 	blkg_rwstat_add(&cfqg->stats.service_bytes, rw, bytes);  in cfqg_stats_update_dispatch()
682 	struct cfqg_stats *stats = &cfqg->stats;  in cfqg_stats_update_completion()  local
686 		blkg_rwstat_add(&stats->service_time, rw, now - io_start_time);  in cfqg_stats_update_completion()
688 		blkg_rwstat_add(&stats->wait_time, rw,  in cfqg_stats_update_completion()
693 static void cfqg_stats_reset(struct cfqg_stats *stats)  in cfqg_stats_reset()  argument
696 	blkg_rwstat_reset(&stats->service_bytes);  in cfqg_stats_reset()
697 	blkg_rwstat_reset(&stats->serviced);  in cfqg_stats_reset()
698 	blkg_rwstat_reset(&stats->merged);  in cfqg_stats_reset()
699 	blkg_rwstat_reset(&stats->service_time);  in cfqg_stats_reset()
700 	blkg_rwstat_reset(&stats->wait_time);  in cfqg_stats_reset()
701 	blkg_stat_reset(&stats->time);  in cfqg_stats_reset()
703 	blkg_stat_reset(&stats->unaccounted_time);  in cfqg_stats_reset()
704 	blkg_stat_reset(&stats->avg_queue_size_sum);  in cfqg_stats_reset()
705 	blkg_stat_reset(&stats->avg_queue_size_samples);  in cfqg_stats_reset()
706 	blkg_stat_reset(&stats->dequeue);  in cfqg_stats_reset()
707 	blkg_stat_reset(&stats->group_wait_time);  in cfqg_stats_reset()
708 	blkg_stat_reset(&stats->idle_time);  in cfqg_stats_reset()
709 	blkg_stat_reset(&stats->empty_time);  in cfqg_stats_reset()
748 	cfqg_stats_merge(&parent->dead_stats, &cfqg->stats);  in cfqg_stats_xfer_dead()
750 	cfqg_stats_reset(&cfqg->stats);  in cfqg_stats_xfer_dead()
1524 static void cfqg_stats_init(struct cfqg_stats *stats)  in cfqg_stats_init()  argument
1526 	blkg_rwstat_init(&stats->service_bytes);  in cfqg_stats_init()
1527 	blkg_rwstat_init(&stats->serviced);  in cfqg_stats_init()
1528 	blkg_rwstat_init(&stats->merged);  in cfqg_stats_init()
1529 	blkg_rwstat_init(&stats->service_time);  in cfqg_stats_init()
1530 	blkg_rwstat_init(&stats->wait_time);  in cfqg_stats_init()
1531 	blkg_rwstat_init(&stats->queued);  in cfqg_stats_init()
1533 	blkg_stat_init(&stats->sectors);  in cfqg_stats_init()
1534 	blkg_stat_init(&stats->time);  in cfqg_stats_init()
1537 	blkg_stat_init(&stats->unaccounted_time);  in cfqg_stats_init()
1538 	blkg_stat_init(&stats->avg_queue_size_sum);  in cfqg_stats_init()
1539 	blkg_stat_init(&stats->avg_queue_size_samples);  in cfqg_stats_init()
1540 	blkg_stat_init(&stats->dequeue);  in cfqg_stats_init()
1541 	blkg_stat_init(&stats->group_wait_time);  in cfqg_stats_init()
1542 	blkg_stat_init(&stats->idle_time);  in cfqg_stats_init()
1543 	blkg_stat_init(&stats->empty_time);  in cfqg_stats_init()
1554 	cfqg_stats_init(&cfqg->stats);  in cfq_pd_init()
1571 					offsetof(struct cfq_group, stats);
1599 	cfqg_stats_reset(&cfqg->stats);  in cfq_pd_reset_stats()
1826 	u64 samples = blkg_stat_read(&cfqg->stats.avg_queue_size_samples);  in cfqg_prfill_avg_queue_size()
1830 		v = blkg_stat_read(&cfqg->stats.avg_queue_size_sum);  in cfqg_prfill_avg_queue_size()
1890 		.private = offsetof(struct cfq_group, stats.time),
1895 		.private = offsetof(struct cfq_group, stats.sectors),
1900 		.private = offsetof(struct cfq_group, stats.service_bytes),
1905 		.private = offsetof(struct cfq_group, stats.serviced),
1910 		.private = offsetof(struct cfq_group, stats.service_time),
1915 		.private = offsetof(struct cfq_group, stats.wait_time),
1920 		.private = offsetof(struct cfq_group, stats.merged),
1925 		.private = offsetof(struct cfq_group, stats.queued),
1932 		.private = offsetof(struct cfq_group, stats.time),
1937 		.private = offsetof(struct cfq_group, stats.sectors),
1942 		.private = offsetof(struct cfq_group, stats.service_bytes),
1947 		.private = offsetof(struct cfq_group, stats.serviced),
1952 		.private = offsetof(struct cfq_group, stats.service_time),
1957 		.private = offsetof(struct cfq_group, stats.wait_time),
1962 		.private = offsetof(struct cfq_group, stats.merged),
1967 		.private = offsetof(struct cfq_group, stats.queued),
1977 		.private = offsetof(struct cfq_group, stats.group_wait_time),
1982 		.private = offsetof(struct cfq_group, stats.idle_time),
1987 		.private = offsetof(struct cfq_group, stats.empty_time),
1992 		.private = offsetof(struct cfq_group, stats.dequeue),
1997 		.private = offsetof(struct cfq_group, stats.unaccounted_time),