64 static void blkg_free(struct blkcg_gq *blkg)  in blkg_free()  argument
68 	if (!blkg)  in blkg_free()
72 		if (blkg->pd[i])  in blkg_free()
73 			blkcg_policy[i]->pd_free_fn(blkg->pd[i]);  in blkg_free()
75 	if (blkg->blkcg != &blkcg_root)  in blkg_free()
76 		blk_exit_rl(&blkg->rl);  in blkg_free()
78 	blkg_rwstat_exit(&blkg->stat_ios);  in blkg_free()
79 	blkg_rwstat_exit(&blkg->stat_bytes);  in blkg_free()
80 	kfree(blkg);  in blkg_free()
94 	struct blkcg_gq *blkg;  in blkg_alloc()  local
98 	blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);  in blkg_alloc()
99 	if (!blkg)  in blkg_alloc()
102 	if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||  in blkg_alloc()
103 	    blkg_rwstat_init(&blkg->stat_ios, gfp_mask))  in blkg_alloc()
106 	blkg->q = q;  in blkg_alloc()
107 	INIT_LIST_HEAD(&blkg->q_node);  in blkg_alloc()
108 	blkg->blkcg = blkcg;  in blkg_alloc()
109 	atomic_set(&blkg->refcnt, 1);  in blkg_alloc()
113 		if (blk_init_rl(&blkg->rl, q, gfp_mask))  in blkg_alloc()
115 		blkg->rl.blkg = blkg;  in blkg_alloc()
130 		blkg->pd[i] = pd;  in blkg_alloc()
131 		pd->blkg = blkg;  in blkg_alloc()
135 	return blkg;  in blkg_alloc()
138 	blkg_free(blkg);  in blkg_alloc()
145 	struct blkcg_gq *blkg;  in blkg_lookup_slowpath()  local
153 	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);  in blkg_lookup_slowpath()
154 	if (blkg && blkg->q == q) {  in blkg_lookup_slowpath()
157 			rcu_assign_pointer(blkcg->blkg_hint, blkg);  in blkg_lookup_slowpath()
159 		return blkg;  in blkg_lookup_slowpath()
174 	struct blkcg_gq *blkg;  in blkg_create()  local
202 	blkg = new_blkg;  in blkg_create()
203 	blkg->wb_congested = wb_congested;  in blkg_create()
207 		blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);  in blkg_create()
208 		if (WARN_ON_ONCE(!blkg->parent)) {  in blkg_create()
212 		blkg_get(blkg->parent);  in blkg_create()
219 		if (blkg->pd[i] && pol->pd_init_fn)  in blkg_create()
220 			pol->pd_init_fn(blkg->pd[i]);  in blkg_create()
225 	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);  in blkg_create()
227 		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);  in blkg_create()
228 		list_add(&blkg->q_node, &q->blkg_list);  in blkg_create()
233 			if (blkg->pd[i] && pol->pd_online_fn)  in blkg_create()
234 				pol->pd_online_fn(blkg->pd[i]);  in blkg_create()
237 	blkg->online = true;  in blkg_create()
241 		return blkg;  in blkg_create()
244 	blkg_put(blkg);  in blkg_create()
273 	struct blkcg_gq *blkg;  in blkg_lookup_create()  local
285 	blkg = __blkg_lookup(blkcg, q, true);  in blkg_lookup_create()
286 	if (blkg)  in blkg_lookup_create()
287 		return blkg;  in blkg_lookup_create()
302 		blkg = blkg_create(pos, q, NULL);  in blkg_lookup_create()
303 		if (pos == blkcg || IS_ERR(blkg))  in blkg_lookup_create()
304 			return blkg;  in blkg_lookup_create()
308 static void blkg_destroy(struct blkcg_gq *blkg)  in blkg_destroy()  argument
310 	struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy()
311 	struct blkcg_gq *parent = blkg->parent;  in blkg_destroy()
314 	lockdep_assert_held(blkg->q->queue_lock);  in blkg_destroy()
318 	WARN_ON_ONCE(list_empty(&blkg->q_node));  in blkg_destroy()
319 	WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));  in blkg_destroy()
324 		if (blkg->pd[i] && pol->pd_offline_fn)  in blkg_destroy()
325 			pol->pd_offline_fn(blkg->pd[i]);  in blkg_destroy()
329 		blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);  in blkg_destroy()
330 		blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);  in blkg_destroy()
333 	blkg->online = false;  in blkg_destroy()
335 	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);  in blkg_destroy()
336 	list_del_init(&blkg->q_node);  in blkg_destroy()
337 	hlist_del_init_rcu(&blkg->blkcg_node);  in blkg_destroy()
344 	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)  in blkg_destroy()
351 	blkg_put(blkg);  in blkg_destroy()
362 	struct blkcg_gq *blkg, *n;  in blkg_destroy_all()  local
366 	list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {  in blkg_destroy_all()
367 		struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy_all()
370 		blkg_destroy(blkg);  in blkg_destroy_all()
375 	q->root_rl.blkg = NULL;  in blkg_destroy_all()
388 	struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);  in __blkg_release_rcu()  local
391 	css_put(&blkg->blkcg->css);  in __blkg_release_rcu()
392 	if (blkg->parent)  in __blkg_release_rcu()
393 		blkg_put(blkg->parent);  in __blkg_release_rcu()
395 	wb_congested_put(blkg->wb_congested);  in __blkg_release_rcu()
397 	blkg_free(blkg);  in __blkg_release_rcu()
409 	struct blkcg_gq *blkg;  in __blk_queue_next_rl()  local
421 		blkg = container_of(rl, struct blkcg_gq, rl);  in __blk_queue_next_rl()
422 		ent = &blkg->q_node;  in __blk_queue_next_rl()
432 	blkg = container_of(ent, struct blkcg_gq, q_node);  in __blk_queue_next_rl()
433 	return &blkg->rl;  in __blk_queue_next_rl()
440 	struct blkcg_gq *blkg;  in blkcg_reset_stats()  local
451 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_reset_stats()
452 		blkg_rwstat_reset(&blkg->stat_bytes);  in blkcg_reset_stats()
453 		blkg_rwstat_reset(&blkg->stat_ios);  in blkcg_reset_stats()
458 			if (blkg->pd[i] && pol->pd_reset_stats_fn)  in blkcg_reset_stats()
459 				pol->pd_reset_stats_fn(blkg->pd[i]);  in blkcg_reset_stats()
468 const char *blkg_dev_name(struct blkcg_gq *blkg)  in blkg_dev_name()  argument
471 	if (blkg->q->backing_dev_info.dev)  in blkg_dev_name()
472 		return dev_name(blkg->q->backing_dev_info.dev);  in blkg_dev_name()
501 	struct blkcg_gq *blkg;  in blkcg_print_blkgs()  local
505 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_blkgs()
506 		spin_lock_irq(blkg->q->queue_lock);  in blkcg_print_blkgs()
507 		if (blkcg_policy_enabled(blkg->q, pol))  in blkcg_print_blkgs()
508 			total += prfill(sf, blkg->pd[pol->plid], data);  in blkcg_print_blkgs()
509 		spin_unlock_irq(blkg->q->queue_lock);  in blkcg_print_blkgs()
528 	const char *dname = blkg_dev_name(pd->blkg);  in __blkg_prfill_u64()
555 	const char *dname = blkg_dev_name(pd->blkg);  in __blkg_prfill_rwstat()
607 	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);  in blkg_prfill_rwstat_field()
650 	struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,  in blkg_prfill_rwstat_field_recursive()
698 u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,  in blkg_stat_recursive_sum()  argument
705 	lockdep_assert_held(blkg->q->queue_lock);  in blkg_stat_recursive_sum()
708 	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {  in blkg_stat_recursive_sum()
717 			stat = (void *)blkg + off;  in blkg_stat_recursive_sum()
740 struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,  in blkg_rwstat_recursive_sum()  argument
748 	lockdep_assert_held(blkg->q->queue_lock);  in blkg_rwstat_recursive_sum()
751 	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {  in blkg_rwstat_recursive_sum()
790 	struct blkcg_gq *blkg;  in blkg_conf_prep()  local
815 		blkg = blkg_lookup_create(blkcg, disk->queue);  in blkg_conf_prep()
817 		blkg = ERR_PTR(-EOPNOTSUPP);  in blkg_conf_prep()
819 	if (IS_ERR(blkg)) {  in blkg_conf_prep()
820 		ret = PTR_ERR(blkg);  in blkg_conf_prep()
838 	ctx->blkg = blkg;  in blkg_conf_prep()
863 	struct blkcg_gq *blkg;  in blkcg_print_stat()  local
867 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_stat()
872 		dname = blkg_dev_name(blkg);  in blkcg_print_stat()
876 		spin_lock_irq(blkg->q->queue_lock);  in blkcg_print_stat()
878 		rwstat = blkg_rwstat_recursive_sum(blkg, NULL,  in blkcg_print_stat()
883 		rwstat = blkg_rwstat_recursive_sum(blkg, NULL,  in blkcg_print_stat()
888 		spin_unlock_irq(blkg->q->queue_lock);  in blkcg_print_stat()
934 		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,  in blkcg_css_offline()  local
936 		struct request_queue *q = blkg->q;  in blkcg_css_offline()
939 			blkg_destroy(blkg);  in blkcg_css_offline()
1048 	struct blkcg_gq *new_blkg, *blkg;  in blkcg_init_queue()  local
1065 	blkg = blkg_create(&blkcg_root, q, new_blkg);  in blkcg_init_queue()
1072 	if (IS_ERR(blkg)) {  in blkcg_init_queue()
1074 		return PTR_ERR(blkg);  in blkcg_init_queue()
1077 	q->root_blkg = blkg;  in blkcg_init_queue()
1078 	q->root_rl.blkg = blkg;  in blkcg_init_queue()
1210 	struct blkcg_gq *blkg;  in blkcg_activate_policy()  local
1228 	list_for_each_entry(blkg, &q->blkg_list, q_node) {  in blkcg_activate_policy()
1231 		if (blkg->pd[pol->plid])  in blkcg_activate_policy()
1242 		blkg->pd[pol->plid] = pd;  in blkcg_activate_policy()
1243 		pd->blkg = blkg;  in blkcg_activate_policy()
1272 	struct blkcg_gq *blkg;  in blkcg_deactivate_policy()  local
1282 	list_for_each_entry(blkg, &q->blkg_list, q_node) {  in blkcg_deactivate_policy()
1284 		spin_lock(&blkg->blkcg->lock);  in blkcg_deactivate_policy()
1286 		if (blkg->pd[pol->plid]) {  in blkcg_deactivate_policy()
1288 				pol->pd_offline_fn(blkg->pd[pol->plid]);  in blkcg_deactivate_policy()
1289 			pol->pd_free_fn(blkg->pd[pol->plid]);  in blkcg_deactivate_policy()
1290 			blkg->pd[pol->plid] = NULL;  in blkcg_deactivate_policy()
1293 		spin_unlock(&blkg->blkcg->lock);  in blkcg_deactivate_policy()