47 static void blkg_free(struct blkcg_gq *blkg)  in blkg_free()  argument
51 	if (!blkg)  in blkg_free()
55 		kfree(blkg->pd[i]);  in blkg_free()
57 	blk_exit_rl(&blkg->rl);  in blkg_free()
58 	kfree(blkg);  in blkg_free()
72 	struct blkcg_gq *blkg;  in blkg_alloc()  local
76 	blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);  in blkg_alloc()
77 	if (!blkg)  in blkg_alloc()
80 	blkg->q = q;  in blkg_alloc()
81 	INIT_LIST_HEAD(&blkg->q_node);  in blkg_alloc()
82 	blkg->blkcg = blkcg;  in blkg_alloc()
83 	atomic_set(&blkg->refcnt, 1);  in blkg_alloc()
87 		if (blk_init_rl(&blkg->rl, q, gfp_mask))  in blkg_alloc()
89 		blkg->rl.blkg = blkg;  in blkg_alloc()
104 		blkg->pd[i] = pd;  in blkg_alloc()
105 		pd->blkg = blkg;  in blkg_alloc()
109 	return blkg;  in blkg_alloc()
112 	blkg_free(blkg);  in blkg_alloc()
130 	struct blkcg_gq *blkg;  in __blkg_lookup()  local
132 	blkg = rcu_dereference(blkcg->blkg_hint);  in __blkg_lookup()
133 	if (blkg && blkg->q == q)  in __blkg_lookup()
134 		return blkg;  in __blkg_lookup()
142 	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);  in __blkg_lookup()
143 	if (blkg && blkg->q == q) {  in __blkg_lookup()
146 			rcu_assign_pointer(blkcg->blkg_hint, blkg);  in __blkg_lookup()
148 		return blkg;  in __blkg_lookup()
181 	struct blkcg_gq *blkg;  in blkg_create()  local
201 	blkg = new_blkg;  in blkg_create()
205 		blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);  in blkg_create()
206 		if (WARN_ON_ONCE(!blkg->parent)) {  in blkg_create()
210 		blkg_get(blkg->parent);  in blkg_create()
217 		if (blkg->pd[i] && pol->pd_init_fn)  in blkg_create()
218 			pol->pd_init_fn(blkg);  in blkg_create()
223 	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);  in blkg_create()
225 		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);  in blkg_create()
226 		list_add(&blkg->q_node, &q->blkg_list);  in blkg_create()
231 			if (blkg->pd[i] && pol->pd_online_fn)  in blkg_create()
232 				pol->pd_online_fn(blkg);  in blkg_create()
235 	blkg->online = true;  in blkg_create()
240 			q->root_blkg = blkg;  in blkg_create()
241 			q->root_rl.blkg = blkg;  in blkg_create()
243 		return blkg;  in blkg_create()
247 	blkg_put(blkg);  in blkg_create()
274 	struct blkcg_gq *blkg;  in blkg_lookup_create()  local
286 	blkg = __blkg_lookup(blkcg, q, true);  in blkg_lookup_create()
287 	if (blkg)  in blkg_lookup_create()
288 		return blkg;  in blkg_lookup_create()
303 		blkg = blkg_create(pos, q, NULL);  in blkg_lookup_create()
304 		if (pos == blkcg || IS_ERR(blkg))  in blkg_lookup_create()
305 			return blkg;  in blkg_lookup_create()
310 static void blkg_destroy(struct blkcg_gq *blkg)  in blkg_destroy()  argument
312 	struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy()
315 	lockdep_assert_held(blkg->q->queue_lock);  in blkg_destroy()
319 	WARN_ON_ONCE(list_empty(&blkg->q_node));  in blkg_destroy()
320 	WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));  in blkg_destroy()
325 		if (blkg->pd[i] && pol->pd_offline_fn)  in blkg_destroy()
326 			pol->pd_offline_fn(blkg);  in blkg_destroy()
328 	blkg->online = false;  in blkg_destroy()
330 	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);  in blkg_destroy()
331 	list_del_init(&blkg->q_node);  in blkg_destroy()
332 	hlist_del_init_rcu(&blkg->blkcg_node);  in blkg_destroy()
339 	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)  in blkg_destroy()
347 		blkg->q->root_blkg = NULL;  in blkg_destroy()
348 		blkg->q->root_rl.blkg = NULL;  in blkg_destroy()
355 	blkg_put(blkg);  in blkg_destroy()
366 	struct blkcg_gq *blkg, *n;  in blkg_destroy_all()  local
370 	list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {  in blkg_destroy_all()
371 		struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy_all()
374 		blkg_destroy(blkg);  in blkg_destroy_all()
389 	struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);  in __blkg_release_rcu()  local
396 		if (blkg->pd[i] && pol->pd_exit_fn)  in __blkg_release_rcu()
397 			pol->pd_exit_fn(blkg);  in __blkg_release_rcu()
401 	css_put(&blkg->blkcg->css);  in __blkg_release_rcu()
402 	if (blkg->parent)  in __blkg_release_rcu()
403 		blkg_put(blkg->parent);  in __blkg_release_rcu()
405 	blkg_free(blkg);  in __blkg_release_rcu()
417 	struct blkcg_gq *blkg;  in __blk_queue_next_rl()  local
429 		blkg = container_of(rl, struct blkcg_gq, rl);  in __blk_queue_next_rl()
430 		ent = &blkg->q_node;  in __blk_queue_next_rl()
440 	blkg = container_of(ent, struct blkcg_gq, q_node);  in __blk_queue_next_rl()
441 	return &blkg->rl;  in __blk_queue_next_rl()
448 	struct blkcg_gq *blkg;  in blkcg_reset_stats()  local
472 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_reset_stats()
476 			if (blkcg_policy_enabled(blkg->q, pol) &&  in blkcg_reset_stats()
478 				pol->pd_reset_stats_fn(blkg);  in blkcg_reset_stats()
487 static const char *blkg_dev_name(struct blkcg_gq *blkg)  in blkg_dev_name()  argument
490 	if (blkg->q->backing_dev_info.dev)  in blkg_dev_name()
491 		return dev_name(blkg->q->backing_dev_info.dev);  in blkg_dev_name()
519 	struct blkcg_gq *blkg;  in blkcg_print_blkgs()  local
523 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_blkgs()
524 		spin_lock_irq(blkg->q->queue_lock);  in blkcg_print_blkgs()
525 		if (blkcg_policy_enabled(blkg->q, pol))  in blkcg_print_blkgs()
526 			total += prfill(sf, blkg->pd[pol->plid], data);  in blkcg_print_blkgs()
527 		spin_unlock_irq(blkg->q->queue_lock);  in blkcg_print_blkgs()
546 	const char *dname = blkg_dev_name(pd->blkg);  in __blkg_prfill_u64()
573 	const char *dname = blkg_dev_name(pd->blkg);  in __blkg_prfill_rwstat()
637 	lockdep_assert_held(pd->blkg->q->queue_lock);  in blkg_stat_recursive_sum()
671 	lockdep_assert_held(pd->blkg->q->queue_lock);  in blkg_rwstat_recursive_sum()
710 	struct blkcg_gq *blkg;  in blkg_conf_prep()  local
730 		blkg = blkg_lookup_create(blkcg, disk->queue);  in blkg_conf_prep()
732 		blkg = ERR_PTR(-EINVAL);  in blkg_conf_prep()
734 	if (IS_ERR(blkg)) {  in blkg_conf_prep()
735 		ret = PTR_ERR(blkg);  in blkg_conf_prep()
753 	ctx->blkg = blkg;  in blkg_conf_prep()
801 		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,  in blkcg_css_offline()  local
803 		struct request_queue *q = blkg->q;  in blkcg_css_offline()
806 			blkg_destroy(blkg);  in blkcg_css_offline()
965 	struct blkcg_gq *blkg, *new_blkg;  in blkcg_activate_policy()  local
990 	blkg = __blkg_lookup(&blkcg_root, q, false);  in blkcg_activate_policy()
991 	if (blkg)  in blkcg_activate_policy()
994 		blkg = blkg_create(&blkcg_root, q, new_blkg);  in blkcg_activate_policy()
1000 	if (IS_ERR(blkg)) {  in blkcg_activate_policy()
1001 		ret = PTR_ERR(blkg);  in blkcg_activate_policy()
1005 	list_for_each_entry(blkg, &q->blkg_list, q_node)  in blkcg_activate_policy()
1026 	list_for_each_entry(blkg, &q->blkg_list, q_node) {  in blkcg_activate_policy()
1036 		spin_lock(&blkg->blkcg->lock);  in blkcg_activate_policy()
1038 		blkg->pd[pol->plid] = pd;  in blkcg_activate_policy()
1039 		pd->blkg = blkg;  in blkcg_activate_policy()
1041 		pol->pd_init_fn(blkg);  in blkcg_activate_policy()
1043 		spin_unlock(&blkg->blkcg->lock);  in blkcg_activate_policy()
1069 	struct blkcg_gq *blkg;  in blkcg_deactivate_policy()  local
1083 	list_for_each_entry(blkg, &q->blkg_list, q_node) {  in blkcg_deactivate_policy()
1085 		spin_lock(&blkg->blkcg->lock);  in blkcg_deactivate_policy()
1088 			pol->pd_offline_fn(blkg);  in blkcg_deactivate_policy()
1090 			pol->pd_exit_fn(blkg);  in blkcg_deactivate_policy()
1092 		kfree(blkg->pd[pol->plid]);  in blkcg_deactivate_policy()
1093 		blkg->pd[pol->plid] = NULL;  in blkcg_deactivate_policy()
1095 		spin_unlock(&blkg->blkcg->lock);  in blkcg_deactivate_policy()