43 struct blkcg blkcg_root;
75 	if (blkg->blkcg != &blkcg_root)  in blkg_free()
91 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,  in blkg_alloc()  argument
108 	blkg->blkcg = blkcg;  in blkg_alloc()
112 	if (blkcg != &blkcg_root) {  in blkg_alloc()
142 struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,  in blkg_lookup_slowpath()  argument
153 	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);  in blkg_lookup_slowpath()
157 			rcu_assign_pointer(blkcg->blkg_hint, blkg);  in blkg_lookup_slowpath()
170 static struct blkcg_gq *blkg_create(struct blkcg *blkcg,  in blkg_create()  argument
182 	if (!css_tryget_online(&blkcg->css)) {  in blkg_create()
188 					       blkcg->css.id, GFP_NOWAIT);  in blkg_create()
196 		new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT);  in blkg_create()
206 	if (blkcg_parent(blkcg)) {  in blkg_create()
207 		blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);  in blkg_create()
224 	spin_lock(&blkcg->lock);  in blkg_create()
225 	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);  in blkg_create()
227 		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);  in blkg_create()
238 	spin_unlock(&blkcg->lock);  in blkg_create()
250 	css_put(&blkcg->css);  in blkg_create()
270 struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,  in blkg_lookup_create()  argument
285 	blkg = __blkg_lookup(blkcg, q, true);  in blkg_lookup_create()
294 		struct blkcg *pos = blkcg;  in blkg_lookup_create()
295 		struct blkcg *parent = blkcg_parent(blkcg);  in blkg_lookup_create()
303 		if (pos == blkcg || IS_ERR(blkg))  in blkg_lookup_create()
310 	struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy()  local
315 	lockdep_assert_held(&blkcg->lock);  in blkg_destroy()
335 	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);  in blkg_destroy()
344 	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)  in blkg_destroy()
345 		rcu_assign_pointer(blkcg->blkg_hint, NULL);  in blkg_destroy()
367 		struct blkcg *blkcg = blkg->blkcg;  in blkg_destroy_all()  local
369 		spin_lock(&blkcg->lock);  in blkg_destroy_all()
371 		spin_unlock(&blkcg->lock);  in blkg_destroy_all()
391 	css_put(&blkg->blkcg->css);  in __blkg_release_rcu()
439 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_reset_stats()  local
444 	spin_lock_irq(&blkcg->lock);  in blkcg_reset_stats()
451 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_reset_stats()
463 	spin_unlock_irq(&blkcg->lock);  in blkcg_reset_stats()
495 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,  in blkcg_print_blkgs()  argument
505 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_blkgs()
785 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,  in blkg_conf_prep()  argument
815 		blkg = blkg_lookup_create(blkcg, disk->queue);  in blkg_conf_prep()
862 	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));  in blkcg_print_stat()  local
867 	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {  in blkcg_print_stat()
929 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_css_offline()  local
931 	spin_lock_irq(&blkcg->lock);  in blkcg_css_offline()
933 	while (!hlist_empty(&blkcg->blkg_list)) {  in blkcg_css_offline()
934 		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,  in blkcg_css_offline()
942 			spin_unlock_irq(&blkcg->lock);  in blkcg_css_offline()
944 			spin_lock_irq(&blkcg->lock);  in blkcg_css_offline()
948 	spin_unlock_irq(&blkcg->lock);  in blkcg_css_offline()
950 	wb_blkcg_offline(blkcg);  in blkcg_css_offline()
955 	struct blkcg *blkcg = css_to_blkcg(css);  in blkcg_css_free()  local
960 	list_del(&blkcg->all_blkcgs_node);  in blkcg_css_free()
963 		if (blkcg->cpd[i])  in blkcg_css_free()
964 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);  in blkcg_css_free()
968 	kfree(blkcg);  in blkcg_css_free()
974 	struct blkcg *blkcg;  in blkcg_css_alloc()  local
981 		blkcg = &blkcg_root;  in blkcg_css_alloc()
983 		blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);  in blkcg_css_alloc()
984 		if (!blkcg) {  in blkcg_css_alloc()
1008 		blkcg->cpd[i] = cpd;  in blkcg_css_alloc()
1009 		cpd->blkcg = blkcg;  in blkcg_css_alloc()
1015 	spin_lock_init(&blkcg->lock);  in blkcg_css_alloc()
1016 	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT);  in blkcg_css_alloc()
1017 	INIT_HLIST_HEAD(&blkcg->blkg_list);  in blkcg_css_alloc()
1019 	INIT_LIST_HEAD(&blkcg->cgwb_list);  in blkcg_css_alloc()
1021 	list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);  in blkcg_css_alloc()
1024 	return &blkcg->css;  in blkcg_css_alloc()
1028 		if (blkcg->cpd[i])  in blkcg_css_alloc()
1029 			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);  in blkcg_css_alloc()
1031 	kfree(blkcg);  in blkcg_css_alloc()
1158 		struct blkcg *blkcg;  in blkcg_bind()  local
1163 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)  in blkcg_bind()
1164 			if (blkcg->cpd[pol->plid])  in blkcg_bind()
1165 				pol->cpd_bind_fn(blkcg->cpd[pol->plid]);  in blkcg_bind()
1284 		spin_lock(&blkg->blkcg->lock);  in blkcg_deactivate_policy()
1293 		spin_unlock(&blkg->blkcg->lock);  in blkcg_deactivate_policy()
1310 	struct blkcg *blkcg;  in blkcg_policy_register()  local
1330 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {  in blkcg_policy_register()
1339 			blkcg->cpd[pol->plid] = cpd;  in blkcg_policy_register()
1340 			cpd->blkcg = blkcg;  in blkcg_policy_register()
1360 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {  in blkcg_policy_register()
1361 			if (blkcg->cpd[pol->plid]) {  in blkcg_policy_register()
1362 				pol->cpd_free_fn(blkcg->cpd[pol->plid]);  in blkcg_policy_register()
1363 				blkcg->cpd[pol->plid] = NULL;  in blkcg_policy_register()
1383 	struct blkcg *blkcg;  in blkcg_policy_unregister()  local
1400 		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {  in blkcg_policy_unregister()
1401 			if (blkcg->cpd[pol->plid]) {  in blkcg_policy_unregister()
1402 				pol->cpd_free_fn(blkcg->cpd[pol->plid]);  in blkcg_policy_unregister()
1403 				blkcg->cpd[pol->plid] = NULL;  in blkcg_policy_unregister()