24 void get_io_context(struct io_context *ioc)  in get_io_context()  argument
26 	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);  in get_io_context()
27 	atomic_long_inc(&ioc->refcount);  in get_io_context()
55 	struct io_context *ioc = icq->ioc;  in ioc_destroy_icq()  local
59 	lockdep_assert_held(&ioc->lock);  in ioc_destroy_icq()
62 	radix_tree_delete(&ioc->icq_tree, icq->q->id);  in ioc_destroy_icq()
71 	if (rcu_access_pointer(ioc->icq_hint) == icq)  in ioc_destroy_icq()
72 		rcu_assign_pointer(ioc->icq_hint, NULL);  in ioc_destroy_icq()
90 	struct io_context *ioc = container_of(work, struct io_context,  in ioc_release_fn()  local
100 	spin_lock_irqsave_nested(&ioc->lock, flags, 1);  in ioc_release_fn()
102 	while (!hlist_empty(&ioc->icq_list)) {  in ioc_release_fn()
103 		struct io_cq *icq = hlist_entry(ioc->icq_list.first,  in ioc_release_fn()
111 			spin_unlock_irqrestore(&ioc->lock, flags);  in ioc_release_fn()
113 			spin_lock_irqsave_nested(&ioc->lock, flags, 1);  in ioc_release_fn()
117 	spin_unlock_irqrestore(&ioc->lock, flags);  in ioc_release_fn()
119 	kmem_cache_free(iocontext_cachep, ioc);  in ioc_release_fn()
129 void put_io_context(struct io_context *ioc)  in put_io_context()  argument
134 	if (ioc == NULL)  in put_io_context()
137 	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);  in put_io_context()
143 	if (atomic_long_dec_and_test(&ioc->refcount)) {  in put_io_context()
144 		spin_lock_irqsave(&ioc->lock, flags);  in put_io_context()
145 		if (!hlist_empty(&ioc->icq_list))  in put_io_context()
147 					&ioc->release_work);  in put_io_context()
150 		spin_unlock_irqrestore(&ioc->lock, flags);  in put_io_context()
154 		kmem_cache_free(iocontext_cachep, ioc);  in put_io_context()
165 void put_io_context_active(struct io_context *ioc)  in put_io_context_active()  argument
170 	if (!atomic_dec_and_test(&ioc->active_ref)) {  in put_io_context_active()
171 		put_io_context(ioc);  in put_io_context_active()
181 	spin_lock_irqsave_nested(&ioc->lock, flags, 1);  in put_io_context_active()
182 	hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) {  in put_io_context_active()
189 			spin_unlock_irqrestore(&ioc->lock, flags);  in put_io_context_active()
194 	spin_unlock_irqrestore(&ioc->lock, flags);  in put_io_context_active()
196 	put_io_context(ioc);  in put_io_context_active()
202 	struct io_context *ioc;  in exit_io_context()  local
205 	ioc = task->io_context;  in exit_io_context()
209 	atomic_dec(&ioc->nr_tasks);  in exit_io_context()
210 	put_io_context_active(ioc);  in exit_io_context()
226 		struct io_context *ioc = icq->ioc;  in ioc_clear_queue()  local
228 		spin_lock(&ioc->lock);  in ioc_clear_queue()
230 		spin_unlock(&ioc->lock);  in ioc_clear_queue()
236 	struct io_context *ioc;  in create_task_io_context()  local
239 	ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO,  in create_task_io_context()
241 	if (unlikely(!ioc))  in create_task_io_context()
245 	atomic_long_set(&ioc->refcount, 1);  in create_task_io_context()
246 	atomic_set(&ioc->nr_tasks, 1);  in create_task_io_context()
247 	atomic_set(&ioc->active_ref, 1);  in create_task_io_context()
248 	spin_lock_init(&ioc->lock);  in create_task_io_context()
249 	INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC | __GFP_HIGH);  in create_task_io_context()
250 	INIT_HLIST_HEAD(&ioc->icq_list);  in create_task_io_context()
251 	INIT_WORK(&ioc->release_work, ioc_release_fn);  in create_task_io_context()
263 		task->io_context = ioc;  in create_task_io_context()
265 		kmem_cache_free(iocontext_cachep, ioc);  in create_task_io_context()
290 	struct io_context *ioc;  in get_task_io_context()  local
296 		ioc = task->io_context;  in get_task_io_context()
297 		if (likely(ioc)) {  in get_task_io_context()
298 			get_io_context(ioc);  in get_task_io_context()
300 			return ioc;  in get_task_io_context()
317 struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q)  in ioc_lookup_icq()  argument
330 	icq = rcu_dereference(ioc->icq_hint);  in ioc_lookup_icq()
334 	icq = radix_tree_lookup(&ioc->icq_tree, q->id);  in ioc_lookup_icq()
336 		rcu_assign_pointer(ioc->icq_hint, icq);	/* allowed to race */  in ioc_lookup_icq()
357 struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q,  in ioc_create_icq()  argument
374 	icq->ioc = ioc;  in ioc_create_icq()
381 	spin_lock(&ioc->lock);  in ioc_create_icq()
383 	if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {  in ioc_create_icq()
384 		hlist_add_head(&icq->ioc_node, &ioc->icq_list);  in ioc_create_icq()
390 		icq = ioc_lookup_icq(ioc, q);  in ioc_create_icq()
395 	spin_unlock(&ioc->lock);  in ioc_create_icq()