36 void blk_free_tags(struct blk_queue_tag *bqt)  in blk_free_tags()  argument
38 	if (atomic_dec_and_test(&bqt->refcnt)) {  in blk_free_tags()
39 		BUG_ON(find_first_bit(bqt->tag_map, bqt->max_depth) <  in blk_free_tags()
40 							bqt->max_depth);  in blk_free_tags()
42 		kfree(bqt->tag_index);  in blk_free_tags()
43 		bqt->tag_index = NULL;  in blk_free_tags()
45 		kfree(bqt->tag_map);  in blk_free_tags()
46 		bqt->tag_map = NULL;  in blk_free_tags()
48 		kfree(bqt);  in blk_free_tags()
63 	struct blk_queue_tag *bqt = q->queue_tags;  in __blk_queue_free_tags()  local
65 	if (!bqt)  in __blk_queue_free_tags()
68 	blk_free_tags(bqt);  in __blk_queue_free_tags()
205 	struct blk_queue_tag *bqt = q->queue_tags;  in blk_queue_resize_tags()  local
210 	if (!bqt)  in blk_queue_resize_tags()
219 	if (new_depth <= bqt->real_max_depth) {  in blk_queue_resize_tags()
220 		bqt->max_depth = new_depth;  in blk_queue_resize_tags()
228 	if (atomic_read(&bqt->refcnt) != 1)  in blk_queue_resize_tags()
234 	tag_index = bqt->tag_index;  in blk_queue_resize_tags()
235 	tag_map = bqt->tag_map;  in blk_queue_resize_tags()
236 	max_depth = bqt->real_max_depth;  in blk_queue_resize_tags()
238 	if (init_tag_map(q, bqt, new_depth))  in blk_queue_resize_tags()
241 	memcpy(bqt->tag_index, tag_index, max_depth * sizeof(struct request *));  in blk_queue_resize_tags()
243 	memcpy(bqt->tag_map, tag_map, nr_ulongs * sizeof(unsigned long));  in blk_queue_resize_tags()
267 	struct blk_queue_tag *bqt = q->queue_tags;  in blk_queue_end_tag()  local
270 	BUG_ON(tag >= bqt->real_max_depth);  in blk_queue_end_tag()
276 	if (unlikely(bqt->tag_index[tag] == NULL))  in blk_queue_end_tag()
280 	bqt->tag_index[tag] = NULL;  in blk_queue_end_tag()
282 	if (unlikely(!test_bit(tag, bqt->tag_map))) {  in blk_queue_end_tag()
291 	clear_bit_unlock(tag, bqt->tag_map);  in blk_queue_end_tag()
315 	struct blk_queue_tag *bqt = q->queue_tags;  in blk_queue_start_tag()  local
334 	max_depth = bqt->max_depth;  in blk_queue_start_tag()
351 		if (bqt->alloc_policy == BLK_TAG_ALLOC_FIFO) {  in blk_queue_start_tag()
352 			tag = find_first_zero_bit(bqt->tag_map, max_depth);  in blk_queue_start_tag()
356 			int start = bqt->next_tag;  in blk_queue_start_tag()
357 			int size = min_t(int, bqt->max_depth, max_depth + start);  in blk_queue_start_tag()
358 			tag = find_next_zero_bit(bqt->tag_map, size, start);  in blk_queue_start_tag()
359 			if (tag >= size && start + size > bqt->max_depth) {  in blk_queue_start_tag()
360 				size = start + size - bqt->max_depth;  in blk_queue_start_tag()
361 				tag = find_first_zero_bit(bqt->tag_map, size);  in blk_queue_start_tag()
367 	} while (test_and_set_bit_lock(tag, bqt->tag_map));  in blk_queue_start_tag()
373 	bqt->next_tag = (tag + 1) % bqt->max_depth;  in blk_queue_start_tag()
376 	bqt->tag_index[tag] = rq;  in blk_queue_start_tag()