61 bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)  in __blk_mq_tag_busy()  argument
63 	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&  in __blk_mq_tag_busy()
64 	    !test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))  in __blk_mq_tag_busy()
65 		atomic_inc(&hctx->tags->active_queues);  in __blk_mq_tag_busy()
100 void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)  in __blk_mq_tag_idle()  argument
102 	struct blk_mq_tags *tags = hctx->tags;  in __blk_mq_tag_idle()
104 	if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))  in __blk_mq_tag_idle()
116 static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,  in hctx_may_queue()  argument
121 	if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED))  in hctx_may_queue()
123 	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))  in hctx_may_queue()
132 	users = atomic_read(&hctx->tags->active_queues);  in hctx_may_queue()
140 	return atomic_read(&hctx->nr_active) < depth;  in hctx_may_queue()
187 static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt,  in __bt_get()  argument
193 	if (!hctx_may_queue(hctx, bt))  in __bt_get()
240 					 struct blk_mq_hw_ctx *hctx)  in bt_wait_ptr()  argument
245 	if (!hctx)  in bt_wait_ptr()
248 	wait_index = atomic_read(&hctx->wait_index);  in bt_wait_ptr()
250 	bt_index_atomic_inc(&hctx->wait_index);  in bt_wait_ptr()
256 		struct blk_mq_hw_ctx *hctx,  in bt_get()  argument
263 	tag = __bt_get(hctx, bt, last_tag, tags);  in bt_get()
270 	bs = bt_wait_ptr(bt, hctx);  in bt_get()
274 		tag = __bt_get(hctx, bt, last_tag, tags);  in bt_get()
284 		if (hctx)  in bt_get()
285 			blk_mq_run_hw_queue(hctx, false);  in bt_get()
291 		tag = __bt_get(hctx, bt, last_tag, tags);  in bt_get()
300 		data->hctx = data->q->mq_ops->map_queue(data->q,  in bt_get()
303 			bt = &data->hctx->tags->breserved_tags;  in bt_get()
306 			hctx = data->hctx;  in bt_get()
307 			bt = &hctx->tags->bitmap_tags;  in bt_get()
310 		bs = bt_wait_ptr(bt, hctx);  in bt_get()
321 	tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx,  in __blk_mq_get_tag()
322 			&data->ctx->last_tag, data->hctx->tags);  in __blk_mq_get_tag()
324 		return tag + data->hctx->tags->nr_reserved_tags;  in __blk_mq_get_tag()
333 	if (unlikely(!data->hctx->tags->nr_reserved_tags)) {  in __blk_mq_get_reserved_tag()
338 	tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL, &zero,  in __blk_mq_get_reserved_tag()
339 		data->hctx->tags);  in __blk_mq_get_reserved_tag()
401 void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, unsigned int tag,  in blk_mq_put_tag()  argument
404 	struct blk_mq_tags *tags = hctx->tags;  in blk_mq_put_tag()
419 static void bt_for_each(struct blk_mq_hw_ctx *hctx,  in bt_for_each()  argument
432 		     	rq = blk_mq_tag_to_rq(hctx->tags, off + bit);  in bt_for_each()
433 			if (rq->q == hctx->queue)  in bt_for_each()
434 				fn(hctx, rq, data, reserved);  in bt_for_each()
441 void blk_mq_tag_busy_iter(struct blk_mq_hw_ctx *hctx, busy_iter_fn *fn,  in blk_mq_tag_busy_iter()  argument
444 	struct blk_mq_tags *tags = hctx->tags;  in blk_mq_tag_busy_iter()
447 		bt_for_each(hctx, &tags->breserved_tags, 0, fn, priv, true);  in blk_mq_tag_busy_iter()
448 	bt_for_each(hctx, &tags->bitmap_tags, tags->nr_reserved_tags, fn, priv,  in blk_mq_tag_busy_iter()
633 	struct blk_mq_hw_ctx *hctx;  in blk_mq_unique_tag()  local
637 		hctx = q->mq_ops->map_queue(q, rq->mq_ctx->cpu);  in blk_mq_unique_tag()
638 		hwq = hctx->queue_num;  in blk_mq_unique_tag()