rqw               268 block/blk-iolatency.c static void iolat_cleanup_cb(struct rq_wait *rqw, void *private_data)
rqw               270 block/blk-iolatency.c 	atomic_dec(&rqw->inflight);
rqw               271 block/blk-iolatency.c 	wake_up(&rqw->wait);
rqw               274 block/blk-iolatency.c static bool iolat_acquire_inflight(struct rq_wait *rqw, void *private_data)
rqw               277 block/blk-iolatency.c 	return rq_wait_inc_below(rqw, iolat->rq_depth.max_depth);
rqw               285 block/blk-iolatency.c 	struct rq_wait *rqw = &iolat->rq_wait;
rqw               299 block/blk-iolatency.c 		atomic_inc(&rqw->inflight);
rqw               303 block/blk-iolatency.c 	rq_qos_wait(rqw, iolat, iolat_acquire_inflight, iolat_cleanup_cb);
rqw               591 block/blk-iolatency.c 	struct rq_wait *rqw;
rqw               617 block/blk-iolatency.c 		rqw = &iolat->rq_wait;
rqw               619 block/blk-iolatency.c 		inflight = atomic_dec_return(&rqw->inflight);
rqw               636 block/blk-iolatency.c 		wake_up(&rqw->wait);
rqw               206 block/blk-rq-qos.c 	struct rq_wait *rqw;
rqw               223 block/blk-rq-qos.c 	if (!data->cb(data->rqw, data->private_data))
rqw               249 block/blk-rq-qos.c void rq_qos_wait(struct rq_wait *rqw, void *private_data,
rqw               259 block/blk-rq-qos.c 		.rqw = rqw,
rqw               265 block/blk-rq-qos.c 	has_sleeper = wq_has_sleeper(&rqw->wait);
rqw               266 block/blk-rq-qos.c 	if (!has_sleeper && acquire_inflight_cb(rqw, private_data))
rqw               269 block/blk-rq-qos.c 	prepare_to_wait_exclusive(&rqw->wait, &data.wq, TASK_UNINTERRUPTIBLE);
rqw               270 block/blk-rq-qos.c 	has_sleeper = !wq_has_single_sleeper(&rqw->wait);
rqw               275 block/blk-rq-qos.c 		if (!has_sleeper && acquire_inflight_cb(rqw, private_data)) {
rqw               276 block/blk-rq-qos.c 			finish_wait(&rqw->wait, &data.wq);
rqw               285 block/blk-rq-qos.c 				cleanup_cb(rqw, private_data);
rqw               292 block/blk-rq-qos.c 	finish_wait(&rqw->wait, &data.wq);
rqw               123 block/blk-rq-qos.h typedef bool (acquire_inflight_cb_t)(struct rq_wait *rqw, void *private_data);
rqw               124 block/blk-rq-qos.h typedef void (cleanup_cb_t)(struct rq_wait *rqw, void *private_data);
rqw               126 block/blk-rq-qos.h void rq_qos_wait(struct rq_wait *rqw, void *private_data,
rqw               120 block/blk-wbt.c 		struct rq_wait *rqw = &rwb->rq_wait[i];
rqw               122 block/blk-wbt.c 		if (wq_has_sleeper(&rqw->wait))
rqw               123 block/blk-wbt.c 			wake_up_all(&rqw->wait);
rqw               127 block/blk-wbt.c static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw,
rqw               132 block/blk-wbt.c 	inflight = atomic_dec_return(&rqw->inflight);
rqw               161 block/blk-wbt.c 	if (wq_has_sleeper(&rqw->wait)) {
rqw               165 block/blk-wbt.c 			wake_up_all(&rqw->wait);
rqw               172 block/blk-wbt.c 	struct rq_wait *rqw;
rqw               177 block/blk-wbt.c 	rqw = get_rq_wait(rwb, wb_acct);
rqw               178 block/blk-wbt.c 	wbt_rqw_done(rwb, rqw, wb_acct);
rqw               500 block/blk-wbt.c static bool wbt_inflight_cb(struct rq_wait *rqw, void *private_data)
rqw               503 block/blk-wbt.c 	return rq_wait_inc_below(rqw, get_limit(data->rwb, data->rw));
rqw               506 block/blk-wbt.c static void wbt_cleanup_cb(struct rq_wait *rqw, void *private_data)
rqw               509 block/blk-wbt.c 	wbt_rqw_done(data->rwb, rqw, data->wb_acct);
rqw               519 block/blk-wbt.c 	struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);
rqw               526 block/blk-wbt.c 	rq_qos_wait(rqw, &data, wbt_inflight_cb, wbt_cleanup_cb);