63 #define for_each_handle_rcu(qh, inst)			\  argument
64 	list_for_each_entry_rcu(qh, &inst->handles, list)
84 	struct knav_queue *qh;  in knav_queue_notify()  local
90 	for_each_handle_rcu(qh, inst) {  in knav_queue_notify()
91 		if (atomic_read(&qh->notifier_enabled) <= 0)  in knav_queue_notify()
93 		if (WARN_ON(!qh->notifier_fn))  in knav_queue_notify()
95 		atomic_inc(&qh->stats.notifies);  in knav_queue_notify()
96 		qh->notifier_fn(qh->notifier_fn_arg);  in knav_queue_notify()
218 	struct knav_queue *qh;  in __knav_queue_open()  local
222 	qh = devm_kzalloc(inst->kdev->dev, sizeof(*qh), GFP_KERNEL);  in __knav_queue_open()
223 	if (!qh)  in __knav_queue_open()
226 	qh->flags = flags;  in __knav_queue_open()
227 	qh->inst = inst;  in __knav_queue_open()
229 	qh->reg_push = &inst->qmgr->reg_push[id];  in __knav_queue_open()
230 	qh->reg_pop = &inst->qmgr->reg_pop[id];  in __knav_queue_open()
231 	qh->reg_peek = &inst->qmgr->reg_peek[id];  in __knav_queue_open()
242 			devm_kfree(inst->kdev->dev, qh);  in __knav_queue_open()
246 	list_add_tail_rcu(&qh->list, &inst->handles);  in __knav_queue_open()
247 	return qh;  in __knav_queue_open()
254 	struct knav_queue *qh;  in knav_queue_open_by_id()  local
258 	qh = ERR_PTR(-ENODEV);  in knav_queue_open_by_id()
263 	qh = ERR_PTR(-EEXIST);  in knav_queue_open_by_id()
267 	qh = ERR_PTR(-EBUSY);  in knav_queue_open_by_id()
272 	qh = __knav_queue_open(inst, name, flags);  in knav_queue_open_by_id()
277 	return qh;  in knav_queue_open_by_id()
284 	struct knav_queue *qh = ERR_PTR(-EINVAL);  in knav_queue_open_by_type()  local
296 		qh = __knav_queue_open(inst, name, flags);  in knav_queue_open_by_type()
302 	return qh;  in knav_queue_open_by_type()
313 static int knav_queue_enable_notifier(struct knav_queue *qh)  in knav_queue_enable_notifier()  argument
315 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_enable_notifier()
318 	if (WARN_ON(!qh->notifier_fn))  in knav_queue_enable_notifier()
322 	first = (atomic_inc_return(&qh->notifier_enabled) == 1);  in knav_queue_enable_notifier()
334 static int knav_queue_disable_notifier(struct knav_queue *qh)  in knav_queue_disable_notifier()  argument
336 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_disable_notifier()
339 	last = (atomic_dec_return(&qh->notifier_enabled) == 0);  in knav_queue_disable_notifier()
350 static int knav_queue_set_notifier(struct knav_queue *qh,  in knav_queue_set_notifier()  argument
353 	knav_queue_notify_fn old_fn = qh->notifier_fn;  in knav_queue_set_notifier()
358 	if (!(qh->inst->range->flags & (RANGE_HAS_ACCUMULATOR | RANGE_HAS_IRQ)))  in knav_queue_set_notifier()
362 		knav_queue_disable_notifier(qh);  in knav_queue_set_notifier()
364 	qh->notifier_fn = cfg->fn;  in knav_queue_set_notifier()
365 	qh->notifier_fn_arg = cfg->fn_arg;  in knav_queue_set_notifier()
368 		knav_queue_enable_notifier(qh);  in knav_queue_set_notifier()
411 	struct knav_queue *qh = qhandle;  in knav_queue_get_count()  local
412 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_get_count()
414 	return readl_relaxed(&qh->reg_peek[0].entry_count) +  in knav_queue_get_count()
422 	struct knav_queue *qh;  in knav_queue_debug_show_instance()  local
429 	for_each_handle_rcu(qh, inst) {  in knav_queue_debug_show_instance()
430 		seq_printf(s, "\t\thandle %p: ", qh);  in knav_queue_debug_show_instance()
432 			   atomic_read(&qh->stats.pushes));  in knav_queue_debug_show_instance()
434 			   atomic_read(&qh->stats.pops));  in knav_queue_debug_show_instance()
436 			   knav_queue_get_count(qh));  in knav_queue_debug_show_instance()
438 			   atomic_read(&qh->stats.notifies));  in knav_queue_debug_show_instance()
440 			   atomic_read(&qh->stats.push_errors));  in knav_queue_debug_show_instance()
442 			   atomic_read(&qh->stats.pop_errors));  in knav_queue_debug_show_instance()
493 static int knav_queue_flush(struct knav_queue *qh)  in knav_queue_flush()  argument
495 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_flush()
520 	struct knav_queue *qh = ERR_PTR(-EINVAL);  in knav_queue_open()  local
526 		qh = knav_queue_open_by_type(name, id, flags);  in knav_queue_open()
530 		qh = knav_queue_open_by_id(name, id, flags);  in knav_queue_open()
533 	return qh;  in knav_queue_open()
543 	struct knav_queue *qh = qhandle;  in knav_queue_close()  local
544 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_close()
546 	while (atomic_read(&qh->notifier_enabled) > 0)  in knav_queue_close()
547 		knav_queue_disable_notifier(qh);  in knav_queue_close()
550 	list_del_rcu(&qh->list);  in knav_queue_close()
559 	devm_kfree(inst->kdev->dev, qh);  in knav_queue_close()
574 	struct knav_queue *qh = qhandle;  in knav_queue_device_control()  local
580 		ret = qh->inst->kdev->base_id + qh->inst->id;  in knav_queue_device_control()
584 		ret = knav_queue_flush(qh);  in knav_queue_device_control()
589 		ret = knav_queue_set_notifier(qh, cfg);  in knav_queue_device_control()
593 		ret = knav_queue_enable_notifier(qh);  in knav_queue_device_control()
597 		ret = knav_queue_disable_notifier(qh);  in knav_queue_device_control()
601 		ret = knav_queue_get_count(qh);  in knav_queue_device_control()
626 	struct knav_queue *qh = qhandle;  in knav_queue_push()  local
630 	writel_relaxed(val, &qh->reg_push[0].ptr_size_thresh);  in knav_queue_push()
632 	atomic_inc(&qh->stats.pushes);  in knav_queue_push()
646 	struct knav_queue *qh = qhandle;  in knav_queue_pop()  local
647 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_pop()
661 		val = readl_relaxed(&qh->reg_pop[0].ptr_size_thresh);  in knav_queue_pop()
670 	atomic_inc(&qh->stats.pops);  in knav_queue_pop()