63 #define for_each_handle_rcu(qh, inst)			\  argument
64 	list_for_each_entry_rcu(qh, &inst->handles, list)
78 	struct knav_queue *qh;  in knav_queue_notify()  local
84 	for_each_handle_rcu(qh, inst) {  in knav_queue_notify()
85 		if (atomic_read(&qh->notifier_enabled) <= 0)  in knav_queue_notify()
87 		if (WARN_ON(!qh->notifier_fn))  in knav_queue_notify()
89 		atomic_inc(&qh->stats.notifies);  in knav_queue_notify()
90 		qh->notifier_fn(qh->notifier_fn_arg);  in knav_queue_notify()
212 	struct knav_queue *qh;  in __knav_queue_open()  local
216 	qh = devm_kzalloc(inst->kdev->dev, sizeof(*qh), GFP_KERNEL);  in __knav_queue_open()
217 	if (!qh)  in __knav_queue_open()
220 	qh->flags = flags;  in __knav_queue_open()
221 	qh->inst = inst;  in __knav_queue_open()
223 	qh->reg_push = &inst->qmgr->reg_push[id];  in __knav_queue_open()
224 	qh->reg_pop = &inst->qmgr->reg_pop[id];  in __knav_queue_open()
225 	qh->reg_peek = &inst->qmgr->reg_peek[id];  in __knav_queue_open()
236 			devm_kfree(inst->kdev->dev, qh);  in __knav_queue_open()
240 	list_add_tail_rcu(&qh->list, &inst->handles);  in __knav_queue_open()
241 	return qh;  in __knav_queue_open()
248 	struct knav_queue *qh;  in knav_queue_open_by_id()  local
252 	qh = ERR_PTR(-ENODEV);  in knav_queue_open_by_id()
257 	qh = ERR_PTR(-EEXIST);  in knav_queue_open_by_id()
261 	qh = ERR_PTR(-EBUSY);  in knav_queue_open_by_id()
266 	qh = __knav_queue_open(inst, name, flags);  in knav_queue_open_by_id()
271 	return qh;  in knav_queue_open_by_id()
278 	struct knav_queue *qh = ERR_PTR(-EINVAL);  in knav_queue_open_by_type()  local
290 		qh = __knav_queue_open(inst, name, flags);  in knav_queue_open_by_type()
296 	return qh;  in knav_queue_open_by_type()
307 static int knav_queue_enable_notifier(struct knav_queue *qh)  in knav_queue_enable_notifier()  argument
309 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_enable_notifier()
312 	if (WARN_ON(!qh->notifier_fn))  in knav_queue_enable_notifier()
316 	first = (atomic_inc_return(&qh->notifier_enabled) == 1);  in knav_queue_enable_notifier()
328 static int knav_queue_disable_notifier(struct knav_queue *qh)  in knav_queue_disable_notifier()  argument
330 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_disable_notifier()
333 	last = (atomic_dec_return(&qh->notifier_enabled) == 0);  in knav_queue_disable_notifier()
344 static int knav_queue_set_notifier(struct knav_queue *qh,  in knav_queue_set_notifier()  argument
347 	knav_queue_notify_fn old_fn = qh->notifier_fn;  in knav_queue_set_notifier()
352 	if (!(qh->inst->range->flags & (RANGE_HAS_ACCUMULATOR | RANGE_HAS_IRQ)))  in knav_queue_set_notifier()
356 		knav_queue_disable_notifier(qh);  in knav_queue_set_notifier()
358 	qh->notifier_fn = cfg->fn;  in knav_queue_set_notifier()
359 	qh->notifier_fn_arg = cfg->fn_arg;  in knav_queue_set_notifier()
362 		knav_queue_enable_notifier(qh);  in knav_queue_set_notifier()
405 	struct knav_queue *qh = qhandle;  in knav_queue_get_count()  local
406 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_get_count()
408 	return readl_relaxed(&qh->reg_peek[0].entry_count) +  in knav_queue_get_count()
416 	struct knav_queue *qh;  in knav_queue_debug_show_instance()  local
423 	for_each_handle_rcu(qh, inst) {  in knav_queue_debug_show_instance()
424 		seq_printf(s, "\t\thandle %p: ", qh);  in knav_queue_debug_show_instance()
426 			   atomic_read(&qh->stats.pushes));  in knav_queue_debug_show_instance()
428 			   atomic_read(&qh->stats.pops));  in knav_queue_debug_show_instance()
430 			   knav_queue_get_count(qh));  in knav_queue_debug_show_instance()
432 			   atomic_read(&qh->stats.notifies));  in knav_queue_debug_show_instance()
434 			   atomic_read(&qh->stats.push_errors));  in knav_queue_debug_show_instance()
436 			   atomic_read(&qh->stats.pop_errors));  in knav_queue_debug_show_instance()
487 static int knav_queue_flush(struct knav_queue *qh)  in knav_queue_flush()  argument
489 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_flush()
514 	struct knav_queue *qh = ERR_PTR(-EINVAL);  in knav_queue_open()  local
520 		qh = knav_queue_open_by_type(name, id, flags);  in knav_queue_open()
524 		qh = knav_queue_open_by_id(name, id, flags);  in knav_queue_open()
527 	return qh;  in knav_queue_open()
537 	struct knav_queue *qh = qhandle;  in knav_queue_close()  local
538 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_close()
540 	while (atomic_read(&qh->notifier_enabled) > 0)  in knav_queue_close()
541 		knav_queue_disable_notifier(qh);  in knav_queue_close()
544 	list_del_rcu(&qh->list);  in knav_queue_close()
553 	devm_kfree(inst->kdev->dev, qh);  in knav_queue_close()
568 	struct knav_queue *qh = qhandle;  in knav_queue_device_control()  local
574 		ret = qh->inst->kdev->base_id + qh->inst->id;  in knav_queue_device_control()
578 		ret = knav_queue_flush(qh);  in knav_queue_device_control()
583 		ret = knav_queue_set_notifier(qh, cfg);  in knav_queue_device_control()
587 		ret = knav_queue_enable_notifier(qh);  in knav_queue_device_control()
591 		ret = knav_queue_disable_notifier(qh);  in knav_queue_device_control()
595 		ret = knav_queue_get_count(qh);  in knav_queue_device_control()
620 	struct knav_queue *qh = qhandle;  in knav_queue_push()  local
624 	writel_relaxed(val, &qh->reg_push[0].ptr_size_thresh);  in knav_queue_push()
626 	atomic_inc(&qh->stats.pushes);  in knav_queue_push()
640 	struct knav_queue *qh = qhandle;  in knav_queue_pop()  local
641 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_pop()
655 		val = readl_relaxed(&qh->reg_pop[0].ptr_size_thresh);  in knav_queue_pop()
664 	atomic_inc(&qh->stats.pops);  in knav_queue_pop()