44 from_fence(struct fence *fence)  in from_fence()  argument
46 	return container_of(fence, struct nouveau_fence, base);  in from_fence()
50 nouveau_fctx(struct nouveau_fence *fence)  in nouveau_fctx()  argument
52 	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);  in nouveau_fctx()
56 nouveau_fence_signal(struct nouveau_fence *fence)  in nouveau_fence_signal()  argument
60 	fence_signal_locked(&fence->base);  in nouveau_fence_signal()
61 	list_del(&fence->head);  in nouveau_fence_signal()
62 	rcu_assign_pointer(fence->channel, NULL);  in nouveau_fence_signal()
64 	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {  in nouveau_fence_signal()
65 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_signal()
71 	fence_put(&fence->base);  in nouveau_fence_signal()
76 nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) {  in nouveau_local_fence()  argument
77 	struct nouveau_fence_priv *priv = (void*)drm->fence;  in nouveau_local_fence()
79 	if (fence->ops != &nouveau_fence_ops_legacy &&  in nouveau_local_fence()
80 	    fence->ops != &nouveau_fence_ops_uevent)  in nouveau_local_fence()
83 	if (fence->context < priv->context_base ||  in nouveau_local_fence()
84 	    fence->context >= priv->context_base + priv->contexts)  in nouveau_local_fence()
87 	return from_fence(fence);  in nouveau_local_fence()
93 	struct nouveau_fence *fence;  in nouveau_fence_context_del()  local
97 		fence = list_entry(fctx->pending.next, typeof(*fence), head);  in nouveau_fence_context_del()
99 		if (nouveau_fence_signal(fence))  in nouveau_fence_context_del()
129 	struct nouveau_fence *fence;  in nouveau_fence_update()  local
134 		fence = list_entry(fctx->pending.next, typeof(*fence), head);  in nouveau_fence_update()
136 		if ((int)(seq - fence->base.seqno) < 0)  in nouveau_fence_update()
139 		drop |= nouveau_fence_signal(fence);  in nouveau_fence_update()
155 		struct nouveau_fence *fence;  in nouveau_fence_wait_uevent_handler()  local
158 		fence = list_entry(fctx->pending.next, typeof(*fence), head);  in nouveau_fence_wait_uevent_handler()
159 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));  in nouveau_fence_wait_uevent_handler()
160 		if (nouveau_fence_update(fence->channel, fctx))  in nouveau_fence_wait_uevent_handler()
171 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;  in nouveau_fence_context_new()
216 static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)  in nouveau_fence_work_cb()  argument
224 nouveau_fence_work(struct fence *fence,  in nouveau_fence_work()  argument
229 	if (fence_is_signaled(fence))  in nouveau_fence_work()
238 		WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,  in nouveau_fence_work()
247 	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)  in nouveau_fence_work()
258 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)  in nouveau_fence_emit()  argument
260 	struct nouveau_fence_chan *fctx = chan->fence;  in nouveau_fence_emit()
261 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;  in nouveau_fence_emit()
264 	fence->channel  = chan;  in nouveau_fence_emit()
265 	fence->timeout  = jiffies + (15 * HZ);  in nouveau_fence_emit()
268 		fence_init(&fence->base, &nouveau_fence_ops_uevent,  in nouveau_fence_emit()
271 		fence_init(&fence->base, &nouveau_fence_ops_legacy,  in nouveau_fence_emit()
275 	trace_fence_emit(&fence->base);  in nouveau_fence_emit()
276 	ret = fctx->emit(fence);  in nouveau_fence_emit()
278 		fence_get(&fence->base);  in nouveau_fence_emit()
284 		list_add_tail(&fence->head, &fctx->pending);  in nouveau_fence_emit()
292 nouveau_fence_done(struct nouveau_fence *fence)  in nouveau_fence_done()  argument
294 	if (fence->base.ops == &nouveau_fence_ops_legacy ||  in nouveau_fence_done()
295 	    fence->base.ops == &nouveau_fence_ops_uevent) {  in nouveau_fence_done()
296 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_done()
300 		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))  in nouveau_fence_done()
304 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));  in nouveau_fence_done()
309 	return fence_is_signaled(&fence->base);  in nouveau_fence_done()
313 nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)  in nouveau_fence_wait_legacy()
315 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_wait_legacy()  local
319 	while (!nouveau_fence_done(fence)) {  in nouveau_fence_wait_legacy()
348 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)  in nouveau_fence_wait_busy()  argument
352 	while (!nouveau_fence_done(fence)) {  in nouveau_fence_wait_busy()
353 		if (time_after_eq(jiffies, fence->timeout)) {  in nouveau_fence_wait_busy()
373 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)  in nouveau_fence_wait()  argument
378 		return nouveau_fence_wait_busy(fence, intr);  in nouveau_fence_wait()
380 	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);  in nouveau_fence_wait()
392 	struct nouveau_fence_chan *fctx = chan->fence;  in nouveau_fence_sync()
393 	struct fence *fence;  in nouveau_fence_sync()  local
407 	fence = reservation_object_get_excl(resv);  in nouveau_fence_sync()
409 	if (fence && (!exclusive || !fobj || !fobj->shared_count)) {  in nouveau_fence_sync()
413 		f = nouveau_local_fence(fence, chan->drm);  in nouveau_fence_sync()
423 			ret = fence_wait(fence, intr);  in nouveau_fence_sync()
435 		fence = rcu_dereference_protected(fobj->shared[i],  in nouveau_fence_sync()
438 		f = nouveau_local_fence(fence, chan->drm);  in nouveau_fence_sync()
448 			ret = fence_wait(fence, intr);  in nouveau_fence_sync()
466 	struct nouveau_fence *fence;  in nouveau_fence_new()  local
469 	if (unlikely(!chan->fence))  in nouveau_fence_new()
472 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);  in nouveau_fence_new()
473 	if (!fence)  in nouveau_fence_new()
476 	fence->sysmem = sysmem;  in nouveau_fence_new()
478 	ret = nouveau_fence_emit(fence, chan);  in nouveau_fence_new()
480 		nouveau_fence_unref(&fence);  in nouveau_fence_new()
482 	*pfence = fence;  in nouveau_fence_new()
486 static const char *nouveau_fence_get_get_driver_name(struct fence *fence)  in nouveau_fence_get_get_driver_name()  argument
491 static const char *nouveau_fence_get_timeline_name(struct fence *f)  in nouveau_fence_get_timeline_name()
493 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_get_timeline_name()  local
494 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_get_timeline_name()
505 static bool nouveau_fence_is_signaled(struct fence *f)  in nouveau_fence_is_signaled()
507 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_is_signaled()  local
508 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_is_signaled()
513 	chan = rcu_dereference(fence->channel);  in nouveau_fence_is_signaled()
515 		ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;  in nouveau_fence_is_signaled()
521 static bool nouveau_fence_no_signaling(struct fence *f)  in nouveau_fence_no_signaling()
523 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_no_signaling()  local
529 	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);  in nouveau_fence_no_signaling()
537 		list_del(&fence->head);  in nouveau_fence_no_signaling()
539 		fence_put(&fence->base);  in nouveau_fence_no_signaling()
546 static void nouveau_fence_release(struct fence *f)  in nouveau_fence_release()
548 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_release()  local
549 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_release()
552 	fence_free(&fence->base);  in nouveau_fence_release()
564 static bool nouveau_fence_enable_signaling(struct fence *f)  in nouveau_fence_enable_signaling()
566 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_enable_signaling()  local
567 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_enable_signaling()
575 		set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);  in nouveau_fence_enable_signaling()