44 from_fence(struct fence *fence)  in from_fence()  argument
46 	return container_of(fence, struct nouveau_fence, base);  in from_fence()
50 nouveau_fctx(struct nouveau_fence *fence)  in nouveau_fctx()  argument
52 	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);  in nouveau_fctx()
56 nouveau_fence_signal(struct nouveau_fence *fence)  in nouveau_fence_signal()  argument
60 	fence_signal_locked(&fence->base);  in nouveau_fence_signal()
61 	list_del(&fence->head);  in nouveau_fence_signal()
62 	rcu_assign_pointer(fence->channel, NULL);  in nouveau_fence_signal()
64 	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {  in nouveau_fence_signal()
65 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_signal()
71 	fence_put(&fence->base);  in nouveau_fence_signal()
76 nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) {  in nouveau_local_fence()  argument
77 	struct nouveau_fence_priv *priv = (void*)drm->fence;  in nouveau_local_fence()
79 	if (fence->ops != &nouveau_fence_ops_legacy &&  in nouveau_local_fence()
80 	    fence->ops != &nouveau_fence_ops_uevent)  in nouveau_local_fence()
83 	if (fence->context < priv->context_base ||  in nouveau_local_fence()
84 	    fence->context >= priv->context_base + priv->contexts)  in nouveau_local_fence()
87 	return from_fence(fence);  in nouveau_local_fence()
93 	struct nouveau_fence *fence;  in nouveau_fence_context_del()  local
97 		fence = list_entry(fctx->pending.next, typeof(*fence), head);  in nouveau_fence_context_del()
99 		if (nouveau_fence_signal(fence))  in nouveau_fence_context_del()
129 	struct nouveau_fence *fence;  in nouveau_fence_update()  local
134 		fence = list_entry(fctx->pending.next, typeof(*fence), head);  in nouveau_fence_update()
136 		if ((int)(seq - fence->base.seqno) < 0)  in nouveau_fence_update()
139 		drop |= nouveau_fence_signal(fence);  in nouveau_fence_update()
155 		struct nouveau_fence *fence;  in nouveau_fence_wait_uevent_handler()  local
158 		fence = list_entry(fctx->pending.next, typeof(*fence), head);  in nouveau_fence_wait_uevent_handler()
159 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));  in nouveau_fence_wait_uevent_handler()
160 		if (nouveau_fence_update(fence->channel, fctx))  in nouveau_fence_wait_uevent_handler()
171 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;  in nouveau_fence_context_new()
217 static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)  in nouveau_fence_work_cb()  argument
225 nouveau_fence_work(struct fence *fence,  in nouveau_fence_work()  argument
230 	if (fence_is_signaled(fence))  in nouveau_fence_work()
239 		WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence,  in nouveau_fence_work()
248 	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)  in nouveau_fence_work()
259 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)  in nouveau_fence_emit()  argument
261 	struct nouveau_fence_chan *fctx = chan->fence;  in nouveau_fence_emit()
262 	struct nouveau_fence_priv *priv = (void*)chan->drm->fence;  in nouveau_fence_emit()
265 	fence->channel  = chan;  in nouveau_fence_emit()
266 	fence->timeout  = jiffies + (15 * HZ);  in nouveau_fence_emit()
269 		fence_init(&fence->base, &nouveau_fence_ops_uevent,  in nouveau_fence_emit()
272 		fence_init(&fence->base, &nouveau_fence_ops_legacy,  in nouveau_fence_emit()
276 	trace_fence_emit(&fence->base);  in nouveau_fence_emit()
277 	ret = fctx->emit(fence);  in nouveau_fence_emit()
279 		fence_get(&fence->base);  in nouveau_fence_emit()
285 		list_add_tail(&fence->head, &fctx->pending);  in nouveau_fence_emit()
293 nouveau_fence_done(struct nouveau_fence *fence)  in nouveau_fence_done()  argument
295 	if (fence->base.ops == &nouveau_fence_ops_legacy ||  in nouveau_fence_done()
296 	    fence->base.ops == &nouveau_fence_ops_uevent) {  in nouveau_fence_done()
297 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_done()
301 		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))  in nouveau_fence_done()
305 		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));  in nouveau_fence_done()
310 	return fence_is_signaled(&fence->base);  in nouveau_fence_done()
314 nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)  in nouveau_fence_wait_legacy()
316 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_wait_legacy()  local
320 	while (!nouveau_fence_done(fence)) {  in nouveau_fence_wait_legacy()
349 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)  in nouveau_fence_wait_busy()  argument
353 	while (!nouveau_fence_done(fence)) {  in nouveau_fence_wait_busy()
354 		if (time_after_eq(jiffies, fence->timeout)) {  in nouveau_fence_wait_busy()
374 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)  in nouveau_fence_wait()  argument
379 		return nouveau_fence_wait_busy(fence, intr);  in nouveau_fence_wait()
381 	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);  in nouveau_fence_wait()
393 	struct nouveau_fence_chan *fctx = chan->fence;  in nouveau_fence_sync()
394 	struct fence *fence;  in nouveau_fence_sync()  local
408 	fence = reservation_object_get_excl(resv);  in nouveau_fence_sync()
410 	if (fence && (!exclusive || !fobj || !fobj->shared_count)) {  in nouveau_fence_sync()
414 		f = nouveau_local_fence(fence, chan->drm);  in nouveau_fence_sync()
424 			ret = fence_wait(fence, intr);  in nouveau_fence_sync()
436 		fence = rcu_dereference_protected(fobj->shared[i],  in nouveau_fence_sync()
439 		f = nouveau_local_fence(fence, chan->drm);  in nouveau_fence_sync()
449 			ret = fence_wait(fence, intr);  in nouveau_fence_sync()
467 	struct nouveau_fence *fence;  in nouveau_fence_new()  local
470 	if (unlikely(!chan->fence))  in nouveau_fence_new()
473 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);  in nouveau_fence_new()
474 	if (!fence)  in nouveau_fence_new()
477 	fence->sysmem = sysmem;  in nouveau_fence_new()
479 	ret = nouveau_fence_emit(fence, chan);  in nouveau_fence_new()
481 		nouveau_fence_unref(&fence);  in nouveau_fence_new()
483 	*pfence = fence;  in nouveau_fence_new()
487 static const char *nouveau_fence_get_get_driver_name(struct fence *fence)  in nouveau_fence_get_get_driver_name()  argument
492 static const char *nouveau_fence_get_timeline_name(struct fence *f)  in nouveau_fence_get_timeline_name()
494 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_get_timeline_name()  local
495 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_get_timeline_name()
506 static bool nouveau_fence_is_signaled(struct fence *f)  in nouveau_fence_is_signaled()
508 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_is_signaled()  local
509 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_is_signaled()
514 	chan = rcu_dereference(fence->channel);  in nouveau_fence_is_signaled()
516 		ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;  in nouveau_fence_is_signaled()
522 static bool nouveau_fence_no_signaling(struct fence *f)  in nouveau_fence_no_signaling()
524 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_no_signaling()  local
530 	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);  in nouveau_fence_no_signaling()
538 		list_del(&fence->head);  in nouveau_fence_no_signaling()
540 		fence_put(&fence->base);  in nouveau_fence_no_signaling()
547 static void nouveau_fence_release(struct fence *f)  in nouveau_fence_release()
549 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_release()  local
550 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_release()
553 	fence_free(&fence->base);  in nouveau_fence_release()
565 static bool nouveau_fence_enable_signaling(struct fence *f)  in nouveau_fence_enable_signaling()
567 	struct nouveau_fence *fence = from_fence(f);  in nouveau_fence_enable_signaling()  local
568 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);  in nouveau_fence_enable_signaling()
576 		set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);  in nouveau_fence_enable_signaling()