66 int fence_signal_locked(struct fence *fence)  in fence_signal_locked()  argument
71 	if (WARN_ON(!fence))  in fence_signal_locked()
74 	if (!ktime_to_ns(fence->timestamp)) {  in fence_signal_locked()
75 		fence->timestamp = ktime_get();  in fence_signal_locked()
79 	if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in fence_signal_locked()
87 		trace_fence_signaled(fence);  in fence_signal_locked()
89 	list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {  in fence_signal_locked()
91 		cur->func(fence, cur);  in fence_signal_locked()
107 int fence_signal(struct fence *fence)  in fence_signal()  argument
111 	if (!fence)  in fence_signal()
114 	if (!ktime_to_ns(fence->timestamp)) {  in fence_signal()
115 		fence->timestamp = ktime_get();  in fence_signal()
119 	if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in fence_signal()
122 	trace_fence_signaled(fence);  in fence_signal()
124 	if (test_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) {  in fence_signal()
127 		spin_lock_irqsave(fence->lock, flags);  in fence_signal()
128 		list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {  in fence_signal()
130 			cur->func(fence, cur);  in fence_signal()
132 		spin_unlock_irqrestore(fence->lock, flags);  in fence_signal()
155 fence_wait_timeout(struct fence *fence, bool intr, signed long timeout)  in fence_wait_timeout()  argument
163 		return fence_is_signaled(fence);  in fence_wait_timeout()
165 	trace_fence_wait_start(fence);  in fence_wait_timeout()
166 	ret = fence->ops->wait(fence, intr, timeout);  in fence_wait_timeout()
167 	trace_fence_wait_end(fence);  in fence_wait_timeout()
174 	struct fence *fence =  in fence_release()  local
175 			container_of(kref, struct fence, refcount);  in fence_release()
177 	trace_fence_destroy(fence);  in fence_release()
179 	BUG_ON(!list_empty(&fence->cb_list));  in fence_release()
181 	if (fence->ops->release)  in fence_release()
182 		fence->ops->release(fence);  in fence_release()
184 		fence_free(fence);  in fence_release()
188 void fence_free(struct fence *fence)  in fence_free()  argument
190 	kfree_rcu(fence, rcu);  in fence_free()
201 void fence_enable_sw_signaling(struct fence *fence)  in fence_enable_sw_signaling()  argument
205 	if (!test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags) &&  in fence_enable_sw_signaling()
206 	    !test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in fence_enable_sw_signaling()
207 		trace_fence_enable_signal(fence);  in fence_enable_sw_signaling()
209 		spin_lock_irqsave(fence->lock, flags);  in fence_enable_sw_signaling()
211 		if (!fence->ops->enable_signaling(fence))  in fence_enable_sw_signaling()
212 			fence_signal_locked(fence);  in fence_enable_sw_signaling()
214 		spin_unlock_irqrestore(fence->lock, flags);  in fence_enable_sw_signaling()
242 int fence_add_callback(struct fence *fence, struct fence_cb *cb,  in fence_add_callback()  argument
249 	if (WARN_ON(!fence || !func))  in fence_add_callback()
252 	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in fence_add_callback()
257 	spin_lock_irqsave(fence->lock, flags);  in fence_add_callback()
259 	was_set = test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags);  in fence_add_callback()
261 	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in fence_add_callback()
264 		trace_fence_enable_signal(fence);  in fence_add_callback()
266 		if (!fence->ops->enable_signaling(fence)) {  in fence_add_callback()
267 			fence_signal_locked(fence);  in fence_add_callback()
274 		list_add_tail(&cb->node, &fence->cb_list);  in fence_add_callback()
277 	spin_unlock_irqrestore(fence->lock, flags);  in fence_add_callback()
299 fence_remove_callback(struct fence *fence, struct fence_cb *cb)  in fence_remove_callback()  argument
304 	spin_lock_irqsave(fence->lock, flags);  in fence_remove_callback()
310 	spin_unlock_irqrestore(fence->lock, flags);  in fence_remove_callback()
322 fence_default_wait_cb(struct fence *fence, struct fence_cb *cb)  in fence_default_wait_cb()  argument
341 fence_default_wait(struct fence *fence, bool intr, signed long timeout)  in fence_default_wait()  argument
348 	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in fence_default_wait()
351 	spin_lock_irqsave(fence->lock, flags);  in fence_default_wait()
358 	was_set = test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags);  in fence_default_wait()
360 	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in fence_default_wait()
364 		trace_fence_enable_signal(fence);  in fence_default_wait()
366 		if (!fence->ops->enable_signaling(fence)) {  in fence_default_wait()
367 			fence_signal_locked(fence);  in fence_default_wait()
374 	list_add(&cb.base.node, &fence->cb_list);  in fence_default_wait()
376 	while (!test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {  in fence_default_wait()
381 		spin_unlock_irqrestore(fence->lock, flags);  in fence_default_wait()
385 		spin_lock_irqsave(fence->lock, flags);  in fence_default_wait()
395 	spin_unlock_irqrestore(fence->lock, flags);  in fence_default_wait()
417 fence_init(struct fence *fence, const struct fence_ops *ops,  in fence_init()  argument
424 	kref_init(&fence->refcount);  in fence_init()
425 	fence->ops = ops;  in fence_init()
426 	INIT_LIST_HEAD(&fence->cb_list);  in fence_init()
427 	fence->lock = lock;  in fence_init()
428 	fence->context = context;  in fence_init()
429 	fence->seqno = seqno;  in fence_init()
430 	fence->flags = 0UL;  in fence_init()
432 	trace_fence_init(fence);  in fence_init()