345 static inline void hb_waiters_inc(struct futex_hash_bucket *hb)  in hb_waiters_inc()  argument
348 	atomic_inc(&hb->waiters);  in hb_waiters_inc()
360 static inline void hb_waiters_dec(struct futex_hash_bucket *hb)  in hb_waiters_dec()  argument
363 	atomic_dec(&hb->waiters);  in hb_waiters_dec()
367 static inline int hb_waiters_pending(struct futex_hash_bucket *hb)  in hb_waiters_pending()  argument
370 	return atomic_read(&hb->waiters);  in hb_waiters_pending()
655 static struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb,  in futex_top_waiter()  argument
660 	plist_for_each_entry(this, &hb->chain, list) {  in futex_top_waiter()
791 	struct futex_hash_bucket *hb;  in exit_pi_state_list()  local
807 		hb = hash_futex(&key);  in exit_pi_state_list()
810 		spin_lock(&hb->lock);  in exit_pi_state_list()
818 			spin_unlock(&hb->lock);  in exit_pi_state_list()
830 		spin_unlock(&hb->lock);  in exit_pi_state_list()
1031 static int lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,  in lookup_pi_state()  argument
1034 	struct futex_q *match = futex_top_waiter(hb, key);  in lookup_pi_state()
1082 static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,  in futex_lock_pi_atomic()  argument
1114 	match = futex_top_waiter(hb, key);  in futex_lock_pi_atomic()
1166 	struct futex_hash_bucket *hb;  in __unqueue_futex()  local
1172 	hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);  in __unqueue_futex()
1173 	plist_del(&q->list, &hb->chain);  in __unqueue_futex()
1174 	hb_waiters_dec(hb);  in __unqueue_futex()
1207 			 struct futex_hash_bucket *hb)  in wake_futex_pi()  argument
1287 	spin_unlock(&hb->lock);  in wake_futex_pi()
1325 	struct futex_hash_bucket *hb;  in futex_wake()  local
1338 	hb = hash_futex(&key);  in futex_wake()
1341 	if (!hb_waiters_pending(hb))  in futex_wake()
1344 	spin_lock(&hb->lock);  in futex_wake()
1346 	plist_for_each_entry_safe(this, next, &hb->chain, list) {  in futex_wake()
1363 	spin_unlock(&hb->lock);  in futex_wake()
1511 			   struct futex_hash_bucket *hb)  in requeue_pi_wake_futex()  argument
1521 	q->lock_ptr = &hb->lock;  in requeue_pi_wake_futex()
1861 	__acquires(&hb->lock)  in queue_lock()
1863 	struct futex_hash_bucket *hb;  in queue_lock()  local
1865 	hb = hash_futex(&q->key);  in queue_lock()
1875 	hb_waiters_inc(hb);  in queue_lock()
1877 	q->lock_ptr = &hb->lock;  in queue_lock()
1879 	spin_lock(&hb->lock); /* implies MB (A) */  in queue_lock()
1880 	return hb;  in queue_lock()
1884 queue_unlock(struct futex_hash_bucket *hb)  in queue_unlock()  argument
1885 	__releases(&hb->lock)  in queue_unlock()
1887 	spin_unlock(&hb->lock);  in queue_unlock()
1888 	hb_waiters_dec(hb);  in queue_unlock()
1903 static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)  in queue_me()  argument
1904 	__releases(&hb->lock)  in queue_me()
1919 	plist_add(&q->list, &hb->chain);  in queue_me()
1921 	spin_unlock(&hb->lock);  in queue_me()
2171 static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,  in futex_wait_queue_me()  argument
2181 	queue_me(q, hb);  in futex_wait_queue_me()
2221 			   struct futex_q *q, struct futex_hash_bucket **hb)  in futex_wait_setup()  argument
2250 	*hb = queue_lock(q);  in futex_wait_setup()
2255 		queue_unlock(*hb);  in futex_wait_setup()
2269 		queue_unlock(*hb);  in futex_wait_setup()
2284 	struct futex_hash_bucket *hb;  in futex_wait()  local
2308 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait()
2313 	futex_wait_queue_me(hb, &q, to);  in futex_wait()
2383 	struct futex_hash_bucket *hb;  in futex_lock_pi()  local
2404 	hb = queue_lock(&q);  in futex_lock_pi()
2406 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, 0);  in futex_lock_pi()
2426 			queue_unlock(hb);  in futex_lock_pi()
2438 	queue_me(&q, hb);  in futex_lock_pi()
2478 	queue_unlock(hb);  in futex_lock_pi()
2488 	queue_unlock(hb);  in futex_lock_pi()
2510 	struct futex_hash_bucket *hb;  in futex_unlock_pi()  local
2527 	hb = hash_futex(&key);  in futex_unlock_pi()
2528 	spin_lock(&hb->lock);  in futex_unlock_pi()
2535 	match = futex_top_waiter(hb, &key);  in futex_unlock_pi()
2537 		ret = wake_futex_pi(uaddr, uval, match, hb);  in futex_unlock_pi()
2555 			spin_unlock(&hb->lock);  in futex_unlock_pi()
2582 	spin_unlock(&hb->lock);  in futex_unlock_pi()
2588 	spin_unlock(&hb->lock);  in futex_unlock_pi()
2615 int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,  in handle_early_requeue_pi_wakeup()  argument
2629 		WARN_ON(q->lock_ptr && (&hb->lock != q->lock_ptr));  in handle_early_requeue_pi_wakeup()
2634 		plist_del(&q->list, &hb->chain);  in handle_early_requeue_pi_wakeup()
2635 		hb_waiters_dec(hb);  in handle_early_requeue_pi_wakeup()
2694 	struct futex_hash_bucket *hb;  in futex_wait_requeue_pi()  local
2736 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait_requeue_pi()
2745 		queue_unlock(hb);  in futex_wait_requeue_pi()
2751 	futex_wait_queue_me(hb, &q, to);  in futex_wait_requeue_pi()
2753 	spin_lock(&hb->lock);  in futex_wait_requeue_pi()
2754 	ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);  in futex_wait_requeue_pi()
2755 	spin_unlock(&hb->lock);  in futex_wait_requeue_pi()