1164 static void __unqueue_futex(struct futex_q *q)  in __unqueue_futex()  argument
1168 	if (WARN_ON_SMP(!q->lock_ptr || !spin_is_locked(q->lock_ptr))  in __unqueue_futex()
1169 	    || WARN_ON(plist_node_empty(&q->list)))  in __unqueue_futex()
1172 	hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);  in __unqueue_futex()
1173 	plist_del(&q->list, &hb->chain);  in __unqueue_futex()
1183 static void mark_wake_futex(struct wake_q_head *wake_q, struct futex_q *q)  in mark_wake_futex()  argument
1185 	struct task_struct *p = q->task;  in mark_wake_futex()
1187 	if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))  in mark_wake_futex()
1195 	__unqueue_futex(q);  in mark_wake_futex()
1203 	q->lock_ptr = NULL;  in mark_wake_futex()
1476 void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,  in requeue_futex()  argument
1485 		plist_del(&q->list, &hb1->chain);  in requeue_futex()
1488 		plist_add(&q->list, &hb2->chain);  in requeue_futex()
1489 		q->lock_ptr = &hb2->lock;  in requeue_futex()
1492 	q->key = *key2;  in requeue_futex()
1510 void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,  in requeue_pi_wake_futex()  argument
1514 	q->key = *key;  in requeue_pi_wake_futex()
1516 	__unqueue_futex(q);  in requeue_pi_wake_futex()
1518 	WARN_ON(!q->rt_waiter);  in requeue_pi_wake_futex()
1519 	q->rt_waiter = NULL;  in requeue_pi_wake_futex()
1521 	q->lock_ptr = &hb->lock;  in requeue_pi_wake_futex()
1523 	wake_up_state(q->task, TASK_NORMAL);  in requeue_pi_wake_futex()
1860 static inline struct futex_hash_bucket *queue_lock(struct futex_q *q)  in queue_lock()  argument
1865 	hb = hash_futex(&q->key);  in queue_lock()
1877 	q->lock_ptr = &hb->lock;  in queue_lock()
1903 static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)  in queue_me()  argument
1918 	plist_node_init(&q->list, prio);  in queue_me()
1919 	plist_add(&q->list, &hb->chain);  in queue_me()
1920 	q->task = current;  in queue_me()
1935 static int unqueue_me(struct futex_q *q)  in unqueue_me()  argument
1942 	lock_ptr = q->lock_ptr;  in unqueue_me()
1959 		if (unlikely(lock_ptr != q->lock_ptr)) {  in unqueue_me()
1963 		__unqueue_futex(q);  in unqueue_me()
1965 		BUG_ON(q->pi_state);  in unqueue_me()
1971 	drop_futex_key_refs(&q->key);  in unqueue_me()
1980 static void unqueue_me_pi(struct futex_q *q)  in unqueue_me_pi()  argument
1981 	__releases(q->lock_ptr)  in unqueue_me_pi()
1983 	__unqueue_futex(q);  in unqueue_me_pi()
1985 	BUG_ON(!q->pi_state);  in unqueue_me_pi()
1986 	free_pi_state(q->pi_state);  in unqueue_me_pi()
1987 	q->pi_state = NULL;  in unqueue_me_pi()
1989 	spin_unlock(q->lock_ptr);  in unqueue_me_pi()
1998 static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,  in fixup_pi_state_owner()  argument
2002 	struct futex_pi_state *pi_state = q->pi_state;  in fixup_pi_state_owner()
2072 	spin_unlock(q->lock_ptr);  in fixup_pi_state_owner()
2076 	spin_lock(q->lock_ptr);  in fixup_pi_state_owner()
2107 static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked)  in fixup_owner()  argument
2117 		if (q->pi_state->owner != current)  in fixup_owner()
2118 			ret = fixup_pi_state_owner(uaddr, q, current);  in fixup_owner()
2126 	if (q->pi_state->owner == current) {  in fixup_owner()
2132 		if (rt_mutex_trylock(&q->pi_state->pi_mutex)) {  in fixup_owner()
2142 		raw_spin_lock(&q->pi_state->pi_mutex.wait_lock);  in fixup_owner()
2143 		owner = rt_mutex_owner(&q->pi_state->pi_mutex);  in fixup_owner()
2145 			owner = rt_mutex_next_owner(&q->pi_state->pi_mutex);  in fixup_owner()
2146 		raw_spin_unlock(&q->pi_state->pi_mutex.wait_lock);  in fixup_owner()
2147 		ret = fixup_pi_state_owner(uaddr, q, owner);  in fixup_owner()
2155 	if (rt_mutex_owner(&q->pi_state->pi_mutex) == current)  in fixup_owner()
2158 				q->pi_state->pi_mutex.owner,  in fixup_owner()
2159 				q->pi_state->owner);  in fixup_owner()
2171 static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,  in futex_wait_queue_me()  argument
2181 	queue_me(q, hb);  in futex_wait_queue_me()
2191 	if (likely(!plist_node_empty(&q->list))) {  in futex_wait_queue_me()
2221 			   struct futex_q *q, struct futex_hash_bucket **hb)  in futex_wait_setup()  argument
2245 	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, VERIFY_READ);  in futex_wait_setup()
2250 	*hb = queue_lock(q);  in futex_wait_setup()
2264 		put_futex_key(&q->key);  in futex_wait_setup()
2275 		put_futex_key(&q->key);  in futex_wait_setup()
2285 	struct futex_q q = futex_q_init;  in futex_wait()  local
2290 	q.bitset = bitset;  in futex_wait()
2308 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait()
2313 	futex_wait_queue_me(hb, &q, to);  in futex_wait()
2318 	if (!unqueue_me(&q))  in futex_wait()
2384 	struct futex_q q = futex_q_init;  in futex_lock_pi()  local
2399 	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, VERIFY_WRITE);  in futex_lock_pi()
2404 	hb = queue_lock(&q);  in futex_lock_pi()
2406 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, 0);  in futex_lock_pi()
2427 			put_futex_key(&q.key);  in futex_lock_pi()
2438 	queue_me(&q, hb);  in futex_lock_pi()
2440 	WARN_ON(!q.pi_state);  in futex_lock_pi()
2445 		ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);  in futex_lock_pi()
2447 		ret = rt_mutex_trylock(&q.pi_state->pi_mutex);  in futex_lock_pi()
2452 	spin_lock(q.lock_ptr);  in futex_lock_pi()
2457 	res = fixup_owner(uaddr, &q, !ret);  in futex_lock_pi()
2469 	if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current))  in futex_lock_pi()
2470 		rt_mutex_unlock(&q.pi_state->pi_mutex);  in futex_lock_pi()
2473 	unqueue_me_pi(&q);  in futex_lock_pi()
2481 	put_futex_key(&q.key);  in futex_lock_pi()
2497 	put_futex_key(&q.key);  in futex_lock_pi()
2616 				   struct futex_q *q, union futex_key *key2,  in handle_early_requeue_pi_wakeup()  argument
2628 	if (!match_futex(&q->key, key2)) {  in handle_early_requeue_pi_wakeup()
2629 		WARN_ON(q->lock_ptr && (&hb->lock != q->lock_ptr));  in handle_early_requeue_pi_wakeup()
2634 		plist_del(&q->list, &hb->chain);  in handle_early_requeue_pi_wakeup()
2696 	struct futex_q q = futex_q_init;  in futex_wait_requeue_pi()  local
2728 	q.bitset = bitset;  in futex_wait_requeue_pi()
2729 	q.rt_waiter = &rt_waiter;  in futex_wait_requeue_pi()
2730 	q.requeue_pi_key = &key2;  in futex_wait_requeue_pi()
2736 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait_requeue_pi()
2744 	if (match_futex(&q.key, &key2)) {  in futex_wait_requeue_pi()
2751 	futex_wait_queue_me(hb, &q, to);  in futex_wait_requeue_pi()
2754 	ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);  in futex_wait_requeue_pi()
2769 	if (!q.rt_waiter) {  in futex_wait_requeue_pi()
2774 		if (q.pi_state && (q.pi_state->owner != current)) {  in futex_wait_requeue_pi()
2775 			spin_lock(q.lock_ptr);  in futex_wait_requeue_pi()
2776 			ret = fixup_pi_state_owner(uaddr2, &q, current);  in futex_wait_requeue_pi()
2781 			free_pi_state(q.pi_state);  in futex_wait_requeue_pi()
2782 			spin_unlock(q.lock_ptr);  in futex_wait_requeue_pi()
2790 		WARN_ON(!q.pi_state);  in futex_wait_requeue_pi()
2791 		pi_mutex = &q.pi_state->pi_mutex;  in futex_wait_requeue_pi()
2795 		spin_lock(q.lock_ptr);  in futex_wait_requeue_pi()
2800 		res = fixup_owner(uaddr2, &q, !ret);  in futex_wait_requeue_pi()
2809 		unqueue_me_pi(&q);  in futex_wait_requeue_pi()
2831 	put_futex_key(&q.key);  in futex_wait_requeue_pi()