11 int __percpu_init_rwsem(struct percpu_rw_semaphore *brw,  in __percpu_init_rwsem()  argument
14 	brw->fast_read_ctr = alloc_percpu(int);  in __percpu_init_rwsem()
15 	if (unlikely(!brw->fast_read_ctr))  in __percpu_init_rwsem()
19 	__init_rwsem(&brw->rw_sem, name, rwsem_key);  in __percpu_init_rwsem()
20 	atomic_set(&brw->write_ctr, 0);  in __percpu_init_rwsem()
21 	atomic_set(&brw->slow_read_ctr, 0);  in __percpu_init_rwsem()
22 	init_waitqueue_head(&brw->write_waitq);  in __percpu_init_rwsem()
26 void percpu_free_rwsem(struct percpu_rw_semaphore *brw)  in percpu_free_rwsem()  argument
28 	free_percpu(brw->fast_read_ctr);  in percpu_free_rwsem()
29 	brw->fast_read_ctr = NULL; /* catch use after free bugs */  in percpu_free_rwsem()
55 static bool update_fast_ctr(struct percpu_rw_semaphore *brw, unsigned int val)  in update_fast_ctr()  argument
60 	if (likely(!atomic_read(&brw->write_ctr))) {  in update_fast_ctr()
61 		__this_cpu_add(*brw->fast_read_ctr, val);  in update_fast_ctr()
77 void percpu_down_read(struct percpu_rw_semaphore *brw)  in percpu_down_read()  argument
80 	if (likely(update_fast_ctr(brw, +1))) {  in percpu_down_read()
81 		rwsem_acquire_read(&brw->rw_sem.dep_map, 0, 0, _RET_IP_);  in percpu_down_read()
85 	down_read(&brw->rw_sem);  in percpu_down_read()
86 	atomic_inc(&brw->slow_read_ctr);  in percpu_down_read()
88 	__up_read(&brw->rw_sem);  in percpu_down_read()
91 void percpu_up_read(struct percpu_rw_semaphore *brw)  in percpu_up_read()  argument
93 	rwsem_release(&brw->rw_sem.dep_map, 1, _RET_IP_);  in percpu_up_read()
95 	if (likely(update_fast_ctr(brw, -1)))  in percpu_up_read()
99 	if (atomic_dec_and_test(&brw->slow_read_ctr))  in percpu_up_read()
100 		wake_up_all(&brw->write_waitq);  in percpu_up_read()
103 static int clear_fast_ctr(struct percpu_rw_semaphore *brw)  in clear_fast_ctr()  argument
109 		sum += per_cpu(*brw->fast_read_ctr, cpu);  in clear_fast_ctr()
110 		per_cpu(*brw->fast_read_ctr, cpu) = 0;  in clear_fast_ctr()
127 void percpu_down_write(struct percpu_rw_semaphore *brw)  in percpu_down_write()  argument
130 	atomic_inc(&brw->write_ctr);  in percpu_down_write()
145 	down_write(&brw->rw_sem);  in percpu_down_write()
148 	atomic_add(clear_fast_ctr(brw), &brw->slow_read_ctr);  in percpu_down_write()
151 	wait_event(brw->write_waitq, !atomic_read(&brw->slow_read_ctr));  in percpu_down_write()
154 void percpu_up_write(struct percpu_rw_semaphore *brw)  in percpu_up_write()  argument
157 	up_write(&brw->rw_sem);  in percpu_up_write()
164 	atomic_dec(&brw->write_ctr);  in percpu_up_write()