11 int __percpu_init_rwsem(struct percpu_rw_semaphore *brw,  in __percpu_init_rwsem()  argument
14 	brw->fast_read_ctr = alloc_percpu(int);  in __percpu_init_rwsem()
15 	if (unlikely(!brw->fast_read_ctr))  in __percpu_init_rwsem()
19 	__init_rwsem(&brw->rw_sem, name, rwsem_key);  in __percpu_init_rwsem()
20 	rcu_sync_init(&brw->rss, RCU_SCHED_SYNC);  in __percpu_init_rwsem()
21 	atomic_set(&brw->slow_read_ctr, 0);  in __percpu_init_rwsem()
22 	init_waitqueue_head(&brw->write_waitq);  in __percpu_init_rwsem()
27 void percpu_free_rwsem(struct percpu_rw_semaphore *brw)  in percpu_free_rwsem()  argument
33 	if (!brw->fast_read_ctr)  in percpu_free_rwsem()
36 	rcu_sync_dtor(&brw->rss);  in percpu_free_rwsem()
37 	free_percpu(brw->fast_read_ctr);  in percpu_free_rwsem()
38 	brw->fast_read_ctr = NULL; /* catch use after free bugs */  in percpu_free_rwsem()
49 static bool update_fast_ctr(struct percpu_rw_semaphore *brw, unsigned int val)  in update_fast_ctr()  argument
54 	success = rcu_sync_is_idle(&brw->rss);  in update_fast_ctr()
56 		__this_cpu_add(*brw->fast_read_ctr, val);  in update_fast_ctr()
70 void percpu_down_read(struct percpu_rw_semaphore *brw)  in percpu_down_read()  argument
73 	rwsem_acquire_read(&brw->rw_sem.dep_map, 0, 0, _RET_IP_);  in percpu_down_read()
75 	if (likely(update_fast_ctr(brw, +1)))  in percpu_down_read()
79 	__down_read(&brw->rw_sem);  in percpu_down_read()
80 	atomic_inc(&brw->slow_read_ctr);  in percpu_down_read()
81 	__up_read(&brw->rw_sem);  in percpu_down_read()
85 int percpu_down_read_trylock(struct percpu_rw_semaphore *brw)  in percpu_down_read_trylock()  argument
87 	if (unlikely(!update_fast_ctr(brw, +1))) {  in percpu_down_read_trylock()
88 		if (!__down_read_trylock(&brw->rw_sem))  in percpu_down_read_trylock()
90 		atomic_inc(&brw->slow_read_ctr);  in percpu_down_read_trylock()
91 		__up_read(&brw->rw_sem);  in percpu_down_read_trylock()
94 	rwsem_acquire_read(&brw->rw_sem.dep_map, 0, 1, _RET_IP_);  in percpu_down_read_trylock()
98 void percpu_up_read(struct percpu_rw_semaphore *brw)  in percpu_up_read()  argument
100 	rwsem_release(&brw->rw_sem.dep_map, 1, _RET_IP_);  in percpu_up_read()
102 	if (likely(update_fast_ctr(brw, -1)))  in percpu_up_read()
106 	if (atomic_dec_and_test(&brw->slow_read_ctr))  in percpu_up_read()
107 		wake_up_all(&brw->write_waitq);  in percpu_up_read()
111 static int clear_fast_ctr(struct percpu_rw_semaphore *brw)  in clear_fast_ctr()  argument
117 		sum += per_cpu(*brw->fast_read_ctr, cpu);  in clear_fast_ctr()
118 		per_cpu(*brw->fast_read_ctr, cpu) = 0;  in clear_fast_ctr()
124 void percpu_down_write(struct percpu_rw_semaphore *brw)  in percpu_down_write()  argument
134 	rcu_sync_enter(&brw->rss);  in percpu_down_write()
137 	down_write(&brw->rw_sem);  in percpu_down_write()
140 	atomic_add(clear_fast_ctr(brw), &brw->slow_read_ctr);  in percpu_down_write()
143 	wait_event(brw->write_waitq, !atomic_read(&brw->slow_read_ctr));  in percpu_down_write()
147 void percpu_up_write(struct percpu_rw_semaphore *brw)  in percpu_up_write()  argument
150 	up_write(&brw->rw_sem);  in percpu_up_write()
156 	rcu_sync_exit(&brw->rss);  in percpu_up_write()