66 void rcu_sync_lockdep_assert(struct rcu_sync *rsp)  in rcu_sync_lockdep_assert()  argument
68 	RCU_LOCKDEP_WARN(!gp_ops[rsp->gp_type].held(),  in rcu_sync_lockdep_assert()
78 void rcu_sync_init(struct rcu_sync *rsp, enum rcu_sync_type type)  in rcu_sync_init()  argument
80 	memset(rsp, 0, sizeof(*rsp));  in rcu_sync_init()
81 	init_waitqueue_head(&rsp->gp_wait);  in rcu_sync_init()
82 	rsp->gp_type = type;  in rcu_sync_init()
100 void rcu_sync_enter(struct rcu_sync *rsp)  in rcu_sync_enter()  argument
104 	spin_lock_irq(&rsp->rss_lock);  in rcu_sync_enter()
105 	need_wait = rsp->gp_count++;  in rcu_sync_enter()
106 	need_sync = rsp->gp_state == GP_IDLE;  in rcu_sync_enter()
108 		rsp->gp_state = GP_PENDING;  in rcu_sync_enter()
109 	spin_unlock_irq(&rsp->rss_lock);  in rcu_sync_enter()
114 		gp_ops[rsp->gp_type].sync();  in rcu_sync_enter()
115 		rsp->gp_state = GP_PASSED;  in rcu_sync_enter()
116 		wake_up_all(&rsp->gp_wait);  in rcu_sync_enter()
118 		wait_event(rsp->gp_wait, rsp->gp_state == GP_PASSED);  in rcu_sync_enter()
125 		BUG_ON(rsp->gp_state != GP_PASSED);  in rcu_sync_enter()
149 	struct rcu_sync *rsp = container_of(rcu, struct rcu_sync, cb_head);  in rcu_sync_func()  local
152 	BUG_ON(rsp->gp_state != GP_PASSED);  in rcu_sync_func()
153 	BUG_ON(rsp->cb_state == CB_IDLE);  in rcu_sync_func()
155 	spin_lock_irqsave(&rsp->rss_lock, flags);  in rcu_sync_func()
156 	if (rsp->gp_count) {  in rcu_sync_func()
160 		rsp->cb_state = CB_IDLE;  in rcu_sync_func()
161 	} else if (rsp->cb_state == CB_REPLAY) {  in rcu_sync_func()
166 		rsp->cb_state = CB_PENDING;  in rcu_sync_func()
167 		gp_ops[rsp->gp_type].call(&rsp->cb_head, rcu_sync_func);  in rcu_sync_func()
173 		rsp->cb_state = CB_IDLE;  in rcu_sync_func()
174 		rsp->gp_state = GP_IDLE;  in rcu_sync_func()
176 	spin_unlock_irqrestore(&rsp->rss_lock, flags);  in rcu_sync_func()
189 void rcu_sync_exit(struct rcu_sync *rsp)  in rcu_sync_exit()  argument
191 	spin_lock_irq(&rsp->rss_lock);  in rcu_sync_exit()
192 	if (!--rsp->gp_count) {  in rcu_sync_exit()
193 		if (rsp->cb_state == CB_IDLE) {  in rcu_sync_exit()
194 			rsp->cb_state = CB_PENDING;  in rcu_sync_exit()
195 			gp_ops[rsp->gp_type].call(&rsp->cb_head, rcu_sync_func);  in rcu_sync_exit()
196 		} else if (rsp->cb_state == CB_PENDING) {  in rcu_sync_exit()
197 			rsp->cb_state = CB_REPLAY;  in rcu_sync_exit()
200 	spin_unlock_irq(&rsp->rss_lock);  in rcu_sync_exit()
207 void rcu_sync_dtor(struct rcu_sync *rsp)  in rcu_sync_dtor()  argument
211 	BUG_ON(rsp->gp_count);  in rcu_sync_dtor()
213 	spin_lock_irq(&rsp->rss_lock);  in rcu_sync_dtor()
214 	if (rsp->cb_state == CB_REPLAY)  in rcu_sync_dtor()
215 		rsp->cb_state = CB_PENDING;  in rcu_sync_dtor()
216 	cb_state = rsp->cb_state;  in rcu_sync_dtor()
217 	spin_unlock_irq(&rsp->rss_lock);  in rcu_sync_dtor()
220 		gp_ops[rsp->gp_type].wait();  in rcu_sync_dtor()
221 		BUG_ON(rsp->cb_state != CB_IDLE);  in rcu_sync_dtor()