99 static int init_srcu_struct_fields(struct srcu_struct *sp)  in init_srcu_struct_fields()  argument
101 	sp->completed = 0;  in init_srcu_struct_fields()
102 	spin_lock_init(&sp->queue_lock);  in init_srcu_struct_fields()
103 	sp->running = false;  in init_srcu_struct_fields()
104 	rcu_batch_init(&sp->batch_queue);  in init_srcu_struct_fields()
105 	rcu_batch_init(&sp->batch_check0);  in init_srcu_struct_fields()
106 	rcu_batch_init(&sp->batch_check1);  in init_srcu_struct_fields()
107 	rcu_batch_init(&sp->batch_done);  in init_srcu_struct_fields()
108 	INIT_DELAYED_WORK(&sp->work, process_srcu);  in init_srcu_struct_fields()
109 	sp->per_cpu_ref = alloc_percpu(struct srcu_struct_array);  in init_srcu_struct_fields()
110 	return sp->per_cpu_ref ? 0 : -ENOMEM;  in init_srcu_struct_fields()
115 int __init_srcu_struct(struct srcu_struct *sp, const char *name,  in __init_srcu_struct()  argument
119 	debug_check_no_locks_freed((void *)sp, sizeof(*sp));  in __init_srcu_struct()
120 	lockdep_init_map(&sp->dep_map, name, key, 0);  in __init_srcu_struct()
121 	return init_srcu_struct_fields(sp);  in __init_srcu_struct()
135 int init_srcu_struct(struct srcu_struct *sp)  in init_srcu_struct()  argument
137 	return init_srcu_struct_fields(sp);  in init_srcu_struct()
147 static unsigned long srcu_readers_seq_idx(struct srcu_struct *sp, int idx)  in srcu_readers_seq_idx()  argument
154 		t = ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->seq[idx]);  in srcu_readers_seq_idx()
164 static unsigned long srcu_readers_active_idx(struct srcu_struct *sp, int idx)  in srcu_readers_active_idx()  argument
171 		t = ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[idx]);  in srcu_readers_active_idx()
186 static bool srcu_readers_active_idx_check(struct srcu_struct *sp, int idx)  in srcu_readers_active_idx_check()  argument
190 	seq = srcu_readers_seq_idx(sp, idx);  in srcu_readers_active_idx_check()
223 	if (srcu_readers_active_idx(sp, idx) != 0)  in srcu_readers_active_idx_check()
251 	return srcu_readers_seq_idx(sp, idx) == seq;  in srcu_readers_active_idx_check()
262 static int srcu_readers_active(struct srcu_struct *sp)  in srcu_readers_active()  argument
268 		sum += ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[0]);  in srcu_readers_active()
269 		sum += ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[1]);  in srcu_readers_active()
281 void cleanup_srcu_struct(struct srcu_struct *sp)  in cleanup_srcu_struct()  argument
283 	if (WARN_ON(srcu_readers_active(sp)))  in cleanup_srcu_struct()
285 	free_percpu(sp->per_cpu_ref);  in cleanup_srcu_struct()
286 	sp->per_cpu_ref = NULL;  in cleanup_srcu_struct()
295 int __srcu_read_lock(struct srcu_struct *sp)  in __srcu_read_lock()  argument
299 	idx = ACCESS_ONCE(sp->completed) & 0x1;  in __srcu_read_lock()
301 	__this_cpu_inc(sp->per_cpu_ref->c[idx]);  in __srcu_read_lock()
303 	__this_cpu_inc(sp->per_cpu_ref->seq[idx]);  in __srcu_read_lock()
315 void __srcu_read_unlock(struct srcu_struct *sp, int idx)  in __srcu_read_unlock()  argument
318 	this_cpu_dec(sp->per_cpu_ref->c[idx]);  in __srcu_read_unlock()
340 static bool try_check_zero(struct srcu_struct *sp, int idx, int trycount)  in try_check_zero()  argument
343 		if (srcu_readers_active_idx_check(sp, idx))  in try_check_zero()
356 static void srcu_flip(struct srcu_struct *sp)  in srcu_flip()  argument
358 	sp->completed++;  in srcu_flip()
388 void call_srcu(struct srcu_struct *sp, struct rcu_head *head,  in call_srcu()  argument
395 	spin_lock_irqsave(&sp->queue_lock, flags);  in call_srcu()
396 	rcu_batch_queue(&sp->batch_queue, head);  in call_srcu()
397 	if (!sp->running) {  in call_srcu()
398 		sp->running = true;  in call_srcu()
399 		queue_delayed_work(system_power_efficient_wq, &sp->work, 0);  in call_srcu()
401 	spin_unlock_irqrestore(&sp->queue_lock, flags);  in call_srcu()
405 static void srcu_advance_batches(struct srcu_struct *sp, int trycount);
406 static void srcu_reschedule(struct srcu_struct *sp);
411 static void __synchronize_srcu(struct srcu_struct *sp, int trycount)  in __synchronize_srcu()  argument
417 	rcu_lockdep_assert(!lock_is_held(&sp->dep_map) &&  in __synchronize_srcu()
428 	spin_lock_irq(&sp->queue_lock);  in __synchronize_srcu()
429 	if (!sp->running) {  in __synchronize_srcu()
431 		sp->running = true;  in __synchronize_srcu()
432 		rcu_batch_queue(&sp->batch_check0, head);  in __synchronize_srcu()
433 		spin_unlock_irq(&sp->queue_lock);  in __synchronize_srcu()
435 		srcu_advance_batches(sp, trycount);  in __synchronize_srcu()
436 		if (!rcu_batch_empty(&sp->batch_done)) {  in __synchronize_srcu()
437 			BUG_ON(sp->batch_done.head != head);  in __synchronize_srcu()
438 			rcu_batch_dequeue(&sp->batch_done);  in __synchronize_srcu()
442 		srcu_reschedule(sp);  in __synchronize_srcu()
444 		rcu_batch_queue(&sp->batch_queue, head);  in __synchronize_srcu()
445 		spin_unlock_irq(&sp->queue_lock);  in __synchronize_srcu()
491 void synchronize_srcu(struct srcu_struct *sp)  in synchronize_srcu()  argument
493 	__synchronize_srcu(sp, rcu_gp_is_expedited()  in synchronize_srcu()
509 void synchronize_srcu_expedited(struct srcu_struct *sp)  in synchronize_srcu_expedited()  argument
511 	__synchronize_srcu(sp, SYNCHRONIZE_SRCU_EXP_TRYCOUNT);  in synchronize_srcu_expedited()
519 void srcu_barrier(struct srcu_struct *sp)  in srcu_barrier()  argument
521 	synchronize_srcu(sp);  in srcu_barrier()
532 unsigned long srcu_batches_completed(struct srcu_struct *sp)  in srcu_batches_completed()  argument
534 	return sp->completed;  in srcu_batches_completed()
545 static void srcu_collect_new(struct srcu_struct *sp)  in srcu_collect_new()  argument
547 	if (!rcu_batch_empty(&sp->batch_queue)) {  in srcu_collect_new()
548 		spin_lock_irq(&sp->queue_lock);  in srcu_collect_new()
549 		rcu_batch_move(&sp->batch_check0, &sp->batch_queue);  in srcu_collect_new()
550 		spin_unlock_irq(&sp->queue_lock);  in srcu_collect_new()
558 static void srcu_advance_batches(struct srcu_struct *sp, int trycount)  in srcu_advance_batches()  argument
560 	int idx = 1 ^ (sp->completed & 1);  in srcu_advance_batches()
570 	if (rcu_batch_empty(&sp->batch_check0) &&  in srcu_advance_batches()
571 	    rcu_batch_empty(&sp->batch_check1))  in srcu_advance_batches()
574 	if (!try_check_zero(sp, idx, trycount))  in srcu_advance_batches()
585 	rcu_batch_move(&sp->batch_done, &sp->batch_check1);  in srcu_advance_batches()
587 	if (rcu_batch_empty(&sp->batch_check0))  in srcu_advance_batches()
589 	srcu_flip(sp);  in srcu_advance_batches()
596 	rcu_batch_move(&sp->batch_check1, &sp->batch_check0);  in srcu_advance_batches()
603 	if (!try_check_zero(sp, idx^1, trycount))  in srcu_advance_batches()
611 	rcu_batch_move(&sp->batch_done, &sp->batch_check1);  in srcu_advance_batches()
619 static void srcu_invoke_callbacks(struct srcu_struct *sp)  in srcu_invoke_callbacks()  argument
625 		head = rcu_batch_dequeue(&sp->batch_done);  in srcu_invoke_callbacks()
638 static void srcu_reschedule(struct srcu_struct *sp)  in srcu_reschedule()  argument
642 	if (rcu_batch_empty(&sp->batch_done) &&  in srcu_reschedule()
643 	    rcu_batch_empty(&sp->batch_check1) &&  in srcu_reschedule()
644 	    rcu_batch_empty(&sp->batch_check0) &&  in srcu_reschedule()
645 	    rcu_batch_empty(&sp->batch_queue)) {  in srcu_reschedule()
646 		spin_lock_irq(&sp->queue_lock);  in srcu_reschedule()
647 		if (rcu_batch_empty(&sp->batch_done) &&  in srcu_reschedule()
648 		    rcu_batch_empty(&sp->batch_check1) &&  in srcu_reschedule()
649 		    rcu_batch_empty(&sp->batch_check0) &&  in srcu_reschedule()
650 		    rcu_batch_empty(&sp->batch_queue)) {  in srcu_reschedule()
651 			sp->running = false;  in srcu_reschedule()
654 		spin_unlock_irq(&sp->queue_lock);  in srcu_reschedule()
659 				   &sp->work, SRCU_INTERVAL);  in srcu_reschedule()
667 	struct srcu_struct *sp;  in process_srcu()  local
669 	sp = container_of(work, struct srcu_struct, work.work);  in process_srcu()
671 	srcu_collect_new(sp);  in process_srcu()
672 	srcu_advance_batches(sp, 1);  in process_srcu()
673 	srcu_invoke_callbacks(sp);  in process_srcu()
674 	srcu_reschedule(sp);  in process_srcu()