99 static int init_srcu_struct_fields(struct srcu_struct *sp)  in init_srcu_struct_fields()  argument
101 	sp->completed = 0;  in init_srcu_struct_fields()
102 	spin_lock_init(&sp->queue_lock);  in init_srcu_struct_fields()
103 	sp->running = false;  in init_srcu_struct_fields()
104 	rcu_batch_init(&sp->batch_queue);  in init_srcu_struct_fields()
105 	rcu_batch_init(&sp->batch_check0);  in init_srcu_struct_fields()
106 	rcu_batch_init(&sp->batch_check1);  in init_srcu_struct_fields()
107 	rcu_batch_init(&sp->batch_done);  in init_srcu_struct_fields()
108 	INIT_DELAYED_WORK(&sp->work, process_srcu);  in init_srcu_struct_fields()
109 	sp->per_cpu_ref = alloc_percpu(struct srcu_struct_array);  in init_srcu_struct_fields()
110 	return sp->per_cpu_ref ? 0 : -ENOMEM;  in init_srcu_struct_fields()
115 int __init_srcu_struct(struct srcu_struct *sp, const char *name,  in __init_srcu_struct()  argument
119 	debug_check_no_locks_freed((void *)sp, sizeof(*sp));  in __init_srcu_struct()
120 	lockdep_init_map(&sp->dep_map, name, key, 0);  in __init_srcu_struct()
121 	return init_srcu_struct_fields(sp);  in __init_srcu_struct()
135 int init_srcu_struct(struct srcu_struct *sp)  in init_srcu_struct()  argument
137 	return init_srcu_struct_fields(sp);  in init_srcu_struct()
147 static unsigned long srcu_readers_seq_idx(struct srcu_struct *sp, int idx)  in srcu_readers_seq_idx()  argument
154 		t = READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->seq[idx]);  in srcu_readers_seq_idx()
164 static unsigned long srcu_readers_active_idx(struct srcu_struct *sp, int idx)  in srcu_readers_active_idx()  argument
171 		t = READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[idx]);  in srcu_readers_active_idx()
186 static bool srcu_readers_active_idx_check(struct srcu_struct *sp, int idx)  in srcu_readers_active_idx_check()  argument
190 	seq = srcu_readers_seq_idx(sp, idx);  in srcu_readers_active_idx_check()
223 	if (srcu_readers_active_idx(sp, idx) != 0)  in srcu_readers_active_idx_check()
251 	return srcu_readers_seq_idx(sp, idx) == seq;  in srcu_readers_active_idx_check()
263 static bool srcu_readers_active(struct srcu_struct *sp)  in srcu_readers_active()  argument
269 		sum += READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[0]);  in srcu_readers_active()
270 		sum += READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[1]);  in srcu_readers_active()
282 void cleanup_srcu_struct(struct srcu_struct *sp)  in cleanup_srcu_struct()  argument
284 	if (WARN_ON(srcu_readers_active(sp)))  in cleanup_srcu_struct()
286 	free_percpu(sp->per_cpu_ref);  in cleanup_srcu_struct()
287 	sp->per_cpu_ref = NULL;  in cleanup_srcu_struct()
296 int __srcu_read_lock(struct srcu_struct *sp)  in __srcu_read_lock()  argument
300 	idx = READ_ONCE(sp->completed) & 0x1;  in __srcu_read_lock()
301 	__this_cpu_inc(sp->per_cpu_ref->c[idx]);  in __srcu_read_lock()
303 	__this_cpu_inc(sp->per_cpu_ref->seq[idx]);  in __srcu_read_lock()
314 void __srcu_read_unlock(struct srcu_struct *sp, int idx)  in __srcu_read_unlock()  argument
317 	this_cpu_dec(sp->per_cpu_ref->c[idx]);  in __srcu_read_unlock()
339 static bool try_check_zero(struct srcu_struct *sp, int idx, int trycount)  in try_check_zero()  argument
342 		if (srcu_readers_active_idx_check(sp, idx))  in try_check_zero()
355 static void srcu_flip(struct srcu_struct *sp)  in srcu_flip()  argument
357 	sp->completed++;  in srcu_flip()
387 void call_srcu(struct srcu_struct *sp, struct rcu_head *head,  in call_srcu()  argument
394 	spin_lock_irqsave(&sp->queue_lock, flags);  in call_srcu()
395 	rcu_batch_queue(&sp->batch_queue, head);  in call_srcu()
396 	if (!sp->running) {  in call_srcu()
397 		sp->running = true;  in call_srcu()
398 		queue_delayed_work(system_power_efficient_wq, &sp->work, 0);  in call_srcu()
400 	spin_unlock_irqrestore(&sp->queue_lock, flags);  in call_srcu()
404 static void srcu_advance_batches(struct srcu_struct *sp, int trycount);
405 static void srcu_reschedule(struct srcu_struct *sp);
410 static void __synchronize_srcu(struct srcu_struct *sp, int trycount)  in __synchronize_srcu()  argument
416 	RCU_LOCKDEP_WARN(lock_is_held(&sp->dep_map) ||  in __synchronize_srcu()
427 	spin_lock_irq(&sp->queue_lock);  in __synchronize_srcu()
428 	if (!sp->running) {  in __synchronize_srcu()
430 		sp->running = true;  in __synchronize_srcu()
431 		rcu_batch_queue(&sp->batch_check0, head);  in __synchronize_srcu()
432 		spin_unlock_irq(&sp->queue_lock);  in __synchronize_srcu()
434 		srcu_advance_batches(sp, trycount);  in __synchronize_srcu()
435 		if (!rcu_batch_empty(&sp->batch_done)) {  in __synchronize_srcu()
436 			BUG_ON(sp->batch_done.head != head);  in __synchronize_srcu()
437 			rcu_batch_dequeue(&sp->batch_done);  in __synchronize_srcu()
441 		srcu_reschedule(sp);  in __synchronize_srcu()
443 		rcu_batch_queue(&sp->batch_queue, head);  in __synchronize_srcu()
444 		spin_unlock_irq(&sp->queue_lock);  in __synchronize_srcu()
490 void synchronize_srcu(struct srcu_struct *sp)  in synchronize_srcu()  argument
492 	__synchronize_srcu(sp, rcu_gp_is_expedited()  in synchronize_srcu()
508 void synchronize_srcu_expedited(struct srcu_struct *sp)  in synchronize_srcu_expedited()  argument
510 	__synchronize_srcu(sp, SYNCHRONIZE_SRCU_EXP_TRYCOUNT);  in synchronize_srcu_expedited()
518 void srcu_barrier(struct srcu_struct *sp)  in srcu_barrier()  argument
520 	synchronize_srcu(sp);  in srcu_barrier()
531 unsigned long srcu_batches_completed(struct srcu_struct *sp)  in srcu_batches_completed()  argument
533 	return sp->completed;  in srcu_batches_completed()
544 static void srcu_collect_new(struct srcu_struct *sp)  in srcu_collect_new()  argument
546 	if (!rcu_batch_empty(&sp->batch_queue)) {  in srcu_collect_new()
547 		spin_lock_irq(&sp->queue_lock);  in srcu_collect_new()
548 		rcu_batch_move(&sp->batch_check0, &sp->batch_queue);  in srcu_collect_new()
549 		spin_unlock_irq(&sp->queue_lock);  in srcu_collect_new()
557 static void srcu_advance_batches(struct srcu_struct *sp, int trycount)  in srcu_advance_batches()  argument
559 	int idx = 1 ^ (sp->completed & 1);  in srcu_advance_batches()
569 	if (rcu_batch_empty(&sp->batch_check0) &&  in srcu_advance_batches()
570 	    rcu_batch_empty(&sp->batch_check1))  in srcu_advance_batches()
573 	if (!try_check_zero(sp, idx, trycount))  in srcu_advance_batches()
584 	rcu_batch_move(&sp->batch_done, &sp->batch_check1);  in srcu_advance_batches()
586 	if (rcu_batch_empty(&sp->batch_check0))  in srcu_advance_batches()
588 	srcu_flip(sp);  in srcu_advance_batches()
595 	rcu_batch_move(&sp->batch_check1, &sp->batch_check0);  in srcu_advance_batches()
602 	if (!try_check_zero(sp, idx^1, trycount))  in srcu_advance_batches()
610 	rcu_batch_move(&sp->batch_done, &sp->batch_check1);  in srcu_advance_batches()
618 static void srcu_invoke_callbacks(struct srcu_struct *sp)  in srcu_invoke_callbacks()  argument
624 		head = rcu_batch_dequeue(&sp->batch_done);  in srcu_invoke_callbacks()
637 static void srcu_reschedule(struct srcu_struct *sp)  in srcu_reschedule()  argument
641 	if (rcu_batch_empty(&sp->batch_done) &&  in srcu_reschedule()
642 	    rcu_batch_empty(&sp->batch_check1) &&  in srcu_reschedule()
643 	    rcu_batch_empty(&sp->batch_check0) &&  in srcu_reschedule()
644 	    rcu_batch_empty(&sp->batch_queue)) {  in srcu_reschedule()
645 		spin_lock_irq(&sp->queue_lock);  in srcu_reschedule()
646 		if (rcu_batch_empty(&sp->batch_done) &&  in srcu_reschedule()
647 		    rcu_batch_empty(&sp->batch_check1) &&  in srcu_reschedule()
648 		    rcu_batch_empty(&sp->batch_check0) &&  in srcu_reschedule()
649 		    rcu_batch_empty(&sp->batch_queue)) {  in srcu_reschedule()
650 			sp->running = false;  in srcu_reschedule()
653 		spin_unlock_irq(&sp->queue_lock);  in srcu_reschedule()
658 				   &sp->work, SRCU_INTERVAL);  in srcu_reschedule()
666 	struct srcu_struct *sp;  in process_srcu()  local
668 	sp = container_of(work, struct srcu_struct, work.work);  in process_srcu()
670 	srcu_collect_new(sp);  in process_srcu()
671 	srcu_advance_batches(sp, 1);  in process_srcu()
672 	srcu_invoke_callbacks(sp);  in process_srcu()
673 	srcu_reschedule(sp);  in process_srcu()