410 #define for_each_pwq(pwq, wq)						\  argument
411 	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node)		\
632 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,  in set_work_pwq()  argument
635 	set_work_data(work, (unsigned long)pwq,  in set_work_pwq()
1074 static void get_pwq(struct pool_workqueue *pwq)  in get_pwq()  argument
1076 	lockdep_assert_held(&pwq->pool->lock);  in get_pwq()
1077 	WARN_ON_ONCE(pwq->refcnt <= 0);  in get_pwq()
1078 	pwq->refcnt++;  in get_pwq()
1088 static void put_pwq(struct pool_workqueue *pwq)  in put_pwq()  argument
1090 	lockdep_assert_held(&pwq->pool->lock);  in put_pwq()
1091 	if (likely(--pwq->refcnt))  in put_pwq()
1093 	if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))  in put_pwq()
1103 	schedule_work(&pwq->unbound_release_work);  in put_pwq()
1112 static void put_pwq_unlocked(struct pool_workqueue *pwq)  in put_pwq_unlocked()  argument
1114 	if (pwq) {  in put_pwq_unlocked()
1119 		spin_lock_irq(&pwq->pool->lock);  in put_pwq_unlocked()
1120 		put_pwq(pwq);  in put_pwq_unlocked()
1121 		spin_unlock_irq(&pwq->pool->lock);  in put_pwq_unlocked()
1127 	struct pool_workqueue *pwq = get_work_pwq(work);  in pwq_activate_delayed_work()  local
1130 	move_linked_works(work, &pwq->pool->worklist, NULL);  in pwq_activate_delayed_work()
1132 	pwq->nr_active++;  in pwq_activate_delayed_work()
1135 static void pwq_activate_first_delayed(struct pool_workqueue *pwq)  in pwq_activate_first_delayed()  argument
1137 	struct work_struct *work = list_first_entry(&pwq->delayed_works,  in pwq_activate_first_delayed()
1154 static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)  in pwq_dec_nr_in_flight()  argument
1160 	pwq->nr_in_flight[color]--;  in pwq_dec_nr_in_flight()
1162 	pwq->nr_active--;  in pwq_dec_nr_in_flight()
1163 	if (!list_empty(&pwq->delayed_works)) {  in pwq_dec_nr_in_flight()
1165 		if (pwq->nr_active < pwq->max_active)  in pwq_dec_nr_in_flight()
1166 			pwq_activate_first_delayed(pwq);  in pwq_dec_nr_in_flight()
1170 	if (likely(pwq->flush_color != color))  in pwq_dec_nr_in_flight()
1174 	if (pwq->nr_in_flight[color])  in pwq_dec_nr_in_flight()
1178 	pwq->flush_color = -1;  in pwq_dec_nr_in_flight()
1184 	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))  in pwq_dec_nr_in_flight()
1185 		complete(&pwq->wq->first_flusher->done);  in pwq_dec_nr_in_flight()
1187 	put_pwq(pwq);  in pwq_dec_nr_in_flight()
1221 	struct pool_workqueue *pwq;  in try_to_grab_pending()  local
1259 	pwq = get_work_pwq(work);  in try_to_grab_pending()
1260 	if (pwq && pwq->pool == pool) {  in try_to_grab_pending()
1274 		pwq_dec_nr_in_flight(pwq, get_work_color(work));  in try_to_grab_pending()
1304 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,  in insert_work()  argument
1307 	struct worker_pool *pool = pwq->pool;  in insert_work()
1310 	set_work_pwq(work, pwq, extra_flags);  in insert_work()
1312 	get_pwq(pwq);  in insert_work()
1344 	struct pool_workqueue *pwq;  in __queue_work()  local
1370 		pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);  in __queue_work()
1372 		pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));  in __queue_work()
1380 	if (last_pool && last_pool != pwq->pool) {  in __queue_work()
1388 			pwq = worker->current_pwq;  in __queue_work()
1392 			spin_lock(&pwq->pool->lock);  in __queue_work()
1395 		spin_lock(&pwq->pool->lock);  in __queue_work()
1406 	if (unlikely(!pwq->refcnt)) {  in __queue_work()
1408 			spin_unlock(&pwq->pool->lock);  in __queue_work()
1418 	trace_workqueue_queue_work(req_cpu, pwq, work);  in __queue_work()
1421 		spin_unlock(&pwq->pool->lock);  in __queue_work()
1425 	pwq->nr_in_flight[pwq->work_color]++;  in __queue_work()
1426 	work_flags = work_color_to_flags(pwq->work_color);  in __queue_work()
1428 	if (likely(pwq->nr_active < pwq->max_active)) {  in __queue_work()
1430 		pwq->nr_active++;  in __queue_work()
1431 		worklist = &pwq->pool->worklist;  in __queue_work()
1434 		worklist = &pwq->delayed_works;  in __queue_work()
1437 	insert_work(pwq, work, worklist, work_flags);  in __queue_work()
1439 	spin_unlock(&pwq->pool->lock);  in __queue_work()
1841 	struct pool_workqueue *pwq = get_work_pwq(work);  in send_mayday()  local
1842 	struct workqueue_struct *wq = pwq->wq;  in send_mayday()
1850 	if (list_empty(&pwq->mayday_node)) {  in send_mayday()
1856 		get_pwq(pwq);  in send_mayday()
1857 		list_add_tail(&pwq->mayday_node, &wq->maydays);  in send_mayday()
2001 	struct pool_workqueue *pwq = get_work_pwq(work);  in process_one_work()  local
2003 	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;  in process_one_work()
2039 	worker->current_pwq = pwq;  in process_one_work()
2073 	lock_map_acquire_read(&pwq->wq->lockdep_map);  in process_one_work()
2083 	lock_map_release(&pwq->wq->lockdep_map);  in process_one_work()
2116 	pwq_dec_nr_in_flight(pwq, work_color);  in process_one_work()
2285 		struct pool_workqueue *pwq = list_first_entry(&wq->maydays,  in rescuer_thread()  local
2287 		struct worker_pool *pool = pwq->pool;  in rescuer_thread()
2291 		list_del_init(&pwq->mayday_node);  in rescuer_thread()
2306 			if (get_work_pwq(work) == pwq)  in rescuer_thread()
2323 				get_pwq(pwq);  in rescuer_thread()
2324 				list_move_tail(&pwq->mayday_node, &wq->maydays);  in rescuer_thread()
2333 		put_pwq(pwq);  in rescuer_thread()
2401 static void insert_wq_barrier(struct pool_workqueue *pwq,  in insert_wq_barrier()  argument
2435 	insert_work(pwq, &barr->work, head,  in insert_wq_barrier()
2474 	struct pool_workqueue *pwq;  in flush_workqueue_prep_pwqs()  local
2481 	for_each_pwq(pwq, wq) {  in flush_workqueue_prep_pwqs()
2482 		struct worker_pool *pool = pwq->pool;  in flush_workqueue_prep_pwqs()
2487 			WARN_ON_ONCE(pwq->flush_color != -1);  in flush_workqueue_prep_pwqs()
2489 			if (pwq->nr_in_flight[flush_color]) {  in flush_workqueue_prep_pwqs()
2490 				pwq->flush_color = flush_color;  in flush_workqueue_prep_pwqs()
2497 			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));  in flush_workqueue_prep_pwqs()
2498 			pwq->work_color = work_color;  in flush_workqueue_prep_pwqs()
2677 	struct pool_workqueue *pwq;  in drain_workqueue()  local
2693 	for_each_pwq(pwq, wq) {  in drain_workqueue()
2696 		spin_lock_irq(&pwq->pool->lock);  in drain_workqueue()
2697 		drained = !pwq->nr_active && list_empty(&pwq->delayed_works);  in drain_workqueue()
2698 		spin_unlock_irq(&pwq->pool->lock);  in drain_workqueue()
2722 	struct pool_workqueue *pwq;  in start_flush_work()  local
2735 	pwq = get_work_pwq(work);  in start_flush_work()
2736 	if (pwq) {  in start_flush_work()
2737 		if (unlikely(pwq->pool != pool))  in start_flush_work()
2743 		pwq = worker->current_pwq;  in start_flush_work()
2746 	insert_wq_barrier(pwq, barr, work, worker);  in start_flush_work()
2755 	if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)  in start_flush_work()
2756 		lock_map_acquire(&pwq->wq->lockdep_map);  in start_flush_work()
2758 		lock_map_acquire_read(&pwq->wq->lockdep_map);  in start_flush_work()
2759 	lock_map_release(&pwq->wq->lockdep_map);  in start_flush_work()
3343 	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,  in pwq_unbound_release_workfn()  local
3345 	struct workqueue_struct *wq = pwq->wq;  in pwq_unbound_release_workfn()
3346 	struct worker_pool *pool = pwq->pool;  in pwq_unbound_release_workfn()
3353 	list_del_rcu(&pwq->pwqs_node);  in pwq_unbound_release_workfn()
3361 	call_rcu_sched(&pwq->rcu, rcu_free_pwq);  in pwq_unbound_release_workfn()
3379 static void pwq_adjust_max_active(struct pool_workqueue *pwq)  in pwq_adjust_max_active()  argument
3381 	struct workqueue_struct *wq = pwq->wq;  in pwq_adjust_max_active()
3388 	if (!freezable && pwq->max_active == wq->saved_max_active)  in pwq_adjust_max_active()
3391 	spin_lock_irq(&pwq->pool->lock);  in pwq_adjust_max_active()
3399 		pwq->max_active = wq->saved_max_active;  in pwq_adjust_max_active()
3401 		while (!list_empty(&pwq->delayed_works) &&  in pwq_adjust_max_active()
3402 		       pwq->nr_active < pwq->max_active)  in pwq_adjust_max_active()
3403 			pwq_activate_first_delayed(pwq);  in pwq_adjust_max_active()
3409 		wake_up_worker(pwq->pool);  in pwq_adjust_max_active()
3411 		pwq->max_active = 0;  in pwq_adjust_max_active()
3414 	spin_unlock_irq(&pwq->pool->lock);  in pwq_adjust_max_active()
3418 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,  in init_pwq()  argument
3421 	BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);  in init_pwq()
3423 	memset(pwq, 0, sizeof(*pwq));  in init_pwq()
3425 	pwq->pool = pool;  in init_pwq()
3426 	pwq->wq = wq;  in init_pwq()
3427 	pwq->flush_color = -1;  in init_pwq()
3428 	pwq->refcnt = 1;  in init_pwq()
3429 	INIT_LIST_HEAD(&pwq->delayed_works);  in init_pwq()
3430 	INIT_LIST_HEAD(&pwq->pwqs_node);  in init_pwq()
3431 	INIT_LIST_HEAD(&pwq->mayday_node);  in init_pwq()
3432 	INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);  in init_pwq()
3436 static void link_pwq(struct pool_workqueue *pwq)  in link_pwq()  argument
3438 	struct workqueue_struct *wq = pwq->wq;  in link_pwq()
3443 	if (!list_empty(&pwq->pwqs_node))  in link_pwq()
3447 	pwq->work_color = wq->work_color;  in link_pwq()
3450 	pwq_adjust_max_active(pwq);  in link_pwq()
3453 	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);  in link_pwq()
3461 	struct pool_workqueue *pwq;  in alloc_unbound_pwq()  local
3469 	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);  in alloc_unbound_pwq()
3470 	if (!pwq) {  in alloc_unbound_pwq()
3475 	init_pwq(pwq, wq, pool);  in alloc_unbound_pwq()
3476 	return pwq;  in alloc_unbound_pwq()
3527 						   struct pool_workqueue *pwq)  in numa_pwq_tbl_install()  argument
3535 	link_pwq(pwq);  in numa_pwq_tbl_install()
3538 	rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);  in numa_pwq_tbl_install()
3731 	struct pool_workqueue *old_pwq = NULL, *pwq;  in wq_update_unbound_numa()  local
3753 	pwq = unbound_pwq_by_node(wq, node);  in wq_update_unbound_numa()
3762 		if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))  in wq_update_unbound_numa()
3771 	pwq = alloc_unbound_pwq(wq, target_attrs);  in wq_update_unbound_numa()
3772 	if (!pwq) {  in wq_update_unbound_numa()
3786 	old_pwq = numa_pwq_tbl_install(wq, node, pwq);  in wq_update_unbound_numa()
3810 			struct pool_workqueue *pwq =  in alloc_and_link_pwqs()  local
3815 			init_pwq(pwq, wq, &cpu_pools[highpri]);  in alloc_and_link_pwqs()
3818 			link_pwq(pwq);  in alloc_and_link_pwqs()
3855 	struct pool_workqueue *pwq;  in __alloc_workqueue_key()  local
3933 	for_each_pwq(pwq, wq)  in __alloc_workqueue_key()
3934 		pwq_adjust_max_active(pwq);  in __alloc_workqueue_key()
3961 	struct pool_workqueue *pwq;  in destroy_workqueue()  local
3969 	for_each_pwq(pwq, wq) {  in destroy_workqueue()
3973 			if (WARN_ON(pwq->nr_in_flight[i])) {  in destroy_workqueue()
3979 		if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) ||  in destroy_workqueue()
3980 		    WARN_ON(pwq->nr_active) ||  in destroy_workqueue()
3981 		    WARN_ON(!list_empty(&pwq->delayed_works))) {  in destroy_workqueue()
4014 			pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);  in destroy_workqueue()
4016 			put_pwq_unlocked(pwq);  in destroy_workqueue()
4023 		pwq = wq->dfl_pwq;  in destroy_workqueue()
4025 		put_pwq_unlocked(pwq);  in destroy_workqueue()
4042 	struct pool_workqueue *pwq;  in workqueue_set_max_active()  local
4054 	for_each_pwq(pwq, wq)  in workqueue_set_max_active()
4055 		pwq_adjust_max_active(pwq);  in workqueue_set_max_active()
4096 	struct pool_workqueue *pwq;  in workqueue_congested()  local
4105 		pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);  in workqueue_congested()
4107 		pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));  in workqueue_congested()
4109 	ret = !list_empty(&pwq->delayed_works);  in workqueue_congested()
4191 	struct pool_workqueue *pwq = NULL;  in print_worker_info()  local
4210 	probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));  in print_worker_info()
4211 	probe_kernel_read(&wq, &pwq->wq, sizeof(wq));  in print_worker_info()
4249 static void show_pwq(struct pool_workqueue *pwq)  in show_pwq()  argument
4251 	struct worker_pool *pool = pwq->pool;  in show_pwq()
4260 	pr_cont(" active=%d/%d%s\n", pwq->nr_active, pwq->max_active,  in show_pwq()
4261 		!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");  in show_pwq()
4264 		if (worker->current_pwq == pwq) {  in show_pwq()
4274 			if (worker->current_pwq != pwq)  in show_pwq()
4279 				worker == pwq->wq->rescuer ? "(RESCUER)" : "",  in show_pwq()
4289 		if (get_work_pwq(work) == pwq) {  in show_pwq()
4299 			if (get_work_pwq(work) != pwq)  in show_pwq()
4308 	if (!list_empty(&pwq->delayed_works)) {  in show_pwq()
4312 		list_for_each_entry(work, &pwq->delayed_works, entry) {  in show_pwq()
4338 		struct pool_workqueue *pwq;  in show_workqueue_state()  local
4341 		for_each_pwq(pwq, wq) {  in show_workqueue_state()
4342 			if (pwq->nr_active || !list_empty(&pwq->delayed_works)) {  in show_workqueue_state()
4352 		for_each_pwq(pwq, wq) {  in show_workqueue_state()
4353 			spin_lock_irqsave(&pwq->pool->lock, flags);  in show_workqueue_state()
4354 			if (pwq->nr_active || !list_empty(&pwq->delayed_works))  in show_workqueue_state()
4355 				show_pwq(pwq);  in show_workqueue_state()
4356 			spin_unlock_irqrestore(&pwq->pool->lock, flags);  in show_workqueue_state()
4698 	struct pool_workqueue *pwq;  in freeze_workqueues_begin()  local
4707 		for_each_pwq(pwq, wq)  in freeze_workqueues_begin()
4708 			pwq_adjust_max_active(pwq);  in freeze_workqueues_begin()
4732 	struct pool_workqueue *pwq;  in freeze_workqueues_busy()  local
4746 		for_each_pwq(pwq, wq) {  in freeze_workqueues_busy()
4747 			WARN_ON_ONCE(pwq->nr_active < 0);  in freeze_workqueues_busy()
4748 			if (pwq->nr_active) {  in freeze_workqueues_busy()
4773 	struct pool_workqueue *pwq;  in thaw_workqueues()  local
4785 		for_each_pwq(pwq, wq)  in thaw_workqueues()
4786 			pwq_adjust_max_active(pwq);  in thaw_workqueues()