167 	struct worker		*manager;	/* L: purely informational */
250 	struct worker		*rescuer;	/* I: rescue worker */
388 #define for_each_pool_worker(worker, pool)				\  argument
389 	list_for_each_entry((worker), &(pool)->workers, node)		\
823 static struct worker *first_idle_worker(struct worker_pool *pool)  in first_idle_worker()
828 	return list_first_entry(&pool->idle_list, struct worker, entry);  in first_idle_worker()
842 	struct worker *worker = first_idle_worker(pool);  in wake_up_worker()  local
844 	if (likely(worker))  in wake_up_worker()
845 		wake_up_process(worker->task);  in wake_up_worker()
861 	struct worker *worker = kthread_data(task);  in wq_worker_waking_up()  local
863 	if (!(worker->flags & WORKER_NOT_RUNNING)) {  in wq_worker_waking_up()
864 		WARN_ON_ONCE(worker->pool->cpu != cpu);  in wq_worker_waking_up()
865 		atomic_inc(&worker->pool->nr_running);  in wq_worker_waking_up()
886 	struct worker *worker = kthread_data(task), *to_wakeup = NULL;  in wq_worker_sleeping()  local
894 	if (worker->flags & WORKER_NOT_RUNNING)  in wq_worker_sleeping()
897 	pool = worker->pool;  in wq_worker_sleeping()
930 static inline void worker_set_flags(struct worker *worker, unsigned int flags)  in worker_set_flags()  argument
932 	struct worker_pool *pool = worker->pool;  in worker_set_flags()
934 	WARN_ON_ONCE(worker->task != current);  in worker_set_flags()
938 	    !(worker->flags & WORKER_NOT_RUNNING)) {  in worker_set_flags()
942 	worker->flags |= flags;  in worker_set_flags()
955 static inline void worker_clr_flags(struct worker *worker, unsigned int flags)  in worker_clr_flags()  argument
957 	struct worker_pool *pool = worker->pool;  in worker_clr_flags()
958 	unsigned int oflags = worker->flags;  in worker_clr_flags()
960 	WARN_ON_ONCE(worker->task != current);  in worker_clr_flags()
962 	worker->flags &= ~flags;  in worker_clr_flags()
970 		if (!(worker->flags & WORKER_NOT_RUNNING))  in worker_clr_flags()
1007 static struct worker *find_worker_executing_work(struct worker_pool *pool,  in find_worker_executing_work()
1010 	struct worker *worker;  in find_worker_executing_work()  local
1012 	hash_for_each_possible(pool->busy_hash, worker, hentry,  in find_worker_executing_work()
1014 		if (worker->current_work == work &&  in find_worker_executing_work()
1015 		    worker->current_func == work->func)  in find_worker_executing_work()
1016 			return worker;  in find_worker_executing_work()
1326 	struct worker *worker;  in is_chained_work()  local
1328 	worker = current_wq_worker();  in is_chained_work()
1333 	return worker && worker->current_pwq->wq == wq;  in is_chained_work()
1376 		struct worker *worker;  in __queue_work()  local
1380 		worker = find_worker_executing_work(last_pool, work);  in __queue_work()
1382 		if (worker && worker->current_pwq->wq == wq) {  in __queue_work()
1383 			pwq = worker->current_pwq;  in __queue_work()
1588 static void worker_enter_idle(struct worker *worker)  in worker_enter_idle()  argument
1590 	struct worker_pool *pool = worker->pool;  in worker_enter_idle()
1592 	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||  in worker_enter_idle()
1593 	    WARN_ON_ONCE(!list_empty(&worker->entry) &&  in worker_enter_idle()
1594 			 (worker->hentry.next || worker->hentry.pprev)))  in worker_enter_idle()
1598 	worker->flags |= WORKER_IDLE;  in worker_enter_idle()
1600 	worker->last_active = jiffies;  in worker_enter_idle()
1603 	list_add(&worker->entry, &pool->idle_list);  in worker_enter_idle()
1628 static void worker_leave_idle(struct worker *worker)  in worker_leave_idle()  argument
1630 	struct worker_pool *pool = worker->pool;  in worker_leave_idle()
1632 	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))  in worker_leave_idle()
1634 	worker_clr_flags(worker, WORKER_IDLE);  in worker_leave_idle()
1636 	list_del_init(&worker->entry);  in worker_leave_idle()
1639 static struct worker *alloc_worker(int node)  in alloc_worker()
1641 	struct worker *worker;  in alloc_worker()  local
1643 	worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);  in alloc_worker()
1644 	if (worker) {  in alloc_worker()
1645 		INIT_LIST_HEAD(&worker->entry);  in alloc_worker()
1646 		INIT_LIST_HEAD(&worker->scheduled);  in alloc_worker()
1647 		INIT_LIST_HEAD(&worker->node);  in alloc_worker()
1649 		worker->flags = WORKER_PREP;  in alloc_worker()
1651 	return worker;  in alloc_worker()
1663 static void worker_attach_to_pool(struct worker *worker,  in worker_attach_to_pool()  argument
1672 	set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);  in worker_attach_to_pool()
1680 		worker->flags |= WORKER_UNBOUND;  in worker_attach_to_pool()
1682 	list_add_tail(&worker->node, &pool->workers);  in worker_attach_to_pool()
1696 static void worker_detach_from_pool(struct worker *worker,  in worker_detach_from_pool()  argument
1702 	list_del(&worker->node);  in worker_detach_from_pool()
1708 	worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);  in worker_detach_from_pool()
1726 static struct worker *create_worker(struct worker_pool *pool)  in create_worker()
1728 	struct worker *worker = NULL;  in create_worker()  local
1737 	worker = alloc_worker(pool->node);  in create_worker()
1738 	if (!worker)  in create_worker()
1741 	worker->pool = pool;  in create_worker()
1742 	worker->id = id;  in create_worker()
1750 	worker->task = kthread_create_on_node(worker_thread, worker, pool->node,  in create_worker()
1752 	if (IS_ERR(worker->task))  in create_worker()
1755 	set_user_nice(worker->task, pool->attrs->nice);  in create_worker()
1756 	kthread_bind_mask(worker->task, pool->attrs->cpumask);  in create_worker()
1759 	worker_attach_to_pool(worker, pool);  in create_worker()
1763 	worker->pool->nr_workers++;  in create_worker()
1764 	worker_enter_idle(worker);  in create_worker()
1765 	wake_up_process(worker->task);  in create_worker()
1768 	return worker;  in create_worker()
1773 	kfree(worker);  in create_worker()
1787 static void destroy_worker(struct worker *worker)  in destroy_worker()  argument
1789 	struct worker_pool *pool = worker->pool;  in destroy_worker()
1794 	if (WARN_ON(worker->current_work) ||  in destroy_worker()
1795 	    WARN_ON(!list_empty(&worker->scheduled)) ||  in destroy_worker()
1796 	    WARN_ON(!(worker->flags & WORKER_IDLE)))  in destroy_worker()
1802 	list_del_init(&worker->entry);  in destroy_worker()
1803 	worker->flags |= WORKER_DIE;  in destroy_worker()
1804 	wake_up_process(worker->task);  in destroy_worker()
1814 		struct worker *worker;  in idle_worker_timeout()  local
1818 		worker = list_entry(pool->idle_list.prev, struct worker, entry);  in idle_worker_timeout()
1819 		expires = worker->last_active + IDLE_WORKER_TIMEOUT;  in idle_worker_timeout()
1826 		destroy_worker(worker);  in idle_worker_timeout()
1951 static bool manage_workers(struct worker *worker)  in manage_workers()  argument
1953 	struct worker_pool *pool = worker->pool;  in manage_workers()
1967 	pool->manager = worker;  in manage_workers()
1990 static void process_one_work(struct worker *worker, struct work_struct *work)  in process_one_work()  argument
1995 	struct worker_pool *pool = worker->pool;  in process_one_work()
1998 	struct worker *collision;  in process_one_work()
2029 	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);  in process_one_work()
2030 	worker->current_work = work;  in process_one_work()
2031 	worker->current_func = work->func;  in process_one_work()
2032 	worker->current_pwq = pwq;  in process_one_work()
2044 		worker_set_flags(worker, WORKER_CPU_INTENSIVE);  in process_one_work()
2069 	worker->current_func(work);  in process_one_work()
2082 		       worker->current_func);  in process_one_work()
2101 		worker_clr_flags(worker, WORKER_CPU_INTENSIVE);  in process_one_work()
2104 	hash_del(&worker->hentry);  in process_one_work()
2105 	worker->current_work = NULL;  in process_one_work()
2106 	worker->current_func = NULL;  in process_one_work()
2107 	worker->current_pwq = NULL;  in process_one_work()
2108 	worker->desc_valid = false;  in process_one_work()
2124 static void process_scheduled_works(struct worker *worker)  in process_scheduled_works()  argument
2126 	while (!list_empty(&worker->scheduled)) {  in process_scheduled_works()
2127 		struct work_struct *work = list_first_entry(&worker->scheduled,  in process_scheduled_works()
2129 		process_one_work(worker, work);  in process_scheduled_works()
2147 	struct worker *worker = __worker;  in worker_thread()  local
2148 	struct worker_pool *pool = worker->pool;  in worker_thread()
2151 	worker->task->flags |= PF_WQ_WORKER;  in worker_thread()
2156 	if (unlikely(worker->flags & WORKER_DIE)) {  in worker_thread()
2158 		WARN_ON_ONCE(!list_empty(&worker->entry));  in worker_thread()
2159 		worker->task->flags &= ~PF_WQ_WORKER;  in worker_thread()
2161 		set_task_comm(worker->task, "kworker/dying");  in worker_thread()
2162 		ida_simple_remove(&pool->worker_ida, worker->id);  in worker_thread()
2163 		worker_detach_from_pool(worker, pool);  in worker_thread()
2164 		kfree(worker);  in worker_thread()
2168 	worker_leave_idle(worker);  in worker_thread()
2175 	if (unlikely(!may_start_working(pool)) && manage_workers(worker))  in worker_thread()
2183 	WARN_ON_ONCE(!list_empty(&worker->scheduled));  in worker_thread()
2192 	worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);  in worker_thread()
2201 			process_one_work(worker, work);  in worker_thread()
2202 			if (unlikely(!list_empty(&worker->scheduled)))  in worker_thread()
2203 				process_scheduled_works(worker);  in worker_thread()
2205 			move_linked_works(work, &worker->scheduled, NULL);  in worker_thread()
2206 			process_scheduled_works(worker);  in worker_thread()
2210 	worker_set_flags(worker, WORKER_PREP);  in worker_thread()
2219 	worker_enter_idle(worker);  in worker_thread()
2249 	struct worker *rescuer = __rescuer;  in rescuer_thread()
2396 			      struct work_struct *target, struct worker *worker)  in insert_wq_barrier()  argument
2416 	if (worker)  in insert_wq_barrier()
2417 		head = worker->scheduled.next;  in insert_wq_barrier()
2713 	struct worker *worker = NULL;  in start_flush_work()  local
2733 		worker = find_worker_executing_work(pool, work);  in start_flush_work()
2734 		if (!worker)  in start_flush_work()
2736 		pwq = worker->current_pwq;  in start_flush_work()
2739 	insert_wq_barrier(pwq, barr, work, worker);  in start_flush_work()
3174 	struct worker *worker;  in put_unbound_pool()  local
3199 	while ((worker = first_idle_worker(pool)))  in put_unbound_pool()
3200 		destroy_worker(worker);  in put_unbound_pool()
3883 		struct worker *rescuer;  in __alloc_workqueue_key()
4051 	struct worker *worker = current_wq_worker();  in current_is_workqueue_rescuer()  local
4053 	return worker && worker->rescue_wq;  in current_is_workqueue_rescuer()
4142 	struct worker *worker = current_wq_worker();  in set_worker_desc()  local
4145 	if (worker) {  in set_worker_desc()
4147 		vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);  in set_worker_desc()
4149 		worker->desc_valid = true;  in set_worker_desc()
4174 	struct worker *worker;  in print_worker_info()  local
4183 	worker = probe_kthread_data(task);  in print_worker_info()
4189 	probe_kernel_read(&fn, &worker->current_func, sizeof(fn));  in print_worker_info()
4190 	probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));  in print_worker_info()
4195 	probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));  in print_worker_info()
4197 		probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);  in print_worker_info()
4233 	struct worker *worker;  in show_pwq()  local
4243 	hash_for_each(pool->busy_hash, bkt, worker, hentry) {  in show_pwq()
4244 		if (worker->current_pwq == pwq) {  in show_pwq()
4253 		hash_for_each(pool->busy_hash, bkt, worker, hentry) {  in show_pwq()
4254 			if (worker->current_pwq != pwq)  in show_pwq()
4258 				task_pid_nr(worker->task),  in show_pwq()
4259 				worker == pwq->wq->rescuer ? "(RESCUER)" : "",  in show_pwq()
4260 				worker->current_func);  in show_pwq()
4261 			list_for_each_entry(work, &worker->scheduled, entry)  in show_pwq()
4341 		struct worker *worker;  in show_workqueue_state()  local
4354 		list_for_each_entry(worker, &pool->idle_list, entry) {  in show_workqueue_state()
4356 				task_pid_nr(worker->task));  in show_workqueue_state()
4386 	struct worker *worker;  in wq_unbind_fn()  local
4399 		for_each_pool_worker(worker, pool)  in wq_unbind_fn()
4400 			worker->flags |= WORKER_UNBOUND;  in wq_unbind_fn()
4444 	struct worker *worker;  in rebind_workers()  local
4455 	for_each_pool_worker(worker, pool)  in rebind_workers()
4456 		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,  in rebind_workers()
4473 	for_each_pool_worker(worker, pool) {  in rebind_workers()
4474 		unsigned int worker_flags = worker->flags;  in rebind_workers()
4485 			wake_up_process(worker->task);  in rebind_workers()
4505 		ACCESS_ONCE(worker->flags) = worker_flags;  in rebind_workers()
4524 	struct worker *worker;  in restore_unbound_workers_cpumask()  local
4538 	for_each_pool_worker(worker, pool)  in restore_unbound_workers_cpumask()
4539 		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,  in restore_unbound_workers_cpumask()