185 static void wake(struct dm_kcopyd_client *kc)  in wake()  argument
187 	queue_work(kc->kcopyd_wq, &kc->kcopyd_work);  in wake()
220 static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl)  in kcopyd_put_pages()  argument
227 		if (kc->nr_free_pages >= kc->nr_reserved_pages)  in kcopyd_put_pages()
230 			pl->next = kc->pages;  in kcopyd_put_pages()
231 			kc->pages = pl;  in kcopyd_put_pages()
232 			kc->nr_free_pages++;  in kcopyd_put_pages()
239 static int kcopyd_get_pages(struct dm_kcopyd_client *kc,  in kcopyd_get_pages()  argument
250 			pl = kc->pages;  in kcopyd_get_pages()
253 			kc->pages = pl->next;  in kcopyd_get_pages()
254 			kc->nr_free_pages--;  in kcopyd_get_pages()
264 		kcopyd_put_pages(kc, *pages);  in kcopyd_get_pages()
285 static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages)  in client_reserve_pages()  argument
301 	kc->nr_reserved_pages += nr_pages;  in client_reserve_pages()
302 	kcopyd_put_pages(kc, pl);  in client_reserve_pages()
307 static void client_free_pages(struct dm_kcopyd_client *kc)  in client_free_pages()  argument
309 	BUG_ON(kc->nr_free_pages != kc->nr_reserved_pages);  in client_free_pages()
310 	drop_pages(kc->pages);  in client_free_pages()
311 	kc->pages = NULL;  in client_free_pages()
312 	kc->nr_free_pages = kc->nr_reserved_pages = 0;  in client_free_pages()
321 	struct dm_kcopyd_client *kc;  member
390 			      struct dm_kcopyd_client *kc)  in pop()  argument
395 	spin_lock_irqsave(&kc->job_lock, flags);  in pop()
401 	spin_unlock_irqrestore(&kc->job_lock, flags);  in pop()
409 	struct dm_kcopyd_client *kc = job->kc;  in push()  local
411 	spin_lock_irqsave(&kc->job_lock, flags);  in push()
413 	spin_unlock_irqrestore(&kc->job_lock, flags);  in push()
420 	struct dm_kcopyd_client *kc = job->kc;  in push_head()  local
422 	spin_lock_irqsave(&kc->job_lock, flags);  in push_head()
424 	spin_unlock_irqrestore(&kc->job_lock, flags);  in push_head()
442 	struct dm_kcopyd_client *kc = job->kc;  in run_complete_job()  local
445 		kcopyd_put_pages(kc, job->pages);  in run_complete_job()
451 		mempool_free(job, kc->job_pool);  in run_complete_job()
454 	if (atomic_dec_and_test(&kc->nr_jobs))  in run_complete_job()
455 		wake_up(&kc->destroyq);  in run_complete_job()
463 	struct dm_kcopyd_client *kc = job->kc;  in complete_io()  local
465 	io_job_finish(kc->throttle);  in complete_io()
474 			push(&kc->complete_jobs, job);  in complete_io()
475 			wake(kc);  in complete_io()
481 		push(&kc->complete_jobs, job);  in complete_io()
485 		push(&kc->io_jobs, job);  in complete_io()
488 	wake(kc);  in complete_io()
505 		.client = job->kc->io_client,  in run_io_job()
508 	io_job_start(job->kc->throttle);  in run_io_job()
523 	r = kcopyd_get_pages(job->kc, nr_pages, &job->pages);  in run_pages_job()
526 		push(&job->kc->io_jobs, job);  in run_pages_job()
541 static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc,  in process_jobs()  argument
547 	while ((job = pop(jobs, kc))) {  in process_jobs()
557 			push(&kc->complete_jobs, job);  in process_jobs()
581 	struct dm_kcopyd_client *kc = container_of(work,  in do_work()  local
593 	process_jobs(&kc->complete_jobs, kc, run_complete_job);  in do_work()
594 	process_jobs(&kc->pages_jobs, kc, run_pages_job);  in do_work()
595 	process_jobs(&kc->io_jobs, kc, run_io_job);  in do_work()
606 	struct dm_kcopyd_client *kc = job->kc;  in dispatch_job()  local
607 	atomic_inc(&kc->nr_jobs);  in dispatch_job()
609 		push(&kc->complete_jobs, job);  in dispatch_job()
611 		push(&kc->io_jobs, job);  in dispatch_job()
613 		push(&kc->pages_jobs, job);  in dispatch_job()
614 	wake(kc);  in dispatch_job()
625 	struct dm_kcopyd_client *kc = job->kc;  in segment_complete()  local
680 		push(&kc->complete_jobs, job);  in segment_complete()
681 		wake(kc);  in segment_complete()
692 	atomic_inc(&master_job->kc->nr_jobs);  in split_job()
701 int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,  in dm_kcopyd_copy()  argument
712 	job = mempool_alloc(kc->job_pool, GFP_NOIO);  in dm_kcopyd_copy()
717 	job->kc = kc;  in dm_kcopyd_copy()
761 int dm_kcopyd_zero(struct dm_kcopyd_client *kc,  in dm_kcopyd_zero()  argument
765 	return dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context);  in dm_kcopyd_zero()
769 void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc,  in dm_kcopyd_prepare_callback()  argument
774 	job = mempool_alloc(kc->job_pool, GFP_NOIO);  in dm_kcopyd_prepare_callback()
777 	job->kc = kc;  in dm_kcopyd_prepare_callback()
782 	atomic_inc(&kc->nr_jobs);  in dm_kcopyd_prepare_callback()
791 	struct dm_kcopyd_client *kc = job->kc;  in dm_kcopyd_do_callback()  local
796 	push(&kc->complete_jobs, job);  in dm_kcopyd_do_callback()
797 	wake(kc);  in dm_kcopyd_do_callback()
819 	struct dm_kcopyd_client *kc;  in dm_kcopyd_client_create()  local
821 	kc = kmalloc(sizeof(*kc), GFP_KERNEL);  in dm_kcopyd_client_create()
822 	if (!kc)  in dm_kcopyd_client_create()
825 	spin_lock_init(&kc->job_lock);  in dm_kcopyd_client_create()
826 	INIT_LIST_HEAD(&kc->complete_jobs);  in dm_kcopyd_client_create()
827 	INIT_LIST_HEAD(&kc->io_jobs);  in dm_kcopyd_client_create()
828 	INIT_LIST_HEAD(&kc->pages_jobs);  in dm_kcopyd_client_create()
829 	kc->throttle = throttle;  in dm_kcopyd_client_create()
831 	kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache);  in dm_kcopyd_client_create()
832 	if (!kc->job_pool)  in dm_kcopyd_client_create()
835 	INIT_WORK(&kc->kcopyd_work, do_work);  in dm_kcopyd_client_create()
836 	kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM, 0);  in dm_kcopyd_client_create()
837 	if (!kc->kcopyd_wq)  in dm_kcopyd_client_create()
840 	kc->pages = NULL;  in dm_kcopyd_client_create()
841 	kc->nr_reserved_pages = kc->nr_free_pages = 0;  in dm_kcopyd_client_create()
842 	r = client_reserve_pages(kc, RESERVE_PAGES);  in dm_kcopyd_client_create()
846 	kc->io_client = dm_io_client_create();  in dm_kcopyd_client_create()
847 	if (IS_ERR(kc->io_client)) {  in dm_kcopyd_client_create()
848 		r = PTR_ERR(kc->io_client);  in dm_kcopyd_client_create()
852 	init_waitqueue_head(&kc->destroyq);  in dm_kcopyd_client_create()
853 	atomic_set(&kc->nr_jobs, 0);  in dm_kcopyd_client_create()
855 	return kc;  in dm_kcopyd_client_create()
858 	client_free_pages(kc);  in dm_kcopyd_client_create()
860 	destroy_workqueue(kc->kcopyd_wq);  in dm_kcopyd_client_create()
862 	mempool_destroy(kc->job_pool);  in dm_kcopyd_client_create()
864 	kfree(kc);  in dm_kcopyd_client_create()
870 void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc)  in dm_kcopyd_client_destroy()  argument
873 	wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs));  in dm_kcopyd_client_destroy()
875 	BUG_ON(!list_empty(&kc->complete_jobs));  in dm_kcopyd_client_destroy()
876 	BUG_ON(!list_empty(&kc->io_jobs));  in dm_kcopyd_client_destroy()
877 	BUG_ON(!list_empty(&kc->pages_jobs));  in dm_kcopyd_client_destroy()
878 	destroy_workqueue(kc->kcopyd_wq);  in dm_kcopyd_client_destroy()
879 	dm_io_client_destroy(kc->io_client);  in dm_kcopyd_client_destroy()
880 	client_free_pages(kc);  in dm_kcopyd_client_destroy()
881 	mempool_destroy(kc->job_pool);  in dm_kcopyd_client_destroy()
882 	kfree(kc);  in dm_kcopyd_client_destroy()