| H A D | padata.c | 67 struct padata_instance *pinst; padata_parallel_worker() local
74 pinst = pd->pinst; padata_parallel_worker()
97 * @pinst: padata instance
106 int padata_do_parallel(struct padata_instance *pinst, padata_do_parallel() argument
115 pd = rcu_dereference_bh(pinst->pd); padata_do_parallel()
118 if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID) padata_do_parallel()
125 if ((pinst->flags & PADATA_RESET)) padata_do_parallel()
143 queue_work_on(target_cpu, pinst->wq, &queue->work); padata_do_parallel()
221 struct padata_instance *pinst = pd->pinst; padata_reorder() local
266 queue_work_on(cb_cpu, pinst->wq, &squeue->work); padata_reorder()
277 && !(pinst->flags & PADATA_RESET)) padata_reorder()
408 static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst, padata_alloc_pd() argument
434 pd->pinst = pinst; padata_alloc_pd()
483 static void __padata_start(struct padata_instance *pinst) __padata_start() argument
485 pinst->flags |= PADATA_INIT; __padata_start()
488 static void __padata_stop(struct padata_instance *pinst) __padata_stop() argument
490 if (!(pinst->flags & PADATA_INIT)) __padata_stop()
493 pinst->flags &= ~PADATA_INIT; __padata_stop()
498 padata_flush_queues(pinst->pd); __padata_stop()
503 static void padata_replace(struct padata_instance *pinst, padata_replace() argument
506 struct parallel_data *pd_old = pinst->pd; padata_replace()
509 pinst->flags |= PADATA_RESET; padata_replace()
511 rcu_assign_pointer(pinst->pd, pd_new); padata_replace()
524 blocking_notifier_call_chain(&pinst->cpumask_change_notifier, padata_replace()
528 pinst->flags &= ~PADATA_RESET; padata_replace()
535 * @pinst: A poineter to padata instance
538 int padata_register_cpumask_notifier(struct padata_instance *pinst, padata_register_cpumask_notifier() argument
541 return blocking_notifier_chain_register(&pinst->cpumask_change_notifier, padata_register_cpumask_notifier()
550 * @pinst: A pointer to data instance.
553 int padata_unregister_cpumask_notifier(struct padata_instance *pinst, padata_unregister_cpumask_notifier() argument
557 &pinst->cpumask_change_notifier, padata_unregister_cpumask_notifier()
564 static bool padata_validate_cpumask(struct padata_instance *pinst, padata_validate_cpumask() argument
568 pinst->flags |= PADATA_INVALID; padata_validate_cpumask()
572 pinst->flags &= ~PADATA_INVALID; padata_validate_cpumask()
576 static int __padata_set_cpumasks(struct padata_instance *pinst, __padata_set_cpumasks() argument
583 valid = padata_validate_cpumask(pinst, pcpumask); __padata_set_cpumasks()
585 __padata_stop(pinst); __padata_set_cpumasks()
589 valid = padata_validate_cpumask(pinst, cbcpumask); __padata_set_cpumasks()
591 __padata_stop(pinst); __padata_set_cpumasks()
594 pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); __padata_set_cpumasks()
598 cpumask_copy(pinst->cpumask.pcpu, pcpumask); __padata_set_cpumasks()
599 cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); __padata_set_cpumasks()
601 padata_replace(pinst, pd); __padata_set_cpumasks()
604 __padata_start(pinst); __padata_set_cpumasks()
614 * @pinst: padata instance
618 int padata_set_cpumasks(struct padata_instance *pinst, cpumask_var_t pcpumask, padata_set_cpumasks() argument
623 mutex_lock(&pinst->lock); padata_set_cpumasks()
626 err = __padata_set_cpumasks(pinst, pcpumask, cbcpumask); padata_set_cpumasks()
629 mutex_unlock(&pinst->lock); padata_set_cpumasks()
640 * @pinst: padata instance
645 int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, padata_set_cpumask() argument
651 mutex_lock(&pinst->lock); padata_set_cpumask()
656 serial_mask = pinst->cpumask.cbcpu; padata_set_cpumask()
660 parallel_mask = pinst->cpumask.pcpu; padata_set_cpumask()
667 err = __padata_set_cpumasks(pinst, parallel_mask, serial_mask); padata_set_cpumask()
671 mutex_unlock(&pinst->lock); padata_set_cpumask()
677 static int __padata_add_cpu(struct padata_instance *pinst, int cpu) __padata_add_cpu() argument
682 pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu, __padata_add_cpu()
683 pinst->cpumask.cbcpu); __padata_add_cpu()
687 padata_replace(pinst, pd); __padata_add_cpu()
689 if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) && __padata_add_cpu()
690 padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) __padata_add_cpu()
691 __padata_start(pinst); __padata_add_cpu()
701 * @pinst: padata instance
709 int padata_add_cpu(struct padata_instance *pinst, int cpu, int mask) padata_add_cpu() argument
716 mutex_lock(&pinst->lock); padata_add_cpu()
720 cpumask_set_cpu(cpu, pinst->cpumask.cbcpu); padata_add_cpu()
722 cpumask_set_cpu(cpu, pinst->cpumask.pcpu); padata_add_cpu()
724 err = __padata_add_cpu(pinst, cpu); padata_add_cpu()
727 mutex_unlock(&pinst->lock); padata_add_cpu()
733 static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) __padata_remove_cpu() argument
739 if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) || __padata_remove_cpu()
740 !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) __padata_remove_cpu()
741 __padata_stop(pinst); __padata_remove_cpu()
743 pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu, __padata_remove_cpu()
744 pinst->cpumask.cbcpu); __padata_remove_cpu()
748 padata_replace(pinst, pd); __padata_remove_cpu()
761 * @pinst: padata instance
768 int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask) padata_remove_cpu() argument
775 mutex_lock(&pinst->lock); padata_remove_cpu()
779 cpumask_clear_cpu(cpu, pinst->cpumask.cbcpu); padata_remove_cpu()
781 cpumask_clear_cpu(cpu, pinst->cpumask.pcpu); padata_remove_cpu()
783 err = __padata_remove_cpu(pinst, cpu); padata_remove_cpu()
786 mutex_unlock(&pinst->lock); padata_remove_cpu()
795 * @pinst: padata instance to start
797 int padata_start(struct padata_instance *pinst) padata_start() argument
801 mutex_lock(&pinst->lock); padata_start()
803 if (pinst->flags & PADATA_INVALID) padata_start()
806 __padata_start(pinst); padata_start()
808 mutex_unlock(&pinst->lock); padata_start()
817 * @pinst: padata instance to stop
819 void padata_stop(struct padata_instance *pinst) padata_stop() argument
821 mutex_lock(&pinst->lock); padata_stop()
822 __padata_stop(pinst); padata_stop()
823 mutex_unlock(&pinst->lock); padata_stop()
829 static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu) pinst_has_cpu() argument
831 return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) || pinst_has_cpu()
832 cpumask_test_cpu(cpu, pinst->cpumask.cbcpu); pinst_has_cpu()
840 struct padata_instance *pinst; padata_cpu_callback() local
843 pinst = container_of(nfb, struct padata_instance, cpu_notifier); padata_cpu_callback()
850 if (!pinst_has_cpu(pinst, cpu)) padata_cpu_callback()
852 mutex_lock(&pinst->lock); padata_cpu_callback()
853 err = __padata_add_cpu(pinst, cpu); padata_cpu_callback()
854 mutex_unlock(&pinst->lock); padata_cpu_callback()
863 if (!pinst_has_cpu(pinst, cpu)) padata_cpu_callback()
865 mutex_lock(&pinst->lock); padata_cpu_callback()
866 err = __padata_remove_cpu(pinst, cpu); padata_cpu_callback()
867 mutex_unlock(&pinst->lock); padata_cpu_callback()
877 static void __padata_free(struct padata_instance *pinst) __padata_free() argument
880 unregister_hotcpu_notifier(&pinst->cpu_notifier); __padata_free()
883 padata_stop(pinst); __padata_free()
884 padata_free_pd(pinst->pd); __padata_free()
885 free_cpumask_var(pinst->cpumask.pcpu); __padata_free()
886 free_cpumask_var(pinst->cpumask.cbcpu); __padata_free()
887 kfree(pinst); __padata_free()
897 struct padata_instance *pinst = kobj2pinst(kobj); padata_sysfs_release() local
898 __padata_free(pinst); padata_sysfs_release()
908 static ssize_t show_cpumask(struct padata_instance *pinst, show_cpumask() argument
914 mutex_lock(&pinst->lock); show_cpumask()
916 cpumask = pinst->cpumask.cbcpu; show_cpumask()
918 cpumask = pinst->cpumask.pcpu; show_cpumask()
922 mutex_unlock(&pinst->lock); show_cpumask()
926 static ssize_t store_cpumask(struct padata_instance *pinst, store_cpumask() argument
944 ret = padata_set_cpumask(pinst, mask_type, new_cpumask); store_cpumask()
977 struct padata_instance *pinst; padata_sysfs_show() local
981 pinst = kobj2pinst(kobj); padata_sysfs_show()
984 ret = pentry->show(pinst, attr, buf); padata_sysfs_show()
992 struct padata_instance *pinst; padata_sysfs_store() local
996 pinst = kobj2pinst(kobj); padata_sysfs_store()
999 ret = pentry->store(pinst, attr, buf, count); padata_sysfs_store()
1040 struct padata_instance *pinst; padata_alloc() local
1043 pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL); padata_alloc()
1044 if (!pinst) padata_alloc()
1048 if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL)) padata_alloc()
1050 if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) { padata_alloc()
1051 free_cpumask_var(pinst->cpumask.pcpu); padata_alloc()
1054 if (!padata_validate_cpumask(pinst, pcpumask) || padata_alloc()
1055 !padata_validate_cpumask(pinst, cbcpumask)) padata_alloc()
1058 pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); padata_alloc()
1062 rcu_assign_pointer(pinst->pd, pd); padata_alloc()
1064 pinst->wq = wq; padata_alloc()
1066 cpumask_copy(pinst->cpumask.pcpu, pcpumask); padata_alloc()
1067 cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); padata_alloc()
1069 pinst->flags = 0; padata_alloc()
1073 BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier); padata_alloc()
1074 kobject_init(&pinst->kobj, &padata_attr_type); padata_alloc()
1075 mutex_init(&pinst->lock); padata_alloc()
1078 pinst->cpu_notifier.notifier_call = padata_cpu_callback; padata_alloc()
1079 pinst->cpu_notifier.priority = 0; padata_alloc()
1080 register_hotcpu_notifier(&pinst->cpu_notifier); padata_alloc()
1083 return pinst; padata_alloc()
1086 free_cpumask_var(pinst->cpumask.pcpu); padata_alloc()
1087 free_cpumask_var(pinst->cpumask.cbcpu); padata_alloc()
1089 kfree(pinst); padata_alloc()
1101 void padata_free(struct padata_instance *pinst) padata_free() argument
1103 kobject_put(&pinst->kobj); padata_free()
|