linux-master/kernel/padata.c

165  *          run on.  If it's not in the serial cpumask of @pinst
178 	struct padata_instance *pinst = ps->pinst;
188 	if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID)
206 	if ((pinst->flags & PADATA_RESET))
227 		queue_work(pinst->parallel_wq, &pw->pw_work);
286 	struct padata_instance *pinst = pd->ps->pinst;
323 		queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work);
340 		queue_work(pinst->serial_wq, &pd->reorder_work);
424 static int padata_setup_cpumasks(struct padata_instance *pinst)
434 	cpumask_copy(attrs->cpumask, pinst->cpumask.pcpu);
435 	err = apply_workqueue_attrs(pinst->parallel_wq, attrs);
578 	struct padata_instance *pinst = ps->pinst;
600 	cpumask_and(pd->cpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask);
601 	cpumask_and(pd->cpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask);
634 static void __padata_start(struct padata_instance *pinst)
636 	pinst->flags |= PADATA_INIT;
639 static void __padata_stop(struct padata_instance *pinst)
641 	if (!(pinst->flags & PADATA_INIT))
644 	pinst->flags &= ~PADATA_INIT;
664 static int padata_replace(struct padata_instance *pinst)
669 	pinst->flags |= PADATA_RESET;
671 	list_for_each_entry(ps, &pinst->pslist, list) {
679 	list_for_each_entry_continue_reverse(ps, &pinst->pslist, list)
683 	pinst->flags &= ~PADATA_RESET;
689 static bool padata_validate_cpumask(struct padata_instance *pinst,
693 		pinst->flags |= PADATA_INVALID;
697 	pinst->flags &= ~PADATA_INVALID;
701 static int __padata_set_cpumasks(struct padata_instance *pinst,
708 	valid = padata_validate_cpumask(pinst, pcpumask);
710 		__padata_stop(pinst);
714 	valid = padata_validate_cpumask(pinst, cbcpumask);
716 		__padata_stop(pinst);
719 	cpumask_copy(pinst->cpumask.pcpu, pcpumask);
720 	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);
722 	err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst);
725 		__padata_start(pinst);
733  * @pinst: padata instance
740 int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,
747 	mutex_lock(&pinst->lock);
751 		serial_mask = pinst->cpumask.cbcpu;
755 		parallel_mask = pinst->cpumask.pcpu;
762 	err =  __padata_set_cpumasks(pinst, parallel_mask, serial_mask);
765 	mutex_unlock(&pinst->lock);
774 static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
779 		err = padata_replace(pinst);
781 		if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&
782 		    padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
783 			__padata_start(pinst);
789 static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)
794 		if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||
795 		    !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
796 			__padata_stop(pinst);
798 		err = padata_replace(pinst);
804 static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)
806 	return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||
807 		cpumask_test_cpu(cpu, pinst->cpumask.cbcpu);
812 	struct padata_instance *pinst;
815 	pinst = hlist_entry_safe(node, struct padata_instance, cpu_online_node);
816 	if (!pinst_has_cpu(pinst, cpu))
819 	mutex_lock(&pinst->lock);
820 	ret = __padata_add_cpu(pinst, cpu);
821 	mutex_unlock(&pinst->lock);
827 	struct padata_instance *pinst;
830 	pinst = hlist_entry_safe(node, struct padata_instance, cpu_dead_node);
831 	if (!pinst_has_cpu(pinst, cpu))
834 	mutex_lock(&pinst->lock);
835 	ret = __padata_remove_cpu(pinst, cpu);
836 	mutex_unlock(&pinst->lock);
843 static void __padata_free(struct padata_instance *pinst)
847 					    &pinst->cpu_dead_node);
848 	cpuhp_state_remove_instance_nocalls(hp_online, &pinst->cpu_online_node);
851 	WARN_ON(!list_empty(&pinst->pslist));
853 	free_cpumask_var(pinst->cpumask.pcpu);
854 	free_cpumask_var(pinst->cpumask.cbcpu);
855 	destroy_workqueue(pinst->serial_wq);
856 	destroy_workqueue(pinst->parallel_wq);
857 	kfree(pinst);
867 	struct padata_instance *pinst = kobj2pinst(kobj);
868 	__padata_free(pinst);
878 static ssize_t show_cpumask(struct padata_instance *pinst,
884 	mutex_lock(&pinst->lock);
886 		cpumask = pinst->cpumask.cbcpu;
888 		cpumask = pinst->cpumask.pcpu;
892 	mutex_unlock(&pinst->lock);
896 static ssize_t store_cpumask(struct padata_instance *pinst,
914 	ret = padata_set_cpumask(pinst, mask_type, new_cpumask);
948 	struct padata_instance *pinst;
952 	pinst = kobj2pinst(kobj);
955 		ret = pentry->show(pinst, attr, buf);
963 	struct padata_instance *pinst;
967 	pinst = kobj2pinst(kobj);
970 		ret = pentry->store(pinst, attr, buf, count);
994 	struct padata_instance *pinst;
996 	pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL);
997 	if (!pinst)
1000 	pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_UNBOUND, 0,
1002 	if (!pinst->parallel_wq)
1007 	pinst->serial_wq = alloc_workqueue("%s_serial", WQ_MEM_RECLAIM |
1009 	if (!pinst->serial_wq)
1012 	if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL))
1014 	if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) {
1015 		free_cpumask_var(pinst->cpumask.pcpu);
1019 	INIT_LIST_HEAD(&pinst->pslist);
1021 	cpumask_copy(pinst->cpumask.pcpu, cpu_possible_mask);
1022 	cpumask_copy(pinst->cpumask.cbcpu, cpu_possible_mask);
1024 	if (padata_setup_cpumasks(pinst))
1027 	__padata_start(pinst);
1029 	kobject_init(&pinst->kobj, &padata_attr_type);
1030 	mutex_init(&pinst->lock);
1034 						    &pinst->cpu_online_node);
1036 						    &pinst->cpu_dead_node);
1041 	return pinst;
1044 	free_cpumask_var(pinst->cpumask.pcpu);
1045 	free_cpumask_var(pinst->cpumask.cbcpu);
1047 	destroy_workqueue(pinst->serial_wq);
1050 	destroy_workqueue(pinst->parallel_wq);
1052 	kfree(pinst);
1061  * @pinst: padata instance to free
1063 void padata_free(struct padata_instance *pinst)
1065 	kobject_put(&pinst->kobj);
1072  * @pinst: Parent padata_instance object.
1076 struct padata_shell *padata_alloc_shell(struct padata_instance *pinst)
1085 	ps->pinst = pinst;
1094 	mutex_lock(&pinst->lock);
1096 	list_add(&ps->list, &pinst->pslist);
1097 	mutex_unlock(&pinst->lock);
1120 	mutex_lock(&ps->pinst->lock);
1125 	mutex_unlock(&ps->pinst->lock);