kernel/sched/fair.c

1534 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv)
1536 	return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv;
1539 static inline unsigned long task_faults(struct task_struct *p, int nid)
1544 	return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1545 		p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)];
1548 static inline unsigned long group_faults(struct task_struct *p, int nid)
1555 	return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1556 		ng->faults[task_faults_idx(NUMA_MEM, nid, 1)];
1559 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid)
1561 	return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] +
1562 		group->faults[task_faults_idx(NUMA_CPU, nid, 1)];
1596 static bool numa_is_active_node(int nid, struct numa_group *ng)
1598 	return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu;
1602 static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
1623 		int dist = node_distance(nid, node);
1627 		 * for placement; nid was already counted.
1629 		if (dist >= max_dist || node == nid)
1673 static inline unsigned long task_weight(struct task_struct *p, int nid,
1686 	faults = task_faults(p, nid);
1687 	faults += score_nearby_nodes(p, nid, dist, true);
1692 static inline unsigned long group_weight(struct task_struct *p, int nid,
1706 	faults = group_faults(p, nid);
1707 	faults += score_nearby_nodes(p, nid, dist, false);
2036 			      struct numa_stats *ns, int nid,
2045 	for_each_cpu(cpu, cpumask_of_node(nid)) {
2067 	ns->weight = cpumask_weight(cpumask_of_node(nid));
2428 	int nid, ret, dist;
2466 	/* Try to find a spot on the preferred nid. */
2478 		for_each_node_state(nid, N_CPU) {
2479 			if (nid == env.src_nid || nid == p->numa_preferred_nid)
2490 			taskimp = task_weight(p, nid, dist) - taskweight;
2491 			groupimp = group_weight(p, nid, dist) - groupweight;
2496 			env.dst_nid = nid;
2512 			nid = env.src_nid;
2514 			nid = cpu_to_node(env.best_cpu);
2516 		if (nid != p->numa_preferred_nid)
2517 			sched_setnuma(p, nid);
2574 	int nid, active_nodes = 0;
2576 	for_each_node_state(nid, N_CPU) {
2577 		faults = group_faults_cpu(numa_group, nid);
2582 	for_each_node_state(nid, N_CPU) {
2583 		faults = group_faults_cpu(numa_group, nid);
2712  * Determine the preferred nid for a task in a numa_group. This needs to
2716 static int preferred_group_nid(struct task_struct *p, int nid)
2723 		return nid;
2732 		int node, max_node = nid;
2747 	 * Finding the preferred nid in a system with NUMA backplane
2786 				 * winner is the preferred nid.
2788 				nid = a;
2796 	return nid;
2801 	int seq, nid, max_nid = NUMA_NO_NODE;
2832 	for_each_online_node(nid) {
2841 			mem_idx = task_faults_idx(NUMA_MEM, nid, priv);
2842 			membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv);
2843 			cpu_idx = task_faults_idx(NUMA_CPU, nid, priv);
2844 			cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv);
2873 				 * nid and priv in a specific region because it
2886 				max_nid = nid;
2890 			max_nid = nid;
3469 	/* New address space, reset the preferred nid */