kernel/sched/fair.c

1388 	 * faults[] array is split into two regions: faults_mem and faults_cpu.
1392 	 * more by CPU use than by memory faults.
1394 	unsigned long faults[];
1422 	 * by the PTE scanner and NUMA hinting faults should be trapped based
1511 /* Shared or private faults. */
1561 	return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] +
1562 		ng->faults[task_faults_idx(NUMA_MEM, nid, 1)];
1567 	return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] +
1568 		group->faults[task_faults_idx(NUMA_CPU, nid, 1)];
1573 	unsigned long faults = 0;
1577 		faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)];
1580 	return faults;
1585 	unsigned long faults = 0;
1589 		faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)];
1592 	return faults;
1596  * A node triggering more than 1/3 as many NUMA faults as the maximum is
1628 		unsigned long faults;
1648 		/* Add up the faults from nearby nodes. */
1650 			faults = task_faults(p, node);
1652 			faults = group_faults(p, node);
1659 		 * The further away a node is, the less the faults count.
1663 			faults *= (max_dist - dist);
1664 			faults /= (max_dist - LOCAL_DISTANCE);
1667 		score += faults;
1682 	unsigned long faults, total_faults;
1692 	faults = task_faults(p, nid);
1693 	faults += score_nearby_nodes(p, nid, dist, true);
1695 	return 1000 * faults / total_faults;
1702 	unsigned long faults, total_faults;
1712 	faults = group_faults(p, nid);
1713 	faults += score_nearby_nodes(p, nid, dist, false);
1715 	return 1000 * faults / total_faults;
1884 	 * Allow first faults or private faults to migrate immediately early in
1902 	 * Our periodic faults will sample this probability and getting the
1914 	/* Always allow migrate on private faults */
2573  * tracking the nodes from which NUMA hinting faults are triggered. This can
2579 	unsigned long faults, max_faults = 0;
2583 		faults = group_faults_cpu(numa_group, nid);
2584 		if (faults > max_faults)
2585 			max_faults = faults;
2589 		faults = group_faults_cpu(numa_group, nid);
2590 		if (faults * ACTIVE_NODE_FRACTION > max_faults)
2625 	 * If there were no record hinting faults then either the task is
2672 		 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS,
2733 	 * scores nodes according to the number of NUMA hinting faults on
2772 			unsigned long faults = 0;
2776 			/* Sum group's NUMA faults; includes a==b case. */
2779 					faults += group_faults(p, b);
2786 			if (faults > max_faults) {
2787 				max_faults = faults;
2837 	/* Find the node with the highest number of faults */
2841 		unsigned long faults = 0, group_faults = 0;
2852 			/* Decay existing window, copy faults since last scan */
2860 			 * number of faults. Tasks with little runtime have
2862 			 * faults are less important.
2872 			faults += p->numa_faults[mem_idx];
2882 				ng->faults[mem_idx] += diff;
2883 				ng->faults[cpu_idx] += f_diff;
2885 				group_faults += ng->faults[mem_idx];
2890 			if (faults > max_faults) {
2891 				max_faults = faults;
2954 			grp->faults[i] = p->numa_faults[i];
3012 		my_grp->faults[i] -= p->numa_faults[i];
3013 		grp->faults[i] += p->numa_faults[i];
3055 			grp->faults[i] -= p->numa_faults[i];
3094 	 * NUMA faults statistics are unnecessary for the slow memory
3102 	/* Allocate buffer to track faults on a per-node basis */
3299 		 * hinting faults in read-only file-backed mappings or the vdso
3496  * Drive the periodic memory faults..
3543 	 * Allow resets if faults have been trapped before one scan
13191 			gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)],
13192 			gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)];