kernel/sched/fair.c

1420 	rss = get_mm_rss(p->mm);
2629 		p->mm->numa_next_scan = jiffies +
2810 	 * The p->mm->numa_scan_seq field gets updated without
2814 	seq = READ_ONCE(p->mm->numa_scan_seq);
2984 	if (tsk->mm == current->mm)
3083 	/* for example, ksmd faulting in a user's mm */
3084 	if (!p->mm)
3156 	 * p->mm->numa_scan_seq is written to without exclusive access
3162 	WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
3163 	p->mm->numa_scan_offset = 0;
3166 static bool vma_is_accessed(struct mm_struct *mm, struct vm_area_struct *vma)
3175 	if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2)
3186 	if (mm->numa_scan_offset > vma->vm_start) {
3187 		trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_IGNORE_PID);
3204 	struct mm_struct *mm = p->mm;
3220 	 * NOTE: make sure not to dereference p->mm before this check,
3222 	 * without p->mm even though we still had it when we enqueued this
3228 	if (!mm->numa_next_scan) {
3229 		mm->numa_next_scan = now +
3236 	migrate = mm->numa_next_scan;
3246 	if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan))
3250 	 * Delay this task enough that another task of this mm will likely win
3262 	if (!mmap_read_trylock(mm))
3273 	start = mm->numa_scan_offset;
3274 	vma_iter_init(&vmi, mm, start);
3286 			trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_UNSUITABLE);
3298 			trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SHARED_RO);
3307 			trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_INACCESSIBLE);
3318 			vma->numab_state->start_scan_seq = mm->numa_scan_seq;
3332 			 vma->numab_state->prev_scan_seq = mm->numa_scan_seq - 1;
3339 		if (mm->numa_scan_seq && time_before(jiffies,
3341 			trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SCAN_DELAY);
3346 		if (mm->numa_scan_seq &&
3355 		if (vma->numab_state->prev_scan_seq == mm->numa_scan_seq) {
3356 			mm->numa_scan_offset = vma->vm_end;
3357 			trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SEQ_COMPLETED);
3365 		if (!vma_pids_forced && !vma_is_accessed(mm, vma)) {
3367 			trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_PID_INACTIVE);
3397 		vma->numab_state->prev_scan_seq = mm->numa_scan_seq;
3425 		mm->numa_scan_offset = start;
3428 	mmap_read_unlock(mm);
3445 	struct mm_struct *mm = p->mm;
3447 	if (mm) {
3448 		mm_users = atomic_read(&mm->mm_users);
3450 			mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
3451 			mm->numa_scan_seq = 0;
3455 	p->numa_scan_seq		= mm ? mm->numa_scan_seq : 0;
3479 	if (mm) {
3500 	if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work)
3517 		if (!time_before(jiffies, curr->mm->numa_next_scan))
3530 	if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING))