1// SPDX-License-Identifier: GPL-2.0-only
2#define _GNU_SOURCE /* for program_invocation_short_name */
3#include <errno.h>
4#include <fcntl.h>
5#include <pthread.h>
6#include <sched.h>
7#include <stdio.h>
8#include <stdlib.h>
9#include <string.h>
10#include <signal.h>
11#include <syscall.h>
12#include <sys/ioctl.h>
13#include <sys/sysinfo.h>
14#include <asm/barrier.h>
15#include <linux/atomic.h>
16#include <linux/rseq.h>
17#include <linux/unistd.h>
18
19#include "kvm_util.h"
20#include "processor.h"
21#include "test_util.h"
22
23#include "../rseq/rseq.c"
24
25/*
26 * Any bug related to task migration is likely to be timing-dependent; perform
27 * a large number of migrations to reduce the odds of a false negative.
28 */
29#define NR_TASK_MIGRATIONS 100000
30
31static pthread_t migration_thread;
32static cpu_set_t possible_mask;
33static int min_cpu, max_cpu;
34static bool done;
35
36static atomic_t seq_cnt;
37
38static void guest_code(void)
39{
40	for (;;)
41		GUEST_SYNC(0);
42}
43
44static int next_cpu(int cpu)
45{
46	/*
47	 * Advance to the next CPU, skipping those that weren't in the original
48	 * affinity set.  Sadly, there is no CPU_SET_FOR_EACH, and cpu_set_t's
49	 * data storage is considered as opaque.  Note, if this task is pinned
50	 * to a small set of discontigous CPUs, e.g. 2 and 1023, this loop will
51	 * burn a lot cycles and the test will take longer than normal to
52	 * complete.
53	 */
54	do {
55		cpu++;
56		if (cpu > max_cpu) {
57			cpu = min_cpu;
58			TEST_ASSERT(CPU_ISSET(cpu, &possible_mask),
59				    "Min CPU = %d must always be usable", cpu);
60			break;
61		}
62	} while (!CPU_ISSET(cpu, &possible_mask));
63
64	return cpu;
65}
66
67static void *migration_worker(void *__rseq_tid)
68{
69	pid_t rseq_tid = (pid_t)(unsigned long)__rseq_tid;
70	cpu_set_t allowed_mask;
71	int r, i, cpu;
72
73	CPU_ZERO(&allowed_mask);
74
75	for (i = 0, cpu = min_cpu; i < NR_TASK_MIGRATIONS; i++, cpu = next_cpu(cpu)) {
76		CPU_SET(cpu, &allowed_mask);
77
78		/*
79		 * Bump the sequence count twice to allow the reader to detect
80		 * that a migration may have occurred in between rseq and sched
81		 * CPU ID reads.  An odd sequence count indicates a migration
82		 * is in-progress, while a completely different count indicates
83		 * a migration occurred since the count was last read.
84		 */
85		atomic_inc(&seq_cnt);
86
87		/*
88		 * Ensure the odd count is visible while getcpu() isn't
89		 * stable, i.e. while changing affinity is in-progress.
90		 */
91		smp_wmb();
92		r = sched_setaffinity(rseq_tid, sizeof(allowed_mask), &allowed_mask);
93		TEST_ASSERT(!r, "sched_setaffinity failed, errno = %d (%s)",
94			    errno, strerror(errno));
95		smp_wmb();
96		atomic_inc(&seq_cnt);
97
98		CPU_CLR(cpu, &allowed_mask);
99
100		/*
101		 * Wait 1-10us before proceeding to the next iteration and more
102		 * specifically, before bumping seq_cnt again.  A delay is
103		 * needed on three fronts:
104		 *
105		 *  1. To allow sched_setaffinity() to prompt migration before
106		 *     ioctl(KVM_RUN) enters the guest so that TIF_NOTIFY_RESUME
107		 *     (or TIF_NEED_RESCHED, which indirectly leads to handling
108		 *     NOTIFY_RESUME) is handled in KVM context.
109		 *
110		 *     If NOTIFY_RESUME/NEED_RESCHED is set after KVM enters
111		 *     the guest, the guest will trigger a IO/MMIO exit all the
112		 *     way to userspace and the TIF flags will be handled by
113		 *     the generic "exit to userspace" logic, not by KVM.  The
114		 *     exit to userspace is necessary to give the test a chance
115		 *     to check the rseq CPU ID (see #2).
116		 *
117		 *     Alternatively, guest_code() could include an instruction
118		 *     to trigger an exit that is handled by KVM, but any such
119		 *     exit requires architecture specific code.
120		 *
121		 *  2. To let ioctl(KVM_RUN) make its way back to the test
122		 *     before the next round of migration.  The test's check on
123		 *     the rseq CPU ID must wait for migration to complete in
124		 *     order to avoid false positive, thus any kernel rseq bug
125		 *     will be missed if the next migration starts before the
126		 *     check completes.
127		 *
128		 *  3. To ensure the read-side makes efficient forward progress,
129		 *     e.g. if getcpu() involves a syscall. Stalling the read-side
130		 *     means the test will spend more time waiting for getcpu()
131		 *     to stabilize and less time trying to hit the timing-dependent
132		 *     bug.
133		 *
134		 * Because any bug in this area is likely to be timing-dependent,
135		 * run with a range of delays at 1us intervals from 1us to 10us
136		 * as a best effort to avoid tuning the test to the point where
137		 * it can hit _only_ the original bug and not detect future
138		 * regressions.
139		 *
140		 * The original bug can reproduce with a delay up to ~500us on
141		 * x86-64, but starts to require more iterations to reproduce
142		 * as the delay creeps above ~10us, and the average runtime of
143		 * each iteration obviously increases as well.  Cap the delay
144		 * at 10us to keep test runtime reasonable while minimizing
145		 * potential coverage loss.
146		 *
147		 * The lower bound for reproducing the bug is likely below 1us,
148		 * e.g. failures occur on x86-64 with nanosleep(0), but at that
149		 * point the overhead of the syscall likely dominates the delay.
150		 * Use usleep() for simplicity and to avoid unnecessary kernel
151		 * dependencies.
152		 */
153		usleep((i % 10) + 1);
154	}
155	done = true;
156	return NULL;
157}
158
159static void calc_min_max_cpu(void)
160{
161	int i, cnt, nproc;
162
163	TEST_REQUIRE(CPU_COUNT(&possible_mask) >= 2);
164
165	/*
166	 * CPU_SET doesn't provide a FOR_EACH helper, get the min/max CPU that
167	 * this task is affined to in order to reduce the time spent querying
168	 * unusable CPUs, e.g. if this task is pinned to a small percentage of
169	 * total CPUs.
170	 */
171	nproc = get_nprocs_conf();
172	min_cpu = -1;
173	max_cpu = -1;
174	cnt = 0;
175
176	for (i = 0; i < nproc; i++) {
177		if (!CPU_ISSET(i, &possible_mask))
178			continue;
179		if (min_cpu == -1)
180			min_cpu = i;
181		max_cpu = i;
182		cnt++;
183	}
184
185	__TEST_REQUIRE(cnt >= 2,
186		       "Only one usable CPU, task migration not possible");
187}
188
189int main(int argc, char *argv[])
190{
191	int r, i, snapshot;
192	struct kvm_vm *vm;
193	struct kvm_vcpu *vcpu;
194	u32 cpu, rseq_cpu;
195
196	r = sched_getaffinity(0, sizeof(possible_mask), &possible_mask);
197	TEST_ASSERT(!r, "sched_getaffinity failed, errno = %d (%s)", errno,
198		    strerror(errno));
199
200	calc_min_max_cpu();
201
202	r = rseq_register_current_thread();
203	TEST_ASSERT(!r, "rseq_register_current_thread failed, errno = %d (%s)",
204		    errno, strerror(errno));
205
206	/*
207	 * Create and run a dummy VM that immediately exits to userspace via
208	 * GUEST_SYNC, while concurrently migrating the process by setting its
209	 * CPU affinity.
210	 */
211	vm = vm_create_with_one_vcpu(&vcpu, guest_code);
212
213	pthread_create(&migration_thread, NULL, migration_worker,
214		       (void *)(unsigned long)syscall(SYS_gettid));
215
216	for (i = 0; !done; i++) {
217		vcpu_run(vcpu);
218		TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC,
219			    "Guest failed?");
220
221		/*
222		 * Verify rseq's CPU matches sched's CPU.  Ensure migration
223		 * doesn't occur between getcpu() and reading the rseq cpu_id
224		 * by rereading both if the sequence count changes, or if the
225		 * count is odd (migration in-progress).
226		 */
227		do {
228			/*
229			 * Drop bit 0 to force a mismatch if the count is odd,
230			 * i.e. if a migration is in-progress.
231			 */
232			snapshot = atomic_read(&seq_cnt) & ~1;
233
234			/*
235			 * Ensure calling getcpu() and reading rseq.cpu_id complete
236			 * in a single "no migration" window, i.e. are not reordered
237			 * across the seq_cnt reads.
238			 */
239			smp_rmb();
240			r = sys_getcpu(&cpu, NULL);
241			TEST_ASSERT(!r, "getcpu failed, errno = %d (%s)",
242				    errno, strerror(errno));
243			rseq_cpu = rseq_current_cpu_raw();
244			smp_rmb();
245		} while (snapshot != atomic_read(&seq_cnt));
246
247		TEST_ASSERT(rseq_cpu == cpu,
248			    "rseq CPU = %d, sched CPU = %d", rseq_cpu, cpu);
249	}
250
251	/*
252	 * Sanity check that the test was able to enter the guest a reasonable
253	 * number of times, e.g. didn't get stalled too often/long waiting for
254	 * getcpu() to stabilize.  A 2:1 migration:KVM_RUN ratio is a fairly
255	 * conservative ratio on x86-64, which can do _more_ KVM_RUNs than
256	 * migrations given the 1us+ delay in the migration task.
257	 */
258	TEST_ASSERT(i > (NR_TASK_MIGRATIONS / 2),
259		    "Only performed %d KVM_RUNs, task stalled too much?", i);
260
261	pthread_join(migration_thread, NULL);
262
263	kvm_vm_free(vm);
264
265	rseq_unregister_current_thread();
266
267	return 0;
268}
269