1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2022 ARM Limited.
4 */
5
6#define _GNU_SOURCE
7#define _POSIX_C_SOURCE 199309L
8
9#include <errno.h>
10#include <getopt.h>
11#include <poll.h>
12#include <signal.h>
13#include <stdbool.h>
14#include <stddef.h>
15#include <stdio.h>
16#include <stdlib.h>
17#include <string.h>
18#include <unistd.h>
19#include <sys/auxv.h>
20#include <sys/epoll.h>
21#include <sys/prctl.h>
22#include <sys/types.h>
23#include <sys/uio.h>
24#include <sys/wait.h>
25#include <asm/hwcap.h>
26
27#include "../../kselftest.h"
28
29#define MAX_VLS 16
30
31struct child_data {
32	char *name, *output;
33	pid_t pid;
34	int stdout;
35	bool output_seen;
36	bool exited;
37	int exit_status;
38};
39
40static int epoll_fd;
41static struct child_data *children;
42static struct epoll_event *evs;
43static int tests;
44static int num_children;
45static bool terminate;
46
47static int startup_pipe[2];
48
49static int num_processors(void)
50{
51	long nproc = sysconf(_SC_NPROCESSORS_CONF);
52	if (nproc < 0) {
53		perror("Unable to read number of processors\n");
54		exit(EXIT_FAILURE);
55	}
56
57	return nproc;
58}
59
60static void child_start(struct child_data *child, const char *program)
61{
62	int ret, pipefd[2], i;
63	struct epoll_event ev;
64
65	ret = pipe(pipefd);
66	if (ret != 0)
67		ksft_exit_fail_msg("Failed to create stdout pipe: %s (%d)\n",
68				   strerror(errno), errno);
69
70	child->pid = fork();
71	if (child->pid == -1)
72		ksft_exit_fail_msg("fork() failed: %s (%d)\n",
73				   strerror(errno), errno);
74
75	if (!child->pid) {
76		/*
77		 * In child, replace stdout with the pipe, errors to
78		 * stderr from here as kselftest prints to stdout.
79		 */
80		ret = dup2(pipefd[1], 1);
81		if (ret == -1) {
82			fprintf(stderr, "dup2() %d\n", errno);
83			exit(EXIT_FAILURE);
84		}
85
86		/*
87		 * Duplicate the read side of the startup pipe to
88		 * FD 3 so we can close everything else.
89		 */
90		ret = dup2(startup_pipe[0], 3);
91		if (ret == -1) {
92			fprintf(stderr, "dup2() %d\n", errno);
93			exit(EXIT_FAILURE);
94		}
95
96		/*
97		 * Very dumb mechanism to clean open FDs other than
98		 * stdio. We don't want O_CLOEXEC for the pipes...
99		 */
100		for (i = 4; i < 8192; i++)
101			close(i);
102
103		/*
104		 * Read from the startup pipe, there should be no data
105		 * and we should block until it is closed.  We just
106		 * carry on on error since this isn't super critical.
107		 */
108		ret = read(3, &i, sizeof(i));
109		if (ret < 0)
110			fprintf(stderr, "read(startp pipe) failed: %s (%d)\n",
111				strerror(errno), errno);
112		if (ret > 0)
113			fprintf(stderr, "%d bytes of data on startup pipe\n",
114				ret);
115		close(3);
116
117		ret = execl(program, program, NULL);
118		fprintf(stderr, "execl(%s) failed: %d (%s)\n",
119			program, errno, strerror(errno));
120
121		exit(EXIT_FAILURE);
122	} else {
123		/*
124		 * In parent, remember the child and close our copy of the
125		 * write side of stdout.
126		 */
127		close(pipefd[1]);
128		child->stdout = pipefd[0];
129		child->output = NULL;
130		child->exited = false;
131		child->output_seen = false;
132
133		ev.events = EPOLLIN | EPOLLHUP;
134		ev.data.ptr = child;
135
136		ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, child->stdout, &ev);
137		if (ret < 0) {
138			ksft_exit_fail_msg("%s EPOLL_CTL_ADD failed: %s (%d)\n",
139					   child->name, strerror(errno), errno);
140		}
141	}
142}
143
144static bool child_output_read(struct child_data *child)
145{
146	char read_data[1024];
147	char work[1024];
148	int ret, len, cur_work, cur_read;
149
150	ret = read(child->stdout, read_data, sizeof(read_data));
151	if (ret < 0) {
152		if (errno == EINTR)
153			return true;
154
155		ksft_print_msg("%s: read() failed: %s (%d)\n",
156			       child->name, strerror(errno),
157			       errno);
158		return false;
159	}
160	len = ret;
161
162	child->output_seen = true;
163
164	/* Pick up any partial read */
165	if (child->output) {
166		strncpy(work, child->output, sizeof(work) - 1);
167		cur_work = strnlen(work, sizeof(work));
168		free(child->output);
169		child->output = NULL;
170	} else {
171		cur_work = 0;
172	}
173
174	cur_read = 0;
175	while (cur_read < len) {
176		work[cur_work] = read_data[cur_read++];
177
178		if (work[cur_work] == '\n') {
179			work[cur_work] = '\0';
180			ksft_print_msg("%s: %s\n", child->name, work);
181			cur_work = 0;
182		} else {
183			cur_work++;
184		}
185	}
186
187	if (cur_work) {
188		work[cur_work] = '\0';
189		ret = asprintf(&child->output, "%s", work);
190		if (ret == -1)
191			ksft_exit_fail_msg("Out of memory\n");
192	}
193
194	return false;
195}
196
197static void child_output(struct child_data *child, uint32_t events,
198			 bool flush)
199{
200	bool read_more;
201
202	if (events & EPOLLIN) {
203		do {
204			read_more = child_output_read(child);
205		} while (read_more);
206	}
207
208	if (events & EPOLLHUP) {
209		close(child->stdout);
210		child->stdout = -1;
211		flush = true;
212	}
213
214	if (flush && child->output) {
215		ksft_print_msg("%s: %s<EOF>\n", child->name, child->output);
216		free(child->output);
217		child->output = NULL;
218	}
219}
220
221static void child_tickle(struct child_data *child)
222{
223	if (child->output_seen && !child->exited)
224		kill(child->pid, SIGUSR2);
225}
226
227static void child_stop(struct child_data *child)
228{
229	if (!child->exited)
230		kill(child->pid, SIGTERM);
231}
232
233static void child_cleanup(struct child_data *child)
234{
235	pid_t ret;
236	int status;
237	bool fail = false;
238
239	if (!child->exited) {
240		do {
241			ret = waitpid(child->pid, &status, 0);
242			if (ret == -1 && errno == EINTR)
243				continue;
244
245			if (ret == -1) {
246				ksft_print_msg("waitpid(%d) failed: %s (%d)\n",
247					       child->pid, strerror(errno),
248					       errno);
249				fail = true;
250				break;
251			}
252		} while (!WIFEXITED(status));
253		child->exit_status = WEXITSTATUS(status);
254	}
255
256	if (!child->output_seen) {
257		ksft_print_msg("%s no output seen\n", child->name);
258		fail = true;
259	}
260
261	if (child->exit_status != 0) {
262		ksft_print_msg("%s exited with error code %d\n",
263			       child->name, child->exit_status);
264		fail = true;
265	}
266
267	ksft_test_result(!fail, "%s\n", child->name);
268}
269
270static void handle_child_signal(int sig, siginfo_t *info, void *context)
271{
272	int i;
273	bool found = false;
274
275	for (i = 0; i < num_children; i++) {
276		if (children[i].pid == info->si_pid) {
277			children[i].exited = true;
278			children[i].exit_status = info->si_status;
279			found = true;
280			break;
281		}
282	}
283
284	if (!found)
285		ksft_print_msg("SIGCHLD for unknown PID %d with status %d\n",
286			       info->si_pid, info->si_status);
287}
288
289static void handle_exit_signal(int sig, siginfo_t *info, void *context)
290{
291	int i;
292
293	/* If we're already exiting then don't signal again */
294	if (terminate)
295		return;
296
297	ksft_print_msg("Got signal, exiting...\n");
298
299	terminate = true;
300
301	/*
302	 * This should be redundant, the main loop should clean up
303	 * after us, but for safety stop everything we can here.
304	 */
305	for (i = 0; i < num_children; i++)
306		child_stop(&children[i]);
307}
308
309static void start_fpsimd(struct child_data *child, int cpu, int copy)
310{
311	int ret;
312
313	ret = asprintf(&child->name, "FPSIMD-%d-%d", cpu, copy);
314	if (ret == -1)
315		ksft_exit_fail_msg("asprintf() failed\n");
316
317	child_start(child, "./fpsimd-test");
318
319	ksft_print_msg("Started %s\n", child->name);
320}
321
322static void start_sve(struct child_data *child, int vl, int cpu)
323{
324	int ret;
325
326	ret = prctl(PR_SVE_SET_VL, vl | PR_SVE_VL_INHERIT);
327	if (ret < 0)
328		ksft_exit_fail_msg("Failed to set SVE VL %d\n", vl);
329
330	ret = asprintf(&child->name, "SVE-VL-%d-%d", vl, cpu);
331	if (ret == -1)
332		ksft_exit_fail_msg("asprintf() failed\n");
333
334	child_start(child, "./sve-test");
335
336	ksft_print_msg("Started %s\n", child->name);
337}
338
339static void start_ssve(struct child_data *child, int vl, int cpu)
340{
341	int ret;
342
343	ret = asprintf(&child->name, "SSVE-VL-%d-%d", vl, cpu);
344	if (ret == -1)
345		ksft_exit_fail_msg("asprintf() failed\n");
346
347	ret = prctl(PR_SME_SET_VL, vl | PR_SME_VL_INHERIT);
348	if (ret < 0)
349		ksft_exit_fail_msg("Failed to set SME VL %d\n", ret);
350
351	child_start(child, "./ssve-test");
352
353	ksft_print_msg("Started %s\n", child->name);
354}
355
356static void start_za(struct child_data *child, int vl, int cpu)
357{
358	int ret;
359
360	ret = prctl(PR_SME_SET_VL, vl | PR_SVE_VL_INHERIT);
361	if (ret < 0)
362		ksft_exit_fail_msg("Failed to set SME VL %d\n", ret);
363
364	ret = asprintf(&child->name, "ZA-VL-%d-%d", vl, cpu);
365	if (ret == -1)
366		ksft_exit_fail_msg("asprintf() failed\n");
367
368	child_start(child, "./za-test");
369
370	ksft_print_msg("Started %s\n", child->name);
371}
372
373static void start_zt(struct child_data *child, int cpu)
374{
375	int ret;
376
377	ret = asprintf(&child->name, "ZT-%d", cpu);
378	if (ret == -1)
379		ksft_exit_fail_msg("asprintf() failed\n");
380
381	child_start(child, "./zt-test");
382
383	ksft_print_msg("Started %s\n", child->name);
384}
385
386static void probe_vls(int vls[], int *vl_count, int set_vl)
387{
388	unsigned int vq;
389	int vl;
390
391	*vl_count = 0;
392
393	for (vq = SVE_VQ_MAX; vq > 0; vq /= 2) {
394		vl = prctl(set_vl, vq * 16);
395		if (vl == -1)
396			ksft_exit_fail_msg("SET_VL failed: %s (%d)\n",
397					   strerror(errno), errno);
398
399		vl &= PR_SVE_VL_LEN_MASK;
400
401		if (*vl_count && (vl == vls[*vl_count - 1]))
402			break;
403
404		vq = sve_vq_from_vl(vl);
405
406		vls[*vl_count] = vl;
407		*vl_count += 1;
408	}
409}
410
411/* Handle any pending output without blocking */
412static void drain_output(bool flush)
413{
414	int ret = 1;
415	int i;
416
417	while (ret > 0) {
418		ret = epoll_wait(epoll_fd, evs, tests, 0);
419		if (ret < 0) {
420			if (errno == EINTR)
421				continue;
422			ksft_print_msg("epoll_wait() failed: %s (%d)\n",
423				       strerror(errno), errno);
424		}
425
426		for (i = 0; i < ret; i++)
427			child_output(evs[i].data.ptr, evs[i].events, flush);
428	}
429}
430
431static const struct option options[] = {
432	{ "timeout",	required_argument, NULL, 't' },
433	{ }
434};
435
436int main(int argc, char **argv)
437{
438	int ret;
439	int timeout = 10;
440	int cpus, i, j, c;
441	int sve_vl_count, sme_vl_count, fpsimd_per_cpu;
442	bool all_children_started = false;
443	int seen_children;
444	int sve_vls[MAX_VLS], sme_vls[MAX_VLS];
445	bool have_sme2;
446	struct sigaction sa;
447
448	while ((c = getopt_long(argc, argv, "t:", options, NULL)) != -1) {
449		switch (c) {
450		case 't':
451			ret = sscanf(optarg, "%d", &timeout);
452			if (ret != 1)
453				ksft_exit_fail_msg("Failed to parse timeout %s\n",
454						   optarg);
455			break;
456		default:
457			ksft_exit_fail_msg("Unknown argument\n");
458		}
459	}
460
461	cpus = num_processors();
462	tests = 0;
463
464	if (getauxval(AT_HWCAP) & HWCAP_SVE) {
465		probe_vls(sve_vls, &sve_vl_count, PR_SVE_SET_VL);
466		tests += sve_vl_count * cpus;
467	} else {
468		sve_vl_count = 0;
469	}
470
471	if (getauxval(AT_HWCAP2) & HWCAP2_SME) {
472		probe_vls(sme_vls, &sme_vl_count, PR_SME_SET_VL);
473		tests += sme_vl_count * cpus * 2;
474	} else {
475		sme_vl_count = 0;
476	}
477
478	if (getauxval(AT_HWCAP2) & HWCAP2_SME2) {
479		tests += cpus;
480		have_sme2 = true;
481	} else {
482		have_sme2 = false;
483	}
484
485	/* Force context switching if we only have FPSIMD */
486	if (!sve_vl_count && !sme_vl_count)
487		fpsimd_per_cpu = 2;
488	else
489		fpsimd_per_cpu = 1;
490	tests += cpus * fpsimd_per_cpu;
491
492	ksft_print_header();
493	ksft_set_plan(tests);
494
495	ksft_print_msg("%d CPUs, %d SVE VLs, %d SME VLs, SME2 %s\n",
496		       cpus, sve_vl_count, sme_vl_count,
497		       have_sme2 ? "present" : "absent");
498
499	if (timeout > 0)
500		ksft_print_msg("Will run for %ds\n", timeout);
501	else
502		ksft_print_msg("Will run until terminated\n");
503
504	children = calloc(sizeof(*children), tests);
505	if (!children)
506		ksft_exit_fail_msg("Unable to allocate child data\n");
507
508	ret = epoll_create1(EPOLL_CLOEXEC);
509	if (ret < 0)
510		ksft_exit_fail_msg("epoll_create1() failed: %s (%d)\n",
511				   strerror(errno), ret);
512	epoll_fd = ret;
513
514	/* Create a pipe which children will block on before execing */
515	ret = pipe(startup_pipe);
516	if (ret != 0)
517		ksft_exit_fail_msg("Failed to create startup pipe: %s (%d)\n",
518				   strerror(errno), errno);
519
520	/* Get signal handers ready before we start any children */
521	memset(&sa, 0, sizeof(sa));
522	sa.sa_sigaction = handle_exit_signal;
523	sa.sa_flags = SA_RESTART | SA_SIGINFO;
524	sigemptyset(&sa.sa_mask);
525	ret = sigaction(SIGINT, &sa, NULL);
526	if (ret < 0)
527		ksft_print_msg("Failed to install SIGINT handler: %s (%d)\n",
528			       strerror(errno), errno);
529	ret = sigaction(SIGTERM, &sa, NULL);
530	if (ret < 0)
531		ksft_print_msg("Failed to install SIGTERM handler: %s (%d)\n",
532			       strerror(errno), errno);
533	sa.sa_sigaction = handle_child_signal;
534	ret = sigaction(SIGCHLD, &sa, NULL);
535	if (ret < 0)
536		ksft_print_msg("Failed to install SIGCHLD handler: %s (%d)\n",
537			       strerror(errno), errno);
538
539	evs = calloc(tests, sizeof(*evs));
540	if (!evs)
541		ksft_exit_fail_msg("Failed to allocated %d epoll events\n",
542				   tests);
543
544	for (i = 0; i < cpus; i++) {
545		for (j = 0; j < fpsimd_per_cpu; j++)
546			start_fpsimd(&children[num_children++], i, j);
547
548		for (j = 0; j < sve_vl_count; j++)
549			start_sve(&children[num_children++], sve_vls[j], i);
550
551		for (j = 0; j < sme_vl_count; j++) {
552			start_ssve(&children[num_children++], sme_vls[j], i);
553			start_za(&children[num_children++], sme_vls[j], i);
554		}
555
556		if (have_sme2)
557			start_zt(&children[num_children++], i);
558	}
559
560	/*
561	 * All children started, close the startup pipe and let them
562	 * run.
563	 */
564	close(startup_pipe[0]);
565	close(startup_pipe[1]);
566
567	for (;;) {
568		/* Did we get a signal asking us to exit? */
569		if (terminate)
570			break;
571
572		/*
573		 * Timeout is counted in seconds with no output, the
574		 * tests print during startup then are silent when
575		 * running so this should ensure they all ran enough
576		 * to install the signal handler, this is especially
577		 * useful in emulation where we will both be slow and
578		 * likely to have a large set of VLs.
579		 */
580		ret = epoll_wait(epoll_fd, evs, tests, 1000);
581		if (ret < 0) {
582			if (errno == EINTR)
583				continue;
584			ksft_exit_fail_msg("epoll_wait() failed: %s (%d)\n",
585					   strerror(errno), errno);
586		}
587
588		/* Output? */
589		if (ret > 0) {
590			for (i = 0; i < ret; i++) {
591				child_output(evs[i].data.ptr, evs[i].events,
592					     false);
593			}
594			continue;
595		}
596
597		/* Otherwise epoll_wait() timed out */
598
599		/*
600		 * If the child processes have not produced output they
601		 * aren't actually running the tests yet .
602		 */
603		if (!all_children_started) {
604			seen_children = 0;
605
606			for (i = 0; i < num_children; i++)
607				if (children[i].output_seen ||
608				    children[i].exited)
609					seen_children++;
610
611			if (seen_children != num_children) {
612				ksft_print_msg("Waiting for %d children\n",
613					       num_children - seen_children);
614				continue;
615			}
616
617			all_children_started = true;
618		}
619
620		ksft_print_msg("Sending signals, timeout remaining: %d\n",
621			       timeout);
622
623		for (i = 0; i < num_children; i++)
624			child_tickle(&children[i]);
625
626		/* Negative timeout means run indefinitely */
627		if (timeout < 0)
628			continue;
629		if (--timeout == 0)
630			break;
631	}
632
633	ksft_print_msg("Finishing up...\n");
634	terminate = true;
635
636	for (i = 0; i < tests; i++)
637		child_stop(&children[i]);
638
639	drain_output(false);
640
641	for (i = 0; i < tests; i++)
642		child_cleanup(&children[i]);
643
644	drain_output(true);
645
646	ksft_print_cnts();
647
648	return 0;
649}
650