1/*
2 * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved.
3 *
4 * Licensed under the Apache License 2.0 (the "License").  You may not use
5 * this file except in compliance with the License.  You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
8 */
9
10#include "../testutil.h"
11#include "output.h"
12#include "tu_local.h"
13
14#include <string.h>
15#include <assert.h>
16
17#include "internal/nelem.h"
18#include <openssl/bio.h>
19
20#include "platform.h"            /* From libapps */
21
22#if defined(_WIN32) && !defined(__BORLANDC__)
23# define strdup _strdup
24#endif
25
26
27/*
28 * Declares the structures needed to register each test case function.
29 */
30typedef struct test_info {
31    const char *test_case_name;
32    int (*test_fn) (void);
33    int (*param_test_fn)(int idx);
34    int num;
35
36    /* flags */
37    int subtest:1;
38} TEST_INFO;
39
40static TEST_INFO all_tests[1024];
41static int num_tests = 0;
42static int show_list = 0;
43static int single_test = -1;
44static int single_iter = -1;
45static int level = 0;
46static int seed = 0;
47static int rand_order = 0;
48
49/*
50 * A parameterised test runs a loop of test cases.
51 * |num_test_cases| counts the total number of non-subtest test cases
52 * across all tests.
53 */
54static int num_test_cases = 0;
55
56static int process_shared_options(void);
57
58
59void add_test(const char *test_case_name, int (*test_fn) (void))
60{
61    assert(num_tests != OSSL_NELEM(all_tests));
62    all_tests[num_tests].test_case_name = test_case_name;
63    all_tests[num_tests].test_fn = test_fn;
64    all_tests[num_tests].num = -1;
65    ++num_tests;
66    ++num_test_cases;
67}
68
69void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
70                   int num, int subtest)
71{
72    assert(num_tests != OSSL_NELEM(all_tests));
73    all_tests[num_tests].test_case_name = test_case_name;
74    all_tests[num_tests].param_test_fn = test_fn;
75    all_tests[num_tests].num = num;
76    all_tests[num_tests].subtest = subtest;
77    ++num_tests;
78    if (subtest)
79        ++num_test_cases;
80    else
81        num_test_cases += num;
82}
83
84static int gcd(int a, int b)
85{
86    while (b != 0) {
87        int t = b;
88        b = a % b;
89        a = t;
90    }
91    return a;
92}
93
94static void set_seed(int s)
95{
96    seed = s;
97    if (seed <= 0)
98        seed = (int)time(NULL);
99    test_random_seed(seed);
100}
101
102
103int setup_test_framework(int argc, char *argv[])
104{
105    char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");
106    char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
107
108    if (TAP_levels != NULL)
109        level = 4 * atoi(TAP_levels);
110    test_adjust_streams_tap_level(level);
111    if (test_seed != NULL) {
112        rand_order = 1;
113        set_seed(atoi(test_seed));
114    } else {
115        set_seed(0);
116    }
117
118#if defined(OPENSSL_SYS_VMS) && defined(__DECC)
119    argv = copy_argv(&argc, argv);
120#elif defined(_WIN32)
121    /*
122     * Replace argv[] with UTF-8 encoded strings.
123     */
124    win32_utf8argv(&argc, &argv);
125#endif
126
127    if (!opt_init(argc, argv, test_get_options()))
128        return 0;
129    return 1;
130}
131
132
133/*
134 * This can only be called after setup() has run, since num_tests and
135 * all_tests[] are setup at this point
136 */
137static int check_single_test_params(char *name, char *testname, char *itname)
138{
139    if (name != NULL) {
140        int i;
141        for (i = 0; i < num_tests; ++i) {
142            if (strcmp(name, all_tests[i].test_case_name) == 0) {
143                single_test = 1 + i;
144                break;
145            }
146        }
147        if (i >= num_tests)
148            single_test = atoi(name);
149    }
150
151
152    /* if only iteration is specified, assume we want the first test */
153    if (single_test == -1 && single_iter != -1)
154        single_test = 1;
155
156    if (single_test != -1) {
157        if (single_test < 1 || single_test > num_tests) {
158            test_printf_stderr("Invalid -%s value "
159                               "(Value must be a valid test name OR a value between %d..%d)\n",
160                               testname, 1, num_tests);
161            return 0;
162        }
163    }
164    if (single_iter != -1) {
165        if (all_tests[single_test - 1].num == -1) {
166            test_printf_stderr("-%s option is not valid for test %d:%s\n",
167                               itname,
168                               single_test,
169                               all_tests[single_test - 1].test_case_name);
170            return 0;
171        } else if (single_iter < 1
172                   || single_iter > all_tests[single_test - 1].num) {
173            test_printf_stderr("Invalid -%s value for test %d:%s\t"
174                               "(Value must be in the range %d..%d)\n",
175                               itname, single_test,
176                               all_tests[single_test - 1].test_case_name,
177                               1, all_tests[single_test - 1].num);
178            return 0;
179        }
180    }
181    return 1;
182}
183
184static int process_shared_options(void)
185{
186    OPTION_CHOICE_DEFAULT o;
187    int value;
188    int ret = -1;
189    char empty[] = "";
190    char *flag_test = empty;
191    char *flag_iter = empty;
192    char *testname = NULL;
193
194    opt_begin();
195    while ((o = opt_next()) != OPT_EOF) {
196        switch (o) {
197        /* Ignore any test options at this level */
198        default:
199            break;
200        case OPT_ERR:
201            return ret;
202        case OPT_TEST_HELP:
203            opt_help(test_get_options());
204            return 0;
205        case OPT_TEST_LIST:
206            show_list = 1;
207            break;
208        case OPT_TEST_SINGLE:
209            flag_test = opt_flag();
210            testname = opt_arg();
211            break;
212        case OPT_TEST_ITERATION:
213            flag_iter = opt_flag();
214            if (!opt_int(opt_arg(), &single_iter))
215                goto end;
216            break;
217        case OPT_TEST_INDENT:
218            if (!opt_int(opt_arg(), &value))
219                goto end;
220            level = 4 * value;
221            test_adjust_streams_tap_level(level);
222            break;
223        case OPT_TEST_SEED:
224            if (!opt_int(opt_arg(), &value))
225                goto end;
226            set_seed(value);
227            break;
228        }
229    }
230    if (!check_single_test_params(testname, flag_test, flag_iter))
231        goto end;
232    ret = 1;
233end:
234    return ret;
235}
236
237
238int pulldown_test_framework(int ret)
239{
240    set_test_title(NULL);
241    return ret;
242}
243
244static void finalize(int success)
245{
246    if (success)
247        ERR_clear_error();
248    else
249        ERR_print_errors_cb(openssl_error_cb, NULL);
250}
251
252static char *test_title = NULL;
253
254void set_test_title(const char *title)
255{
256    free(test_title);
257    test_title = title == NULL ? NULL : strdup(title);
258}
259
260PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,
261                                             const char *description, ...)
262{
263    va_list ap;
264
265    test_flush_stdout();
266    test_flush_stderr();
267
268    if (verdict == 0 && seed != 0)
269        test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed);
270    test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok");
271    va_start(ap, description);
272    test_vprintf_tapout(description, ap);
273    va_end(ap);
274    if (verdict == TEST_SKIP_CODE)
275        test_printf_tapout(" # skipped");
276    test_printf_tapout("\n");
277    test_flush_tapout();
278}
279
280int run_tests(const char *test_prog_name)
281{
282    int num_failed = 0;
283    int verdict = 1;
284    int ii, i, jj, j, jstep;
285    int test_case_count = 0;
286    int subtest_case_count = 0;
287    int permute[OSSL_NELEM(all_tests)];
288
289    i = process_shared_options();
290    if (i == 0)
291        return EXIT_SUCCESS;
292    if (i == -1)
293        return EXIT_FAILURE;
294
295    if (num_tests < 1) {
296        test_printf_tapout("1..0 # Skipped: %s\n", test_prog_name);
297    } else if (show_list == 0 && single_test == -1) {
298        if (level > 0) {
299            test_printf_stdout("Subtest: %s\n", test_prog_name);
300            test_flush_stdout();
301        }
302        test_printf_tapout("1..%d\n", num_test_cases);
303    }
304
305    test_flush_tapout();
306
307    for (i = 0; i < num_tests; i++)
308        permute[i] = i;
309    if (rand_order != 0)
310        for (i = num_tests - 1; i >= 1; i--) {
311            j = test_random() % (1 + i);
312            ii = permute[j];
313            permute[j] = permute[i];
314            permute[i] = ii;
315        }
316
317    for (ii = 0; ii != num_tests; ++ii) {
318        i = permute[ii];
319
320        if (single_test != -1 && ((i+1) != single_test)) {
321            continue;
322        }
323        else if (show_list) {
324            if (all_tests[i].num != -1) {
325                test_printf_tapout("%d - %s (%d..%d)\n", ii + 1,
326                                   all_tests[i].test_case_name, 1,
327                                   all_tests[i].num);
328            } else {
329                test_printf_tapout("%d - %s\n", ii + 1,
330                                   all_tests[i].test_case_name);
331            }
332            test_flush_tapout();
333        } else if (all_tests[i].num == -1) {
334            set_test_title(all_tests[i].test_case_name);
335            ERR_clear_error();
336            verdict = all_tests[i].test_fn();
337            finalize(verdict != 0);
338            test_verdict(verdict, "%d - %s", test_case_count + 1, test_title);
339            if (verdict == 0)
340                num_failed++;
341            test_case_count++;
342        } else {
343            verdict = TEST_SKIP_CODE;
344            set_test_title(all_tests[i].test_case_name);
345            if (all_tests[i].subtest) {
346                level += 4;
347                test_adjust_streams_tap_level(level);
348                if (single_iter == -1) {
349                    test_printf_stdout("Subtest: %s\n", test_title);
350                    test_printf_tapout("%d..%d\n", 1, all_tests[i].num);
351                    test_flush_stdout();
352                    test_flush_tapout();
353                }
354            }
355
356            j = -1;
357            if (rand_order == 0 || all_tests[i].num < 3)
358                jstep = 1;
359            else
360                do
361                    jstep = test_random() % all_tests[i].num;
362                while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
363
364            for (jj = 0; jj < all_tests[i].num; jj++) {
365                int v;
366
367                j = (j + jstep) % all_tests[i].num;
368                if (single_iter != -1 && ((jj + 1) != single_iter))
369                    continue;
370                ERR_clear_error();
371                v = all_tests[i].param_test_fn(j);
372
373                if (v == 0) {
374                    verdict = 0;
375                } else if (v != TEST_SKIP_CODE && verdict != 0) {
376                    verdict = 1;
377                }
378
379                finalize(v != 0);
380
381                if (all_tests[i].subtest)
382                    test_verdict(v, "%d - iteration %d",
383                                 subtest_case_count + 1, j + 1);
384                else
385                    test_verdict(v, "%d - %s - iteration %d",
386                                 test_case_count + subtest_case_count + 1,
387                                 test_title, j + 1);
388                subtest_case_count++;
389            }
390
391            if (all_tests[i].subtest) {
392                level -= 4;
393                test_adjust_streams_tap_level(level);
394            }
395            if (verdict == 0)
396                ++num_failed;
397            if (all_tests[i].num == -1 || all_tests[i].subtest)
398                test_verdict(verdict, "%d - %s", test_case_count + 1,
399                             all_tests[i].test_case_name);
400            test_case_count++;
401        }
402    }
403    if (num_failed != 0)
404        return EXIT_FAILURE;
405    return EXIT_SUCCESS;
406}
407
408/*
409 * Glue an array of strings together and return it as an allocated string.
410 * Optionally return the whole length of this string in |out_len|
411 */
412char *glue_strings(const char *list[], size_t *out_len)
413{
414    size_t len = 0;
415    char *p, *ret;
416    int i;
417
418    for (i = 0; list[i] != NULL; i++)
419        len += strlen(list[i]);
420
421    if (out_len != NULL)
422        *out_len = len;
423
424    if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1)))
425        return NULL;
426
427    for (i = 0; list[i] != NULL; i++)
428        p += strlen(strcpy(p, list[i]));
429
430    return ret;
431}
432
433char *test_mk_file_path(const char *dir, const char *file)
434{
435# ifndef OPENSSL_SYS_VMS
436    const char *sep = "/";
437# else
438    const char *sep = "";
439    char *dir_end;
440    char dir_end_sep;
441# endif
442    size_t dirlen = dir != NULL ? strlen(dir) : 0;
443    size_t len = dirlen + strlen(sep) + strlen(file) + 1;
444    char *full_file = OPENSSL_zalloc(len);
445
446    if (full_file != NULL) {
447        if (dir != NULL && dirlen > 0) {
448            OPENSSL_strlcpy(full_file, dir, len);
449# ifdef OPENSSL_SYS_VMS
450            /*
451             * If |file| contains a directory spec, we need to do some
452             * careful merging.
453             * "vol:[dir.dir]" + "[.certs]sm2-root.crt" should become
454             * "vol:[dir.dir.certs]sm2-root.crt"
455             */
456            dir_end = &full_file[strlen(full_file) - 1];
457            dir_end_sep = *dir_end;
458            if ((dir_end_sep == ']' || dir_end_sep == '>')
459                && (file[0] == '[' || file[0] == '<')) {
460                file++;
461                if (file[0] == '.')
462                    *dir_end = '\0';
463                else
464                    *dir_end = '.';
465            }
466#else
467            OPENSSL_strlcat(full_file, sep, len);
468#endif
469        }
470        OPENSSL_strlcat(full_file, file, len);
471    }
472
473    return full_file;
474}
475