1/*
2 * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved.
3 *
4 * Licensed under the Apache License 2.0 (the "License").  You may not use
5 * this file except in compliance with the License.  You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
8 */
9
10#include "../testutil.h"
11#include "output.h"
12#include "tu_local.h"
13
14#include <string.h>
15#include <assert.h>
16
17#include "internal/nelem.h"
18#include <openssl/bio.h>
19
20#include "platform.h"            /* From libapps */
21
22#if defined(_WIN32) && !defined(__BORLANDC__)
23# define strdup _strdup
24#endif
25
26
27/*
28 * Declares the structures needed to register each test case function.
29 */
30typedef struct test_info {
31    const char *test_case_name;
32    int (*test_fn) (void);
33    int (*param_test_fn)(int idx);
34    int num;
35
36    /* flags */
37    int subtest:1;
38} TEST_INFO;
39
40static TEST_INFO all_tests[1024];
41static int num_tests = 0;
42static int show_list = 0;
43static int single_test = -1;
44static int single_iter = -1;
45static int level = 0;
46static int seed = 0;
47static int rand_order = 0;
48
49/*
50 * A parameterised test runs a loop of test cases.
51 * |num_test_cases| counts the total number of non-subtest test cases
52 * across all tests.
53 */
54static int num_test_cases = 0;
55
56static int process_shared_options(void);
57
58
59void add_test(const char *test_case_name, int (*test_fn) (void))
60{
61    assert(num_tests != OSSL_NELEM(all_tests));
62    all_tests[num_tests].test_case_name = test_case_name;
63    all_tests[num_tests].test_fn = test_fn;
64    all_tests[num_tests].num = -1;
65    ++num_tests;
66    ++num_test_cases;
67}
68
69void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
70                   int num, int subtest)
71{
72    assert(num_tests != OSSL_NELEM(all_tests));
73    all_tests[num_tests].test_case_name = test_case_name;
74    all_tests[num_tests].param_test_fn = test_fn;
75    all_tests[num_tests].num = num;
76    all_tests[num_tests].subtest = subtest;
77    ++num_tests;
78    if (subtest)
79        ++num_test_cases;
80    else
81        num_test_cases += num;
82}
83
84static int gcd(int a, int b)
85{
86    while (b != 0) {
87        int t = b;
88        b = a % b;
89        a = t;
90    }
91    return a;
92}
93
94static void set_seed(int s)
95{
96    seed = s;
97    if (seed <= 0)
98        seed = (int)time(NULL);
99    test_random_seed(seed);
100}
101
102
103int setup_test_framework(int argc, char *argv[])
104{
105    char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");
106    char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
107
108    if (TAP_levels != NULL)
109        level = 4 * atoi(TAP_levels);
110    test_adjust_streams_tap_level(level);
111    if (test_seed != NULL) {
112        rand_order = 1;
113        set_seed(atoi(test_seed));
114    } else {
115        set_seed(0);
116    }
117
118#if defined(OPENSSL_SYS_VMS) && defined(__DECC)
119    argv = copy_argv(&argc, argv);
120#elif defined(_WIN32)
121    /*
122     * Replace argv[] with UTF-8 encoded strings.
123     */
124    win32_utf8argv(&argc, &argv);
125#endif
126
127    if (!opt_init(argc, argv, test_get_options()))
128        return 0;
129    return 1;
130}
131
132
133/*
134 * This can only be called after setup() has run, since num_tests and
135 * all_tests[] are setup at this point
136 */
137static int check_single_test_params(char *name, char *testname, char *itname)
138{
139    if (name != NULL) {
140        int i;
141        for (i = 0; i < num_tests; ++i) {
142            if (strcmp(name, all_tests[i].test_case_name) == 0) {
143                single_test = 1 + i;
144                break;
145            }
146        }
147        if (i >= num_tests)
148            single_test = atoi(name);
149    }
150
151
152    /* if only iteration is specified, assume we want the first test */
153    if (single_test == -1 && single_iter != -1)
154        single_test = 1;
155
156    if (single_test != -1) {
157        if (single_test < 1 || single_test > num_tests) {
158            test_printf_stderr("Invalid -%s value "
159                               "(Value must be a valid test name OR a value between %d..%d)\n",
160                               testname, 1, num_tests);
161            return 0;
162        }
163    }
164    if (single_iter != -1) {
165        if (all_tests[single_test - 1].num == -1) {
166            test_printf_stderr("-%s option is not valid for test %d:%s\n",
167                               itname,
168                               single_test,
169                               all_tests[single_test - 1].test_case_name);
170            return 0;
171        } else if (single_iter < 1
172                   || single_iter > all_tests[single_test - 1].num) {
173            test_printf_stderr("Invalid -%s value for test %d:%s\t"
174                               "(Value must be in the range %d..%d)\n",
175                               itname, single_test,
176                               all_tests[single_test - 1].test_case_name,
177                               1, all_tests[single_test - 1].num);
178            return 0;
179        }
180    }
181    return 1;
182}
183
184static int process_shared_options(void)
185{
186    OPTION_CHOICE_DEFAULT o;
187    int value;
188    int ret = -1;
189    char *flag_test = "";
190    char *flag_iter = "";
191    char *testname = NULL;
192
193    opt_begin();
194    while ((o = opt_next()) != OPT_EOF) {
195        switch (o) {
196        /* Ignore any test options at this level */
197        default:
198            break;
199        case OPT_ERR:
200            return ret;
201        case OPT_TEST_HELP:
202            opt_help(test_get_options());
203            return 0;
204        case OPT_TEST_LIST:
205            show_list = 1;
206            break;
207        case OPT_TEST_SINGLE:
208            flag_test = opt_flag();
209            testname = opt_arg();
210            break;
211        case OPT_TEST_ITERATION:
212            flag_iter = opt_flag();
213            if (!opt_int(opt_arg(), &single_iter))
214                goto end;
215            break;
216        case OPT_TEST_INDENT:
217            if (!opt_int(opt_arg(), &value))
218                goto end;
219            level = 4 * value;
220            test_adjust_streams_tap_level(level);
221            break;
222        case OPT_TEST_SEED:
223            if (!opt_int(opt_arg(), &value))
224                goto end;
225            set_seed(value);
226            break;
227        }
228    }
229    if (!check_single_test_params(testname, flag_test, flag_iter))
230        goto end;
231    ret = 1;
232end:
233    return ret;
234}
235
236
237int pulldown_test_framework(int ret)
238{
239    set_test_title(NULL);
240    return ret;
241}
242
243static void finalize(int success)
244{
245    if (success)
246        ERR_clear_error();
247    else
248        ERR_print_errors_cb(openssl_error_cb, NULL);
249}
250
251static char *test_title = NULL;
252
253void set_test_title(const char *title)
254{
255    free(test_title);
256    test_title = title == NULL ? NULL : strdup(title);
257}
258
259PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,
260                                             const char *description, ...)
261{
262    va_list ap;
263
264    test_flush_stdout();
265    test_flush_stderr();
266
267    if (verdict == 0 && seed != 0)
268        test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed);
269    test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok");
270    va_start(ap, description);
271    test_vprintf_tapout(description, ap);
272    va_end(ap);
273    if (verdict == TEST_SKIP_CODE)
274        test_printf_tapout(" # skipped");
275    test_printf_tapout("\n");
276    test_flush_tapout();
277}
278
279int run_tests(const char *test_prog_name)
280{
281    int num_failed = 0;
282    int verdict = 1;
283    int ii, i, jj, j, jstep;
284    int test_case_count = 0;
285    int subtest_case_count = 0;
286    int permute[OSSL_NELEM(all_tests)];
287
288    i = process_shared_options();
289    if (i == 0)
290        return EXIT_SUCCESS;
291    if (i == -1)
292        return EXIT_FAILURE;
293
294    if (num_tests < 1) {
295        test_printf_tapout("1..0 # Skipped: %s\n", test_prog_name);
296    } else if (show_list == 0 && single_test == -1) {
297        if (level > 0) {
298            test_printf_stdout("Subtest: %s\n", test_prog_name);
299            test_flush_stdout();
300        }
301        test_printf_tapout("1..%d\n", num_test_cases);
302    }
303
304    test_flush_tapout();
305
306    for (i = 0; i < num_tests; i++)
307        permute[i] = i;
308    if (rand_order != 0)
309        for (i = num_tests - 1; i >= 1; i--) {
310            j = test_random() % (1 + i);
311            ii = permute[j];
312            permute[j] = permute[i];
313            permute[i] = ii;
314        }
315
316    for (ii = 0; ii != num_tests; ++ii) {
317        i = permute[ii];
318
319        if (single_test != -1 && ((i+1) != single_test)) {
320            continue;
321        }
322        else if (show_list) {
323            if (all_tests[i].num != -1) {
324                test_printf_tapout("%d - %s (%d..%d)\n", ii + 1,
325                                   all_tests[i].test_case_name, 1,
326                                   all_tests[i].num);
327            } else {
328                test_printf_tapout("%d - %s\n", ii + 1,
329                                   all_tests[i].test_case_name);
330            }
331            test_flush_tapout();
332        } else if (all_tests[i].num == -1) {
333            set_test_title(all_tests[i].test_case_name);
334            ERR_clear_error();
335            verdict = all_tests[i].test_fn();
336            finalize(verdict != 0);
337            test_verdict(verdict, "%d - %s", test_case_count + 1, test_title);
338            if (verdict == 0)
339                num_failed++;
340            test_case_count++;
341        } else {
342            verdict = TEST_SKIP_CODE;
343            set_test_title(all_tests[i].test_case_name);
344            if (all_tests[i].subtest) {
345                level += 4;
346                test_adjust_streams_tap_level(level);
347                if (single_iter == -1) {
348                    test_printf_stdout("Subtest: %s\n", test_title);
349                    test_printf_tapout("%d..%d\n", 1, all_tests[i].num);
350                    test_flush_stdout();
351                    test_flush_tapout();
352                }
353            }
354
355            j = -1;
356            if (rand_order == 0 || all_tests[i].num < 3)
357                jstep = 1;
358            else
359                do
360                    jstep = test_random() % all_tests[i].num;
361                while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
362
363            for (jj = 0; jj < all_tests[i].num; jj++) {
364                int v;
365
366                j = (j + jstep) % all_tests[i].num;
367                if (single_iter != -1 && ((jj + 1) != single_iter))
368                    continue;
369                ERR_clear_error();
370                v = all_tests[i].param_test_fn(j);
371
372                if (v == 0) {
373                    verdict = 0;
374                } else if (v != TEST_SKIP_CODE && verdict != 0) {
375                    verdict = 1;
376                }
377
378                finalize(v != 0);
379
380                if (all_tests[i].subtest)
381                    test_verdict(v, "%d - iteration %d",
382                                 subtest_case_count + 1, j + 1);
383                else
384                    test_verdict(v, "%d - %s - iteration %d",
385                                 test_case_count + subtest_case_count + 1,
386                                 test_title, j + 1);
387                subtest_case_count++;
388            }
389
390            if (all_tests[i].subtest) {
391                level -= 4;
392                test_adjust_streams_tap_level(level);
393            }
394            if (verdict == 0)
395                ++num_failed;
396            if (all_tests[i].num == -1 || all_tests[i].subtest)
397                test_verdict(verdict, "%d - %s", test_case_count + 1,
398                             all_tests[i].test_case_name);
399            test_case_count++;
400        }
401    }
402    if (num_failed != 0)
403        return EXIT_FAILURE;
404    return EXIT_SUCCESS;
405}
406
407/*
408 * Glue an array of strings together and return it as an allocated string.
409 * Optionally return the whole length of this string in |out_len|
410 */
411char *glue_strings(const char *list[], size_t *out_len)
412{
413    size_t len = 0;
414    char *p, *ret;
415    int i;
416
417    for (i = 0; list[i] != NULL; i++)
418        len += strlen(list[i]);
419
420    if (out_len != NULL)
421        *out_len = len;
422
423    if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1)))
424        return NULL;
425
426    for (i = 0; list[i] != NULL; i++)
427        p += strlen(strcpy(p, list[i]));
428
429    return ret;
430}
431
432char *test_mk_file_path(const char *dir, const char *file)
433{
434# ifndef OPENSSL_SYS_VMS
435    const char *sep = "/";
436# else
437    const char *sep = "";
438    char *dir_end;
439    char dir_end_sep;
440# endif
441    size_t dirlen = dir != NULL ? strlen(dir) : 0;
442    size_t len = dirlen + strlen(sep) + strlen(file) + 1;
443    char *full_file = OPENSSL_zalloc(len);
444
445    if (full_file != NULL) {
446        if (dir != NULL && dirlen > 0) {
447            OPENSSL_strlcpy(full_file, dir, len);
448# ifdef OPENSSL_SYS_VMS
449            /*
450             * If |file| contains a directory spec, we need to do some
451             * careful merging.
452             * "vol:[dir.dir]" + "[.certs]sm2-root.crt" should become
453             * "vol:[dir.dir.certs]sm2-root.crt"
454             */
455            dir_end = &full_file[strlen(full_file) - 1];
456            dir_end_sep = *dir_end;
457            if ((dir_end_sep == ']' || dir_end_sep == '>')
458                && (file[0] == '[' || file[0] == '<')) {
459                file++;
460                if (file[0] == '.')
461                    *dir_end = '\0';
462                else
463                    *dir_end = '.';
464            }
465#else
466            OPENSSL_strlcat(full_file, sep, len);
467#endif
468        }
469        OPENSSL_strlcat(full_file, file, len);
470    }
471
472    return full_file;
473}
474