1/*
2 * Copyright (c) 2014, ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group.
8 */
9#define _GNU_SOURCE
10
11#include <stdlib.h>
12#include <stdio.h>
13#include <stdint.h>
14#include <string.h>
15#include <assert.h>
16#include <stdbool.h>
17#include <sys/time.h>
18
19#include <sched.h> /* CPU_SET et. al. */
20#include <pthread.h>
21#include <pthread_np.h>
22#include <sys/cpuset.h>
23#ifdef BARRELFISH
24#include <barrelfish/barrelfish.h>
25#include <octopus/octopus.h>
26#else
27#define MAX_COREID 1024
28#define USER_PANIC_ERR(x...)
29#define USER_PANIC(x...)
30#define err_is_fail(x) false
31typedef size_t errval_t;
32#endif
33
34#define NUM_INCR 1000000
35static size_t sanity = 0;
36static size_t cpu_count = 4;
37
38static void* prj_thread(void* param)
39{
40#if BARRELFISH
41    printf("Hello from thread %"PRIuPTR" on core %"PRIuCOREID"\n",
42           (uintptr_t)param, disp_get_core_id());
43#endif
44    sanity = 1;
45    //messages_handler_loop();
46    return 0;
47}
48
49static int pthread_create_join_test(void) {
50    pthread_t tid;
51    pthread_attr_t attr;
52    pthread_attr_init(&attr);
53
54    for (size_t rep = 0; rep < 20; rep++) {
55        printf("Create a new thread rep=%zu\n", rep);
56        int rv = pthread_create(&tid, &attr, prj_thread, NULL);
57        if (rv){
58            printf("[ERROR] return code from pthread_create() is %d\n", rv);
59            return 1;
60        }
61
62        /* wait for threads to finish */
63        pthread_join(tid, NULL);
64
65        assert(sanity == 1);
66        sanity = 0;
67        printf("Joined thread.\n");
68    }
69
70    printf("%s PASS\n", __FUNCTION__);
71    return 0;
72}
73
74static int pthread_setaffinity_test(void) {
75
76    pthread_t tid;
77    pthread_attr_t attr;
78    pthread_attr_init(&attr);
79    cpuset_t set;
80
81    for (size_t rep = 0; rep < cpu_count; rep++) {
82        printf("Create a new thread on core %zu\n", rep);
83        sanity = 0;
84
85        CPU_ZERO(&set);
86        CPU_SET(rep, &set);
87        pthread_attr_setaffinity_np(&attr, sizeof(cpuset_t), &set);
88        int rv = pthread_create(&tid, &attr, prj_thread, NULL);
89        if (rv){
90            printf("[ERROR] return code from pthread_create() is %d\n", rv);
91            return 1;
92        }
93
94        /* wait for threads to finish */
95        pthread_join(tid, NULL);
96        //assert(sanity == 1);
97    }
98
99    printf("%s PASS\n", __FUNCTION__);
100    return 0;
101}
102
103static pthread_mutex_t lock;
104
105static size_t atomic_counter;
106static void* mutex_increment(void* param)
107{
108    struct timeval start;
109    struct timeval end;
110
111    gettimeofday(&start, NULL);
112    for (size_t i=0; i < NUM_INCR; i++) {
113        pthread_mutex_lock(&lock);
114        if (atomic_counter % 10000 == 0) {
115            debug_printf("progress\n");
116        }
117        atomic_counter++;
118        pthread_mutex_unlock(&lock);
119    }
120    gettimeofday(&end, NULL);
121
122    assert(atomic_counter >= NUM_INCR);
123    double diff_usec = (((end).tv_sec*1000000L + (end).tv_usec) - ((start).tv_sec*1000000L+(start).tv_usec));
124
125    debug_printf("%s:%s:%d: end-start [usec] = %.4lf\n",
126                 __FILE__, __FUNCTION__, __LINE__, diff_usec);
127
128    return 0;
129}
130
131static int pthread_mutex_performance(void) {
132    pthread_t tid[cpu_count];
133    pthread_attr_t attr;
134    pthread_attr_init(&attr);
135    pthread_mutex_init(&lock, NULL);
136    cpuset_t set;
137
138    for (size_t rep = 0; rep < cpu_count; rep++) {
139        printf("Create a new thread on core %zu\n", rep);
140        CPU_ZERO(&set);
141        CPU_SET(rep, &set);
142        pthread_attr_setaffinity_np(&attr, sizeof(cpuset_t), &set);
143        int rv = pthread_create(&tid[rep], &attr, mutex_increment, NULL);
144        if (rv){
145            printf("[ERROR] return code from pthread_create() is %d\n", rv);
146            return 1;
147        }
148    }
149    for (size_t rep = 0; rep < cpu_count; rep++) {
150        printf("%s:%s:%d: waiting for %zu\n", __FILE__, __FUNCTION__, __LINE__, rep);
151        pthread_join(tid[rep], NULL);
152    }
153    assert(atomic_counter == cpu_count*NUM_INCR);
154
155
156    printf("%s PASS\n", __FUNCTION__);
157
158    return 0;
159}
160
161static bool is_spanned[MAX_COREID] = { false };
162
163static void domain_spanned_callback(void *arg, errval_t err)
164{
165    if (err_is_fail(err)) {
166        USER_PANIC_ERR(err, "domain spanning failed.");
167    }
168    uintptr_t i = (uintptr_t) arg;
169    is_spanned[i] = true;
170}
171
172int main(int argc, char** argv)
173{
174    int r = 0;
175#ifdef BARRELFISH
176    oct_init();
177    static char* local_apics = "r'hw\\.processor\\.[0-9]+' { enabled: 1 }";
178    char** names;
179    errval_t err = oct_get_names(&names, &cpu_count, local_apics);
180    if (err_is_fail(err)) {
181        DEBUG_ERR(err, "Can not get core count");
182        return 1;
183    }
184    oct_free_names(names, cpu_count);
185
186    // Spawn to other cores:
187    for (size_t rep = 0; rep < cpu_count; rep++) {
188        if (rep != disp_get_core_id()) {
189            err = domain_new_dispatcher(rep, domain_spanned_callback, (void*)(uintptr_t)rep);
190            if (err_is_fail(err)) {
191                USER_PANIC_ERR(err, "domain_new_dispatcher failed");
192            }
193
194            while (!is_spanned[rep]) {
195                thread_yield();
196            }
197        }
198    }
199#endif
200
201    r = pthread_mutex_performance();
202    if (r != 0) {
203        USER_PANIC("pthread_mutex_performance failed");
204    }
205
206    r = pthread_setaffinity_test();
207    if (r != 0) {
208        USER_PANIC("pthread_setaffinity_test failed");
209    }
210
211    r = pthread_create_join_test();
212    if (r != 0) {
213        USER_PANIC("pthread_create_join_test failed");
214    }
215
216    printf("TESTS PASSED\n");
217
218#ifdef BARRELFISH
219    // XXX: threads on remote cores page-fault if parent exits :(
220    messages_handler_loop();
221#endif
222    return 0;
223}