1#include "ahcid.h"
2#include "test.h"
3
4#include <stdarg.h>
5#include <barrelfish/deferred.h>
6#include <bench/bench.h>
7#include <devif/backends/blk/ahci_devq.h>
8#include <devif/queue_interface.h>
9
10
11static uint64_t finish_counter = 0;
12
13struct dma_mem {
14    lvaddr_t vaddr;         ///< virtual address of the mapped region
15    lpaddr_t paddr;         ///< physical address of the underlying frame
16    uint64_t bytes;         ///< size of the region in bytes
17    uint64_t requested;     ///< requested size of the region in bytes (<= bytes)
18    struct capref frame;    ///< frame capability backing this region
19};
20
21void test_runner(int n, ...)
22{
23    va_list arguments;
24    va_start(arguments, n);
25
26    // sleep for 5 sec
27    barrelfish_usleep(5000*1000);
28
29    for (size_t i=0; i<n; i++) {
30        enum AhciTest test = va_arg(arguments, enum AhciTest);
31        switch (test) {
32            case AhciTest_READ:
33                ahci_perf_sequential(1024*1024*1024, 512, false);
34                ahci_perf_sequential(1024*1024*1024, 1024, false);
35                ahci_perf_sequential(1024*1024*1024, 2048, false);
36                ahci_perf_sequential(1024*1024*1024, 4096, false);
37                ahci_perf_sequential(1024*1024*1024, 8192, false);
38                ahci_perf_sequential(1024*1024*1024, 16384, false);
39                ahci_perf_sequential(1024*1024*1024, 32768, false);
40                ahci_perf_sequential(1024*1024*1024, 65536, false);
41                ahci_perf_sequential(1024*1024*1024, 131072, false);
42                ahci_perf_sequential(1024*1024*1024, 262144, false);
43                ahci_perf_sequential(1024*1024*1024, 524288, false);
44                ahci_perf_sequential(1024*1024*1024, 1048576, false);
45                ahci_perf_sequential(1024*1024*1024, 1048576*2, false);
46                //ahci_perf_sequential(1024*1024*1024, 1048576*4, false); // ERROR: tilsiter1 OOM
47            break;
48
49            case AhciTest_WRITE:
50                ahci_perf_sequential(1024*1024*256, 512, true);
51                ahci_perf_sequential(1024*1024*256, 1024, true);
52                ahci_perf_sequential(1024*1024*256, 2048, true);
53                ahci_perf_sequential(1024*1024*256, 4096, true);
54                ahci_perf_sequential(1024*1024*256, 8192, true);
55                ahci_perf_sequential(1024*1024*256, 16384, true);
56                ahci_perf_sequential(1024*1024*256, 32768, true);
57                ahci_perf_sequential(1024*1024*256, 65536, true);
58                ahci_perf_sequential(1024*1024*256, 131072, true);
59                ahci_perf_sequential(1024*1024*256, 262144, true);
60                ahci_perf_sequential(1024*1024*256, 524288, true);
61                ahci_perf_sequential(1024*1024*256, 1048576, true);
62                ahci_perf_sequential(1024*1024*256, 1048576*2, true);
63                //ahci_perf_sequential(1024*1024*256, 1048576*4, true); // ERROR: tilsiter1 OOM
64            break;
65
66            case AhciTest_VERIFY:
67                ahci_verify_sequential(1024*1024*256, 512);
68                ahci_verify_sequential(1024*1024*256, 1024);
69                ahci_verify_sequential(1024*1024*256, 2048);
70                ahci_verify_sequential(1024*1024*256, 4096);
71                ahci_verify_sequential(1024*1024*256, 8192);
72                ahci_verify_sequential(1024*1024*256, 16384);
73                ahci_verify_sequential(1024*1024*256, 32768);
74                ahci_verify_sequential(1024*1024*256, 65536);
75                ahci_verify_sequential(1024*1024*256, 131072);
76                ahci_verify_sequential(1024*1024*256, 262144);
77                ahci_verify_sequential(1024*1024*256, 524288);
78                ahci_verify_sequential(1024*1024*256, 1048576);
79                ahci_verify_sequential(1024*1024*256, 1048576*2);
80                ahci_verify_sequential(1024*1024*256, 1048576*4);
81            break;
82
83            case AhciTest_BASIC:
84                ahci_simple_test();
85            break;
86
87            default:
88                USER_PANIC("Unknown test?");
89            break;
90        }
91    }
92    // Harness line
93    printf("AHCI testing completed.\n");
94}
95
96static void frame_alloc_identify(size_t size, struct capref *frame,
97                                 struct frame_identity *id)
98{
99    errval_t err;
100    size_t retbytes;
101
102    err = frame_alloc(frame, size, &retbytes);
103    if (err_is_fail(err)) {
104        USER_PANIC_ERR(err, "frame_alloc");
105    }
106
107    err = invoke_frame_identify(*frame, id);
108    if (err_is_fail(err)) {
109        USER_PANIC_ERR(err, "invoke_frame_identify");
110    }
111}
112
113static void wait_for_interrupt(void)
114{
115    errval_t err = event_dispatch(&disk_ws);
116    if (err_is_fail(err)) {
117        USER_PANIC_ERR(err, "error in event_dispatch for wait_for_interrupt");
118    }
119}
120void ahci_simple_test(void)
121{
122    errval_t err;
123    regionid_t region_id = 0;
124    genoffset_t offset = 0;
125    genoffset_t length = 0;
126    genoffset_t valid_data = 0;
127    genoffset_t valid_length = 0;
128
129    // Allocate a buffer:
130    struct dma_mem mem;
131
132    struct capref frame;
133    struct frame_identity id;
134    //void* va;
135
136    err = frame_alloc(&frame, 4096, NULL);
137    if (err_is_fail(err)) {
138        USER_PANIC_ERR(err, "frame alloc");
139    }
140
141    /*
142    err = vspace_map_one_frame_attr(&va, 4096, frame, VREGION_FLAGS_READ_WRITE,
143                                    NULL, NULL);
144    if (err_is_fail(err)) {
145        USER_PANIC_ERR(err, "map frame");
146    }
147    */
148
149    err = invoke_frame_identify(frame, &id);
150    if (err_is_fail(err)) {
151        USER_PANIC_ERR(err, "frame identify");
152    }
153
154    err = devq_register(dq, frame, &region_id);
155    if (err_is_fail(err)) {
156        USER_PANIC_ERR(err, "devq register");
157    }
158
159    uint64_t flags = 0x0;
160    devq_enqueue(dq, region_id, 0, 512, 0, 512, flags);
161    if (err_is_fail(err)) {
162        USER_PANIC_ERR(err, "devq enqueue");
163    }
164
165    do {
166        err = devq_dequeue(dq, &region_id, &offset, &length, &valid_data,
167                           &valid_length, &flags);
168        if (err_is_ok(err)) {
169            break;
170        }
171        if (err_is_fail(err) && err_no(err) != DEVQ_ERR_QUEUE_EMPTY) {
172            USER_PANIC_ERR(err, "devq dequeue");
173        }
174        wait_for_interrupt();
175    } while (err_no(err) == DEVQ_ERR_QUEUE_EMPTY);
176
177    assert (offset == 0);
178    assert (length == 512);
179
180    err = devq_deregister(dq, region_id, &mem.frame);
181    if (err_is_fail(err)) {
182        USER_PANIC_ERR(err, "devq_deregister failed.");
183    }
184
185    printf("[%s]: DONE\n", __FUNCTION__);
186}
187
188static void blocking_dequeue(void* q, regionid_t* region_id,
189                             genoffset_t* offset, genoffset_t* length,
190                             genoffset_t* valid_data, genoffset_t* valid_length)
191{
192    uint64_t flags;
193    errval_t err;
194    do {
195        err = devq_dequeue(q, region_id, offset, length, valid_data,
196                           valid_length, &flags);
197        if (err_is_ok(err)) {
198            break;
199        }
200        if (err_is_fail(err) && err_no(err) != DEVQ_ERR_QUEUE_EMPTY) {
201            USER_PANIC_ERR(err, "devq dequeue");
202        }
203
204        assert(err_no(err) == DEVQ_ERR_QUEUE_EMPTY);
205        wait_for_interrupt();
206    } while (err_no(err) == DEVQ_ERR_QUEUE_EMPTY);
207}
208
209static void receive_block(void)
210{
211    regionid_t rid = 0;
212    genoffset_t offset = 0;
213    genoffset_t length = 0;
214    genoffset_t valid_data = 0;
215    genoffset_t valid_length = 0;
216    blocking_dequeue(dq, &rid, &offset, &length, &valid_data,
217                     &valid_length);
218    finish_counter++;
219}
220
221void ahci_perf_sequential(size_t buffer_size, size_t block_size, bool write)
222{
223    finish_counter = 0;
224    bench_init();
225    errval_t err;
226    assert(buffer_size % block_size == 0);
227
228    size_t read_requests = buffer_size / block_size;
229    regionid_t region_id = 0;
230
231    static struct capref frame;
232    struct frame_identity id;
233    frame_alloc_identify(buffer_size, &frame, &id);
234
235    err = devq_register(dq, frame, &region_id);
236    if (err_is_fail(err)) {
237        USER_PANIC_ERR(err, "devq register");
238    }
239
240    uint64_t write_flag = (write) ? (1ULL << 63) : 0;
241    bufferid_t *received = calloc(1, sizeof(bufferid_t) * read_requests);
242    cycles_t t1 = bench_tsc();
243
244    for (size_t i=0; i < read_requests; i++) {
245        uint64_t disk_block = write_flag | i;
246        do {
247            err = devq_enqueue(dq, region_id, (i*block_size),
248                               block_size, 0,
249                               block_size, disk_block);
250            if (err_is_ok(err)) {
251                break;
252            }
253            else if (err_no(err) == DEVQ_ERR_QUEUE_FULL) {
254                receive_block();
255            }
256            else {
257                USER_PANIC_ERR(err, "Can't receive block.");
258            }
259        } while (true);
260    }
261
262    // Make sure we have all requests:
263    while (finish_counter < read_requests) {
264        receive_block();
265    }
266
267    cycles_t t2 = bench_tsc();
268    cycles_t result = (t2 - t1 - bench_tscoverhead());
269
270    double result_ms = (double)bench_tsc_to_ms(result);
271    double bw = buffer_size / result_ms / 1000; // 1e3 to sec, 10e6 to MB
272    char* cmd = write ? "Write" : "Read";
273    printf("[%s] %s sequential size %zu bs %zu: %.2f [MB/s]\n", __FUNCTION__, cmd, buffer_size, block_size, bw);
274
275    err = devq_deregister(dq, region_id, &frame);
276    if (err_is_fail(err)) {
277        USER_PANIC_ERR(err, "devq_deregister failed.");
278    }
279
280    free(received);
281
282    cap_destroy(frame);
283}
284void ahci_verify_sequential(size_t buffer_size, size_t block_size)
285{
286    finish_counter = 0;
287    bench_init();
288    errval_t err;
289    assert(buffer_size % block_size == 0);
290
291    size_t requests = buffer_size / block_size;
292    regionid_t region_id = 0;
293
294    struct capref frame;
295    struct frame_identity id;
296    frame_alloc_identify(buffer_size, &frame, &id);
297    err = devq_register(dq, frame, &region_id);
298    if (err_is_fail(err)) {
299        USER_PANIC_ERR(err, "devq register");
300    }
301
302    struct capref fcopy;
303    err = slot_alloc(&fcopy);
304    assert(err_is_ok(err));
305    err = cap_copy(fcopy, frame);
306    if (err_is_fail(err)) {
307        USER_PANIC_ERR(err, "copy failed.");
308    }
309    void* retaddr;
310    err = vspace_map_one_frame(&retaddr, id.bytes, fcopy, NULL, NULL);
311    if (err_is_fail(err)) {
312        USER_PANIC_ERR(err, "map copy failed.");
313    }
314
315    uint8_t rbyte = (uint8_t) rdtsc();
316    if (rbyte == 0) rbyte++;
317    memset(retaddr, rbyte, buffer_size);
318
319    uint64_t write_flag = (1ULL << 63);
320    bufferid_t *received = calloc(1, sizeof(bufferid_t) * requests);
321    for (size_t i=0; i < requests; i++) {
322        uint64_t disk_block = write_flag | i;
323        do {
324            err = devq_enqueue(dq, region_id, (i*block_size), block_size,
325                               0, block_size, disk_block);
326            if (err_is_ok(err)) {
327                break;
328            }
329            else if (err_no(err) == DEVQ_ERR_QUEUE_FULL) {
330                receive_block();
331            }
332            else {
333                USER_PANIC_ERR(err, "Can't receive block.");
334            }
335        } while (true);
336    }
337    // Make sure we have all requests:
338    while (finish_counter < requests) {
339        receive_block();
340    }
341
342    memset(retaddr, 0x00, id.bytes);
343    memset((void*)received, 0x0, sizeof(bufferid_t)*requests);
344    finish_counter = 0;
345
346    for (size_t i=0; i < requests; i++) {
347        //printf("%s:%s:%d: i: %zu requests: %zu\n", __FILE__, __FUNCTION__, __LINE__, i, requests);
348        uint64_t disk_block = i;
349        do {
350            err = devq_enqueue(dq, region_id, (i*block_size),
351                               block_size, 0, block_size,
352                               disk_block);
353            if (err_is_ok(err)) {
354                break;
355            }
356            else if (err_no(err) == DEVQ_ERR_QUEUE_FULL) {
357                receive_block();
358            }
359            else {
360                USER_PANIC_ERR(err, "Can't receive block.");
361            }
362        } while (true);
363    }
364
365    // Make sure we have all requests:
366    while (finish_counter < requests) {
367        receive_block();
368    }
369
370    for (size_t i=0; i < buffer_size; i++) {
371        uint8_t* carr = retaddr;
372        if (carr[i] != rbyte) {
373            printf("%s:%s:%d: carr[%zu]=%d != rbyte=%d\n", __FILE__, __FUNCTION__, __LINE__, i, carr[i], rbyte);
374        }
375        assert(carr[i] == rbyte);
376    }
377
378    printf("[%s] SUCCESS (%zu %zu)\n", __FUNCTION__, buffer_size, block_size);
379    cap_destroy(fcopy);
380
381    err = devq_deregister(dq, region_id, &frame);
382    if (err_is_fail(err)) {
383        USER_PANIC_ERR(err, "devq_deregister failed.");
384    }
385}
386