1/*
2 * Copyright 2017, Data61
3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO)
4 * ABN 41 687 119 230.
5 *
6 * This software may be distributed and modified according to the terms of
7 * the BSD 2-Clause license. Note that NO WARRANTY is provided.
8 * See "LICENSE_BSD2.txt" for details.
9 *
10 * @TAG(DATA61_BSD)
11 */
12#pragma once
13
14#include <autoconf.h>
15#include <sel4bench/types.h>
16#include <sel4bench/armv/private.h>
17#include <sel4/sel4.h>
18#include <utils/util.h>
19
20//utility macros
21#define MODIFY_PMCR(op, val) sel4bench_private_write_pmcr(sel4bench_private_read_pmcr() op (val))
22
23#define SEL4BENCH_READ_CCNT(var) PMU_READ(PMCCNTR, var);
24
25#define SEL4BENCH_RESET_CCNT do {\
26    MODIFY_PMCR(| , SEL4BENCH_ARMV8A_PMCR_RESET_CCNT);\
27} while(0)
28
29/* Silence warnings about including the following functions when seL4_DebugRun
30 * is not enabled when we are not calling them. If we actually call these
31 * functions without seL4_DebugRun enabled, we'll get a link failure, so this
32 * should be OK.
33 */
34void seL4_DebugRun(void (* userfn)(void *), void *userarg);
35
36static FASTFN void sel4bench_init()
37{
38    //do kernel-mode PMC init
39#ifndef CONFIG_EXPORT_PMU_USER
40    seL4_DebugRun(&sel4bench_private_init, NULL);
41#endif
42
43    //ensure all counters are in the stopped state
44    sel4bench_private_write_cntenc(-1);
45
46    //Clear div 64 flag
47    MODIFY_PMCR(&, ~SEL4BENCH_ARMV8A_PMCR_DIV64);
48
49    //Reset all counters
50    MODIFY_PMCR( |, SEL4BENCH_ARMV8A_PMCR_RESET_ALL | SEL4BENCH_ARMV8A_PMCR_RESET_CCNT);
51
52    //Enable counters globally.
53    MODIFY_PMCR( |, SEL4BENCH_ARMV8A_PMCR_ENABLE);
54
55#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT
56    // Select instruction count incl. PL2 by default */
57    sel4bench_private_write_pmnxsel(0x1f);
58    sel4bench_private_write_evtsel(BIT(27));
59#endif
60    //start CCNT
61    sel4bench_private_write_cntens(BIT(SEL4BENCH_ARMV8A_COUNTER_CCNT));
62}
63
64static FASTFN void sel4bench_destroy()
65{
66    //stop all performance counters
67    sel4bench_private_write_cntenc(-1);
68
69    //Disable counters globally.
70    MODIFY_PMCR(&, ~SEL4BENCH_ARMV8A_PMCR_ENABLE);
71
72    //disable user-mode performance-counter access
73#ifndef CONFIG_EXPORT_PMU_USER
74    seL4_DebugRun(&sel4bench_private_deinit, NULL);
75#endif
76}
77
78static FASTFN seL4_Word sel4bench_get_num_counters()
79{
80    return SEL4BENCH_ARMV8A_PMCR_N(sel4bench_private_read_pmcr());
81}
82
83static FASTFN ccnt_t sel4bench_get_cycle_count()
84{
85    ccnt_t val;
86    uint32_t enable_word = sel4bench_private_read_cntens(); //store running state
87
88    sel4bench_private_write_cntenc(BIT(SEL4BENCH_ARMV8A_COUNTER_CCNT)); //stop CCNT
89    SEL4BENCH_READ_CCNT(val); //read its value
90    sel4bench_private_write_cntens(enable_word); //start it again if it was running
91
92    return val;
93}
94
95/* being declared FASTFN allows this function (once inlined) to cache miss; I
96 * think it's worthwhile in the general case, for performance reasons.
97 * moreover, it's small enough that it'll be suitably aligned most of the time
98 */
99static FASTFN ccnt_t sel4bench_get_counter(counter_t counter)
100{
101    sel4bench_private_write_pmnxsel(counter); //select the counter on the PMU
102
103    counter = BIT(counter); //from here on in, we operate on a bitfield
104
105    uint32_t enable_word = sel4bench_private_read_cntens();
106
107    sel4bench_private_write_cntenc(counter); //stop the counter
108    uint32_t val = sel4bench_private_read_pmcnt(); //read its value
109    sel4bench_private_write_cntens(enable_word); //start it again if it was running
110
111    return val;
112}
113
114/* this reader function is too complex to be inlined, so we force it to be
115 * cacheline-aligned in order to avoid icache misses with the counters off.
116 * (relevant note: GCC compiles this function to be exactly one ARMV7 cache
117 * line in size) however, the pointer dereference is overwhelmingly likely to
118 * produce a dcache miss, which will occur with the counters off
119 */
120static CACHESENSFN ccnt_t sel4bench_get_counters(counter_bitfield_t mask, ccnt_t *values)
121{
122    //we don't really have time for a NULL or bounds check here
123
124    uint32_t enable_word = sel4bench_private_read_cntens(); //store current running state
125
126    sel4bench_private_write_cntenc(
127        enable_word); //stop running counters (we do this instead of stopping the ones we're interested in because it saves an instruction)
128
129    unsigned int counter = 0;
130    for (; mask != 0; mask >>= 1, counter++) { //for each counter...
131        if (mask & 1) { //... if we care about it...
132            sel4bench_private_write_pmnxsel(counter); //select it,
133            values[counter] = sel4bench_private_read_pmcnt(); //and read its value
134        }
135    }
136
137    ccnt_t ccnt;
138    SEL4BENCH_READ_CCNT(ccnt); //finally, read CCNT
139
140    sel4bench_private_write_cntens(enable_word); //start the counters again
141
142    return ccnt;
143}
144
145static FASTFN void sel4bench_set_count_event(counter_t counter, event_id_t event)
146{
147    sel4bench_private_write_pmnxsel(counter); //select counter
148    sel4bench_private_write_pmcnt(0); //reset it
149    return sel4bench_private_write_evtsel(event); //change the event
150}
151
152static FASTFN void sel4bench_start_counters(counter_bitfield_t mask)
153{
154    /* conveniently, ARM performance counters work exactly like this,
155     * so we just write the value directly to COUNTER_ENABLE_SET
156     */
157    return sel4bench_private_write_cntens(mask);
158}
159
160static FASTFN void sel4bench_stop_counters(counter_bitfield_t mask)
161{
162    /* conveniently, ARM performance counters work exactly like this,
163     * so we just write the value directly to COUNTER_ENABLE_SET
164     * (protecting the CCNT)
165     */
166    return sel4bench_private_write_cntenc(mask & ~BIT(SEL4BENCH_ARMV8A_COUNTER_CCNT));
167}
168
169static FASTFN void sel4bench_reset_counters(void)
170{
171    //Reset all counters except the CCNT
172    MODIFY_PMCR( |, SEL4BENCH_ARMV8A_PMCR_RESET_ALL);
173}
174