1/*
2 * Copyright 2017, Data61
3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO)
4 * ABN 41 687 119 230.
5 *
6 * This software may be distributed and modified according to the terms of
7 * the BSD 2-Clause license. Note that NO WARRANTY is provided.
8 * See "LICENSE_BSD2.txt" for details.
9 *
10 * @TAG(DATA61_BSD)
11 */
12#pragma once
13
14#include <autoconf.h>
15#include <sel4bench/types.h>
16#include <sel4bench/armv/private.h>
17#include <sel4/sel4.h>
18#include <utils/util.h>
19
20#define SEL4BENCH_READ_CCNT(var) do { \
21    asm volatile("mrc p15, 0, %0, c9, c13, 0\n" \
22        : "=r"(var) \
23    ); \
24} while(0)
25
26//utility macros
27#define MODIFY_PMCR(op, val) sel4bench_private_write_pmcr(sel4bench_private_read_pmcr() op (val))
28
29#define SEL4BENCH_RESET_CCNT do {\
30    MODIFY_PMCR(| , SEL4BENCH_ARMV7A_PMCR_RESET_CCNT);\
31} while(0)
32
33/* Silence warnings about including the following functions when seL4_DebugRun
34 * is not enabled when we are not calling them. If we actually call these
35 * functions without seL4_DebugRun enabled, we'll get a link failure, so this
36 * should be OK.
37 */
38void seL4_DebugRun(void (* userfn)(void *), void *userarg);
39
40static FASTFN void sel4bench_init()
41{
42    //do kernel-mode PMC init
43#ifndef CONFIG_EXPORT_PMU_USER
44    seL4_DebugRun(&sel4bench_private_init, NULL);
45#endif
46
47    //ensure all counters are in the stopped state
48    sel4bench_private_write_cntenc(-1);
49
50    //Clear div 64 flag
51    MODIFY_PMCR(&, ~SEL4BENCH_ARMV7A_PMCR_DIV64);
52
53    //Reset all counters
54    MODIFY_PMCR( |, SEL4BENCH_ARMV7A_PMCR_RESET_ALL | SEL4BENCH_ARMV7A_PMCR_RESET_CCNT);
55
56    //Enable counters globally.
57    MODIFY_PMCR( |, SEL4BENCH_ARMV7A_PMCR_ENABLE);
58
59#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT
60    // Select instruction count incl. PL2 by default */
61    sel4bench_private_write_pmnxsel(0x1f);
62    sel4bench_private_write_evtsel(BIT(27));
63#endif
64    //start CCNT
65    sel4bench_private_write_cntens(BIT(SEL4BENCH_ARMV7A_COUNTER_CCNT));
66}
67
68static FASTFN void sel4bench_destroy()
69{
70    //stop all performance counters
71    sel4bench_private_write_cntenc(-1);
72
73    //Disable counters globally.
74    MODIFY_PMCR(&, ~SEL4BENCH_ARMV7A_PMCR_ENABLE);
75
76    //disable user-mode performance-counter access
77#ifndef CONFIG_EXPORT_PMU_USER
78    seL4_DebugRun(&sel4bench_private_deinit, NULL);
79#endif
80}
81
82static FASTFN seL4_Word sel4bench_get_num_counters()
83{
84#ifdef CORTEX_A8
85    return 4;
86#else //CORTEX_A8
87    return SEL4BENCH_ARMV7A_PMCR_N(sel4bench_private_read_pmcr());
88#endif //CORTEX_A8
89}
90
91static FASTFN ccnt_t sel4bench_get_cycle_count()
92{
93    ccnt_t val;
94    uint32_t enable_word = sel4bench_private_read_cntens(); //store running state
95
96    sel4bench_private_write_cntenc(BIT(SEL4BENCH_ARMV7A_COUNTER_CCNT)); //stop CCNT
97    SEL4BENCH_READ_CCNT(val); //read its value
98    sel4bench_private_write_cntens(enable_word); //start it again if it was running
99
100    return val;
101}
102
103/* being declared FASTFN allows this function (once inlined) to cache miss; I
104 * think it's worthwhile in the general case, for performance reasons.
105 * moreover, it's small enough that it'll be suitably aligned most of the time
106 */
107static FASTFN ccnt_t sel4bench_get_counter(counter_t counter)
108{
109    sel4bench_private_write_pmnxsel(counter); //select the counter on the PMU
110
111    counter = BIT(counter); //from here on in, we operate on a bitfield
112
113    uint32_t enable_word = sel4bench_private_read_cntens();
114
115    sel4bench_private_write_cntenc(counter); //stop the counter
116    uint32_t val = sel4bench_private_read_pmcnt(); //read its value
117    sel4bench_private_write_cntens(enable_word); //start it again if it was running
118
119    return val;
120}
121
122/* this reader function is too complex to be inlined, so we force it to be
123 * cacheline-aligned in order to avoid icache misses with the counters off.
124 * (relevant note: GCC compiles this function to be exactly one ARMV7 cache
125 * line in size) however, the pointer dereference is overwhelmingly likely to
126 * produce a dcache miss, which will occur with the counters off
127 */
128static CACHESENSFN ccnt_t sel4bench_get_counters(counter_bitfield_t mask, ccnt_t *values)
129{
130    //we don't really have time for a NULL or bounds check here
131
132    uint32_t enable_word = sel4bench_private_read_cntens(); //store current running state
133
134    sel4bench_private_write_cntenc(
135        enable_word); //stop running counters (we do this instead of stopping the ones we're interested in because it saves an instruction)
136
137    unsigned int counter = 0;
138    for (; mask != 0; mask >>= 1, counter++) { //for each counter...
139        if (mask & 1) { //... if we care about it...
140            sel4bench_private_write_pmnxsel(counter); //select it,
141            values[counter] = sel4bench_private_read_pmcnt(); //and read its value
142        }
143    }
144
145    ccnt_t ccnt;
146    SEL4BENCH_READ_CCNT(ccnt); //finally, read CCNT
147
148    sel4bench_private_write_cntens(enable_word); //start the counters again
149
150    return ccnt;
151}
152
153static FASTFN void sel4bench_set_count_event(counter_t counter, event_id_t event)
154{
155    sel4bench_private_write_pmnxsel(counter); //select counter
156    sel4bench_private_write_pmcnt(0); //reset it
157    return sel4bench_private_write_evtsel(event); //change the event
158}
159
160static FASTFN void sel4bench_start_counters(counter_bitfield_t mask)
161{
162    /* conveniently, ARM performance counters work exactly like this,
163     * so we just write the value directly to COUNTER_ENABLE_SET
164     */
165    return sel4bench_private_write_cntens(mask);
166}
167
168static FASTFN void sel4bench_stop_counters(counter_bitfield_t mask)
169{
170    /* conveniently, ARM performance counters work exactly like this,
171     * so we just write the value directly to COUNTER_ENABLE_SET
172     * (protecting the CCNT)
173     */
174    return sel4bench_private_write_cntenc(mask & ~BIT(SEL4BENCH_ARMV7A_COUNTER_CCNT));
175}
176
177static FASTFN void sel4bench_reset_counters(void)
178{
179    //Reset all counters except the CCNT
180    MODIFY_PMCR( |, SEL4BENCH_ARMV7A_PMCR_RESET_ALL | SEL4BENCH_ARMV7A_PMCR_RESET_CCNT);
181}
182