1/*
2 * Copyright (c) 2016 ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Universitaetstr 6, CH-8092 Zurich. Attn: Systems Group.
8 */
9
10#ifndef __ARMV7_CACHE_H
11#define __ARMV7_CACHE_H
12
13#include <bitmacros.h>
14#include <cp15.h>
15#include <stdio.h>
16
17/* The cache level at which the instruction cache, data cache and table walks
18 * are unified, from a single processor. */
19static inline size_t
20cache_get_louu(void) {
21    uint32_t clidr= cp15_read_clidr();
22    return (clidr >> 27) & MASK(3);
23}
24
25enum armv7_cache_type {
26    armv7_cache_none    = 0x0,
27    armv7_cache_i       = 0x1,
28    armv7_cache_d       = 0x2,
29    armv7_cache_id      = 0x3,
30    armv7_cache_unified = 0x4,
31};
32
33static inline enum armv7_cache_type
34cache_get_type(size_t level) {
35    assert(level >= 1 && level <= 7);
36
37    uint32_t clidr= cp15_read_clidr();
38
39    /* Each cache level has a 3-bit field with LSB at 3*(n-1). */
40    return (clidr >> (3 * (level-1))) & MASK(3);
41}
42
43static inline uint32_t
44cache_get_ccsidr(size_t level, int icache) {
45    assert(level >= 1 && level <= 7);
46
47    /* Select the cache level and type. */
48    cp15_write_csselr((((level - 1) & MASK(3)) << 1) | (icache & MASK(1)));
49
50    /* Read the size register. */
51    return cp15_read_ccsidr();
52}
53
54/* Count set bits. */
55static inline size_t
56csb(uint32_t x) {
57    return 32 - __builtin_clz(x);
58}
59
60/* Invalidate an entire data cache. */
61static inline void
62invalidate_data_cache(size_t level, bool clean) {
63    assert(level >= 1 && level <= 7);
64
65    enum armv7_cache_type type= cache_get_type(level);
66
67    /* If there's no data cache at this level, there's nothing to do. */
68    if(type == armv7_cache_none || type == armv7_cache_i) return;
69
70    /* Get the details of the data cache at this level. See TRM B4.1.19. */
71    uint32_t ccsidr= cache_get_ccsidr(level, 0);
72    size_t L= (ccsidr & MASK(3)) + 2;       /* log2(LINELEN) */
73
74    size_t x=    (ccsidr >>  3) & MASK(10); /* WAYS - 1 */
75    size_t A=    csb(x);                    /* ceil(log2(WAYS)) */
76    size_t ways= x + 1;
77
78    size_t y=    (ccsidr >> 13) & MASK(15); /* SETS - 1 */
79    size_t S=    csb(x);                    /* ceil(log2(SETS)) */
80    size_t sets= y + 1;
81
82    /* Calculate the set/way register format. See TRM B4.2.1. */
83    size_t w_shift= 32 - A;
84    size_t s_shift= L + S;
85
86    for(size_t w= 0; w < ways; w++) {
87        for(size_t s= 0; s < sets; s++) {
88            uint32_t wsl= (w << w_shift) |
89                          (s << s_shift) |
90                          ((level - 1) << 1);
91            if(clean) cp15_write_dccisw(wsl);
92            else      cp15_write_dcisw(wsl);
93        }
94    }
95}
96
97/* Invalidate (and possibly clean) all data caches to point of unification. */
98static inline void
99invalidate_data_caches_pouu(bool clean) {
100    size_t louu= cache_get_louu();
101
102    for(size_t level= 1; level <= louu; level++)
103        invalidate_data_cache(level, clean);
104}
105
106/* Invalidate this core's instruction cache. */
107static inline void
108invalidate_instruction_cache(void) {
109    cp15_write_iciallu(0); /* The argument is ignored. */
110}
111
112/* Invalidate all TLBs on this core. */
113static inline void
114invalidate_tlb(void) {
115    cp15_write_tlbiall(0); /* The argument is ignored. */
116
117    /* Ensure the invalidate has completed. */
118    dsb();
119
120    /* Ensure that the invalidate completes in program order. */
121    isb();
122}
123
124/* Clean a cache line to point of unification - for table walks and
125 * instruction fetches.  Takes a virtual address. */
126static inline void
127clean_to_pou(void *addr) {
128    cp15_write_dccmvau((uint32_t)addr);
129}
130
131/* Clean a cache line to point of coherency. */
132static inline void
133clean_to_poc(void *addr) {
134    cp15_write_dccmvac((uint32_t)addr);
135}
136
137/* Invalidate a cache line to point of coherency. */
138static inline void
139invalidate_to_poc(void *addr) {
140    cp15_write_dcimvac((uint32_t)addr);
141}
142
143enum armv7_cache_range_op {
144    CLEAN_TO_POU,
145    CLEAN_TO_POC,
146    INVALIDATE_TO_POC,
147};
148
149static inline void
150cache_range_op(void *start, void *end, enum armv7_cache_range_op op) {
151    for(;start < end; start++) {
152        switch(op) {
153            case CLEAN_TO_POC:
154                clean_to_poc(start);
155                break;
156            case CLEAN_TO_POU:
157                clean_to_pou(start);
158                break;
159            case INVALIDATE_TO_POC:
160                invalidate_to_poc(start);
161                break;
162            default:
163                panic("Invalid cache operation.\n");
164                break;
165        }
166    }
167}
168
169/* Clean and invalidate a cache line to point of coherency - for communicating
170 * with other cores.  Takes a virtual address. */
171static inline void
172clean_invalidate_to_poc(void *addr) {
173    cp15_write_dccimvac((uint32_t)addr);
174}
175
176#endif /* __ARMV7_CACHE_H */
177