1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) 7 * Copyright (C) 2007 MIPS Technologies, Inc. 8 */ 9#include <linux/cacheinfo.h> 10#include <linux/export.h> 11#include <linux/fs.h> 12#include <linux/highmem.h> 13#include <linux/kernel.h> 14#include <linux/linkage.h> 15#include <linux/mm.h> 16#include <linux/sched.h> 17#include <linux/syscalls.h> 18 19#include <asm/bootinfo.h> 20#include <asm/cacheflush.h> 21#include <asm/cpu.h> 22#include <asm/cpu-features.h> 23#include <asm/loongarch.h> 24#include <asm/numa.h> 25#include <asm/processor.h> 26#include <asm/setup.h> 27 28void cache_error_setup(void) 29{ 30 extern char __weak except_vec_cex; 31 set_merr_handler(0x0, &except_vec_cex, 0x80); 32} 33 34/* 35 * LoongArch maintains ICache/DCache coherency by hardware, 36 * we just need "ibar" to avoid instruction hazard here. 37 */ 38void local_flush_icache_range(unsigned long start, unsigned long end) 39{ 40 asm volatile ("\tibar 0\n"::); 41} 42EXPORT_SYMBOL(local_flush_icache_range); 43 44static void flush_cache_leaf(unsigned int leaf) 45{ 46 int i, j, nr_nodes; 47 uint64_t addr = CSR_DMW0_BASE; 48 struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf; 49 50 nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes; 51 52 do { 53 for (i = 0; i < cdesc->sets; i++) { 54 for (j = 0; j < cdesc->ways; j++) { 55 flush_cache_line(leaf, addr); 56 addr++; 57 } 58 59 addr -= cdesc->ways; 60 addr += cdesc->linesz; 61 } 62 addr += (1ULL << NODE_ADDRSPACE_SHIFT); 63 } while (--nr_nodes > 0); 64} 65 66asmlinkage __visible void __flush_cache_all(void) 67{ 68 int leaf; 69 struct cache_desc *cdesc = current_cpu_data.cache_leaves; 70 unsigned int cache_present = current_cpu_data.cache_leaves_present; 71 72 leaf = cache_present - 1; 73 if (cache_inclusive(cdesc + leaf)) { 74 flush_cache_leaf(leaf); 75 return; 76 } 77 78 for (leaf = 0; leaf < cache_present; leaf++) 79 flush_cache_leaf(leaf); 80} 81 82#define L1IUPRE (1 << 0) 83#define L1IUUNIFY (1 << 1) 84#define L1DPRE (1 << 2) 85 86#define LXIUPRE (1 << 0) 87#define LXIUUNIFY (1 << 1) 88#define LXIUPRIV (1 << 2) 89#define LXIUINCL (1 << 3) 90#define LXDPRE (1 << 4) 91#define LXDPRIV (1 << 5) 92#define LXDINCL (1 << 6) 93 94#define populate_cache_properties(cfg0, cdesc, level, leaf) \ 95do { \ 96 unsigned int cfg1; \ 97 \ 98 cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf); \ 99 if (level == 1) { \ 100 cdesc->flags |= CACHE_PRIVATE; \ 101 } else { \ 102 if (cfg0 & LXIUPRIV) \ 103 cdesc->flags |= CACHE_PRIVATE; \ 104 if (cfg0 & LXIUINCL) \ 105 cdesc->flags |= CACHE_INCLUSIVE; \ 106 } \ 107 cdesc->level = level; \ 108 cdesc->flags |= CACHE_PRESENT; \ 109 cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1; \ 110 cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS); \ 111 cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE); \ 112 cdesc++; leaf++; \ 113} while (0) 114 115void cpu_cache_init(void) 116{ 117 unsigned int leaf = 0, level = 1; 118 unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16); 119 struct cache_desc *cdesc = current_cpu_data.cache_leaves; 120 121 if (config & L1IUPRE) { 122 if (config & L1IUUNIFY) 123 cdesc->type = CACHE_TYPE_UNIFIED; 124 else 125 cdesc->type = CACHE_TYPE_INST; 126 populate_cache_properties(config, cdesc, level, leaf); 127 } 128 129 if (config & L1DPRE) { 130 cdesc->type = CACHE_TYPE_DATA; 131 populate_cache_properties(config, cdesc, level, leaf); 132 } 133 134 config = config >> 3; 135 for (level = 2; level <= CACHE_LEVEL_MAX; level++) { 136 if (!config) 137 break; 138 139 if (config & LXIUPRE) { 140 if (config & LXIUUNIFY) 141 cdesc->type = CACHE_TYPE_UNIFIED; 142 else 143 cdesc->type = CACHE_TYPE_INST; 144 populate_cache_properties(config, cdesc, level, leaf); 145 } 146 147 if (config & LXDPRE) { 148 cdesc->type = CACHE_TYPE_DATA; 149 populate_cache_properties(config, cdesc, level, leaf); 150 } 151 152 config = config >> 7; 153 } 154 155 BUG_ON(leaf > CACHE_LEAVES_MAX); 156 157 current_cpu_data.cache_leaves_present = leaf; 158 current_cpu_data.options |= LOONGARCH_CPU_PREFETCH; 159} 160 161static const pgprot_t protection_map[16] = { 162 [VM_NONE] = __pgprot(_CACHE_CC | _PAGE_USER | 163 _PAGE_PROTNONE | _PAGE_NO_EXEC | 164 _PAGE_NO_READ), 165 [VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 166 _PAGE_USER | _PAGE_PRESENT | 167 _PAGE_NO_EXEC), 168 [VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 169 _PAGE_USER | _PAGE_PRESENT | 170 _PAGE_NO_EXEC), 171 [VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 172 _PAGE_USER | _PAGE_PRESENT | 173 _PAGE_NO_EXEC), 174 [VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID | 175 _PAGE_USER | _PAGE_PRESENT), 176 [VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 177 _PAGE_USER | _PAGE_PRESENT), 178 [VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 179 _PAGE_USER | _PAGE_PRESENT), 180 [VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 181 _PAGE_USER | _PAGE_PRESENT), 182 [VM_SHARED] = __pgprot(_CACHE_CC | _PAGE_USER | 183 _PAGE_PROTNONE | _PAGE_NO_EXEC | 184 _PAGE_NO_READ), 185 [VM_SHARED | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 186 _PAGE_USER | _PAGE_PRESENT | 187 _PAGE_NO_EXEC), 188 [VM_SHARED | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 189 _PAGE_USER | _PAGE_PRESENT | 190 _PAGE_NO_EXEC | _PAGE_WRITE), 191 [VM_SHARED | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 192 _PAGE_USER | _PAGE_PRESENT | 193 _PAGE_NO_EXEC | _PAGE_WRITE), 194 [VM_SHARED | VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID | 195 _PAGE_USER | _PAGE_PRESENT), 196 [VM_SHARED | VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 197 _PAGE_USER | _PAGE_PRESENT), 198 [VM_SHARED | VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 199 _PAGE_USER | _PAGE_PRESENT | 200 _PAGE_WRITE), 201 [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 202 _PAGE_USER | _PAGE_PRESENT | 203 _PAGE_WRITE) 204}; 205DECLARE_VM_GET_PAGE_PROT 206