1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Cache maintenance
4 *
5 * Copyright (C) 2001 Deep Blue Solutions Ltd.
6 * Copyright (C) 2012 ARM Ltd.
7 */
8
9#include <linux/errno.h>
10#include <linux/linkage.h>
11#include <linux/init.h>
12#include <asm/assembler.h>
13#include <asm/cpufeature.h>
14#include <asm/alternative.h>
15#include <asm/asm-uaccess.h>
16
17/*
18 *	caches_clean_inval_pou_macro(start,end) [fixup]
19 *
20 *	Ensure that the I and D caches are coherent within specified region.
21 *	This is typically used when code has been written to a memory region,
22 *	and will be executed.
23 *
24 *	- start   - virtual start address of region
25 *	- end     - virtual end address of region
26 *	- fixup   - optional label to branch to on user fault
27 */
28.macro	caches_clean_inval_pou_macro, fixup
29alternative_if ARM64_HAS_CACHE_IDC
30	dsb     ishst
31	b       .Ldc_skip_\@
32alternative_else_nop_endif
33	mov     x2, x0
34	mov     x3, x1
35	dcache_by_line_op cvau, ish, x2, x3, x4, x5, \fixup
36.Ldc_skip_\@:
37alternative_if ARM64_HAS_CACHE_DIC
38	isb
39	b	.Lic_skip_\@
40alternative_else_nop_endif
41	invalidate_icache_by_line x0, x1, x2, x3, \fixup
42.Lic_skip_\@:
43.endm
44
45/*
46 *	caches_clean_inval_pou(start,end)
47 *
48 *	Ensure that the I and D caches are coherent within specified region.
49 *	This is typically used when code has been written to a memory region,
50 *	and will be executed.
51 *
52 *	- start   - virtual start address of region
53 *	- end     - virtual end address of region
54 */
55SYM_FUNC_START(caches_clean_inval_pou)
56	caches_clean_inval_pou_macro
57	ret
58SYM_FUNC_END(caches_clean_inval_pou)
59SYM_FUNC_ALIAS(__pi_caches_clean_inval_pou, caches_clean_inval_pou)
60
61/*
62 *	caches_clean_inval_user_pou(start,end)
63 *
64 *	Ensure that the I and D caches are coherent within specified region.
65 *	This is typically used when code has been written to a memory region,
66 *	and will be executed.
67 *
68 *	- start   - virtual start address of region
69 *	- end     - virtual end address of region
70 */
71SYM_FUNC_START(caches_clean_inval_user_pou)
72	uaccess_ttbr0_enable x2, x3, x4
73
74	caches_clean_inval_pou_macro 2f
75	mov	x0, xzr
761:
77	uaccess_ttbr0_disable x1, x2
78	ret
792:
80	mov	x0, #-EFAULT
81	b	1b
82SYM_FUNC_END(caches_clean_inval_user_pou)
83
84/*
85 *	icache_inval_pou(start,end)
86 *
87 *	Ensure that the I cache is invalid within specified region.
88 *
89 *	- start   - virtual start address of region
90 *	- end     - virtual end address of region
91 */
92SYM_FUNC_START(icache_inval_pou)
93alternative_if ARM64_HAS_CACHE_DIC
94	isb
95	ret
96alternative_else_nop_endif
97
98	invalidate_icache_by_line x0, x1, x2, x3
99	ret
100SYM_FUNC_END(icache_inval_pou)
101
102/*
103 *	dcache_clean_inval_poc(start, end)
104 *
105 *	Ensure that any D-cache lines for the interval [start, end)
106 *	are cleaned and invalidated to the PoC.
107 *
108 *	- start   - virtual start address of region
109 *	- end     - virtual end address of region
110 */
111SYM_FUNC_START(__pi_dcache_clean_inval_poc)
112	dcache_by_line_op civac, sy, x0, x1, x2, x3
113	ret
114SYM_FUNC_END(__pi_dcache_clean_inval_poc)
115SYM_FUNC_ALIAS(dcache_clean_inval_poc, __pi_dcache_clean_inval_poc)
116
117/*
118 *	dcache_clean_pou(start, end)
119 *
120 * 	Ensure that any D-cache lines for the interval [start, end)
121 * 	are cleaned to the PoU.
122 *
123 *	- start   - virtual start address of region
124 *	- end     - virtual end address of region
125 */
126SYM_FUNC_START(dcache_clean_pou)
127alternative_if ARM64_HAS_CACHE_IDC
128	dsb	ishst
129	ret
130alternative_else_nop_endif
131	dcache_by_line_op cvau, ish, x0, x1, x2, x3
132	ret
133SYM_FUNC_END(dcache_clean_pou)
134
135/*
136 *	dcache_inval_poc(start, end)
137 *
138 * 	Ensure that any D-cache lines for the interval [start, end)
139 * 	are invalidated. Any partial lines at the ends of the interval are
140 *	also cleaned to PoC to prevent data loss.
141 *
142 *	- start   - kernel start address of region
143 *	- end     - kernel end address of region
144 */
145SYM_FUNC_START(__pi_dcache_inval_poc)
146	dcache_line_size x2, x3
147	sub	x3, x2, #1
148	tst	x1, x3				// end cache line aligned?
149	bic	x1, x1, x3
150	b.eq	1f
151	dc	civac, x1			// clean & invalidate D / U line
1521:	tst	x0, x3				// start cache line aligned?
153	bic	x0, x0, x3
154	b.eq	2f
155	dc	civac, x0			// clean & invalidate D / U line
156	b	3f
1572:	dc	ivac, x0			// invalidate D / U line
1583:	add	x0, x0, x2
159	cmp	x0, x1
160	b.lo	2b
161	dsb	sy
162	ret
163SYM_FUNC_END(__pi_dcache_inval_poc)
164SYM_FUNC_ALIAS(dcache_inval_poc, __pi_dcache_inval_poc)
165
166/*
167 *	dcache_clean_poc(start, end)
168 *
169 * 	Ensure that any D-cache lines for the interval [start, end)
170 * 	are cleaned to the PoC.
171 *
172 *	- start   - virtual start address of region
173 *	- end     - virtual end address of region
174 */
175SYM_FUNC_START(__pi_dcache_clean_poc)
176	dcache_by_line_op cvac, sy, x0, x1, x2, x3
177	ret
178SYM_FUNC_END(__pi_dcache_clean_poc)
179SYM_FUNC_ALIAS(dcache_clean_poc, __pi_dcache_clean_poc)
180
181/*
182 *	dcache_clean_pop(start, end)
183 *
184 * 	Ensure that any D-cache lines for the interval [start, end)
185 * 	are cleaned to the PoP.
186 *
187 *	- start   - virtual start address of region
188 *	- end     - virtual end address of region
189 */
190SYM_FUNC_START(__pi_dcache_clean_pop)
191	alternative_if_not ARM64_HAS_DCPOP
192	b	dcache_clean_poc
193	alternative_else_nop_endif
194	dcache_by_line_op cvap, sy, x0, x1, x2, x3
195	ret
196SYM_FUNC_END(__pi_dcache_clean_pop)
197SYM_FUNC_ALIAS(dcache_clean_pop, __pi_dcache_clean_pop)
198