1/*	$OpenBSD: cache_mips64r2.c,v 1.4 2022/08/29 02:08:13 jsg Exp $	*/
2
3/*
4 * Copyright (c) 2014 Miodrag Vallat.
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19/*
20 * Cache handling code for mips64r2 compatible processors
21 */
22
23#include <sys/param.h>
24#include <sys/systm.h>
25
26#include <mips64/cache.h>
27#include <mips64/mips_cpu.h>
28#include <machine/cpu.h>
29
30#include <uvm/uvm_extern.h>
31
32#define	IndexInvalidate_I	0x00
33#define	IndexWBInvalidate_D	0x01
34#define	IndexWBInvalidate_T	0x02
35#define	IndexWBInvalidate_S	0x03
36
37#define	HitInvalidate_D		0x11
38#define	HitInvalidate_T		0x12
39#define	HitInvalidate_S		0x13
40
41#define	HitWBInvalidate_D	0x15
42#define	HitWBInvalidate_T	0x16
43#define	HitWBInvalidate_S	0x17
44
45#define	cache(op,addr) \
46    __asm__ volatile \
47      ("cache %0, 0(%1)" :: "i"(op), "r"(addr) : "memory")
48
49static __inline__ void	mips64r2_hitinv_primary(vaddr_t, vsize_t, vsize_t);
50static __inline__ void	mips64r2_hitinv_secondary(vaddr_t, vsize_t, vsize_t);
51static __inline__ void	mips64r2_hitinv_ternary(vaddr_t, vsize_t, vsize_t);
52static __inline__ void	mips64r2_hitwbinv_primary(vaddr_t, vsize_t, vsize_t);
53static __inline__ void	mips64r2_hitwbinv_secondary(vaddr_t, vsize_t, vsize_t);
54static __inline__ void	mips64r2_hitwbinv_ternary(vaddr_t, vsize_t, vsize_t);
55
56void
57mips64r2_ConfigCache(struct cpu_info *ci)
58{
59	uint32_t cfg, valias_mask;
60	uint32_t s, l, a;
61
62	cfg = cp0_get_config();
63	if ((cfg & 0x80000000) == 0)
64		panic("no M bit in cfg0.0");
65
66	cfg = cp0_get_config_1();
67
68	a = 1 + ((cfg & CONFIG1_DA) >> CONFIG1_DA_SHIFT);
69	l = (cfg & CONFIG1_DL) >> CONFIG1_DL_SHIFT;
70	s = (cfg & CONFIG1_DS) >> CONFIG1_DS_SHIFT;
71	ci->ci_l1data.linesize = 2 << l;
72	ci->ci_l1data.setsize = (64 << s) * ci->ci_l1data.linesize;
73	ci->ci_l1data.sets = a;
74	ci->ci_l1data.size = ci->ci_l1data.sets * ci->ci_l1data.setsize;
75
76	a = 1 + ((cfg & CONFIG1_IA) >> CONFIG1_IA_SHIFT);
77	l = (cfg & CONFIG1_IL) >> CONFIG1_IL_SHIFT;
78	s = (cfg & CONFIG1_IS) >> CONFIG1_IS_SHIFT;
79	ci->ci_l1inst.linesize = 2 << l;
80	ci->ci_l1inst.setsize = (64 << s) * ci->ci_l1inst.linesize;
81	ci->ci_l1inst.sets = a;
82	ci->ci_l1inst.size = ci->ci_l1inst.sets * ci->ci_l1inst.setsize;
83
84	memset(&ci->ci_l2, 0, sizeof(struct cache_info));
85	memset(&ci->ci_l3, 0, sizeof(struct cache_info));
86
87	if ((cfg & 0x80000000) != 0) {
88		cfg = cp0_get_config_2();
89
90		a = 1 + ((cfg >> 0) & 0x0f);
91		l = (cfg >> 4) & 0x0f;
92		s = (cfg >> 8) & 0x0f;
93		if (l != 0) {
94			ci->ci_l2.linesize = 2 << l;
95			ci->ci_l2.setsize = (64 << s) * ci->ci_l2.linesize;
96			ci->ci_l2.sets = a;
97			ci->ci_l2.size = ci->ci_l2.sets * ci->ci_l2.setsize;
98		}
99
100		a = 1 + ((cfg >> 16) & 0x0f);
101		l = (cfg >> 20) & 0x0f;
102		s = (cfg >> 24) & 0x0f;
103		if (l != 0) {
104			ci->ci_l3.linesize = 2 << l;
105			ci->ci_l3.setsize = (64 << s) * ci->ci_l3.linesize;
106			ci->ci_l3.sets = a;
107			ci->ci_l3.size = ci->ci_l3.sets * ci->ci_l3.setsize;
108		}
109	}
110
111	valias_mask = (max(ci->ci_l1inst.setsize, ci->ci_l1data.setsize) - 1) &
112	    ~PAGE_MASK;
113
114	if (valias_mask != 0) {
115		valias_mask |= PAGE_MASK;
116#ifdef MULTIPROCESSOR
117		if (valias_mask > cache_valias_mask) {
118#endif
119			cache_valias_mask = valias_mask;
120			pmap_prefer_mask = valias_mask;
121#ifdef MULTIPROCESSOR
122		}
123#endif
124	}
125
126	ci->ci_SyncCache = mips64r2_SyncCache;
127	ci->ci_InvalidateICache = mips64r2_InvalidateICache;
128	ci->ci_InvalidateICachePage = mips64r2_InvalidateICachePage;
129	ci->ci_SyncICache = mips64r2_SyncICache;
130	ci->ci_SyncDCachePage = mips64r2_SyncDCachePage;
131	ci->ci_HitSyncDCachePage = mips64r2_HitSyncDCachePage;
132	ci->ci_HitSyncDCache = mips64r2_HitSyncDCache;
133	ci->ci_HitInvalidateDCache = mips64r2_HitInvalidateDCache;
134	ci->ci_IOSyncDCache = mips64r2_IOSyncDCache;
135}
136
137static __inline__ void
138mips64r2_hitwbinv_primary(vaddr_t va, vsize_t sz, vsize_t line)
139{
140	vaddr_t eva;
141
142	eva = va + sz;
143	while (va != eva) {
144		cache(HitWBInvalidate_D, va);
145		va += line;
146	}
147}
148
149static __inline__ void
150mips64r2_hitwbinv_secondary(vaddr_t va, vsize_t sz, vsize_t line)
151{
152	vaddr_t eva;
153
154	eva = va + sz;
155	while (va != eva) {
156		cache(HitWBInvalidate_S, va);
157		va += line;
158	}
159}
160
161static __inline__ void
162mips64r2_hitwbinv_ternary(vaddr_t va, vsize_t sz, vsize_t line)
163{
164	vaddr_t eva;
165
166	eva = va + sz;
167	while (va != eva) {
168		cache(HitWBInvalidate_T, va);
169		va += line;
170	}
171}
172
173static __inline__ void
174mips64r2_hitinv_primary(vaddr_t va, vsize_t sz, vsize_t line)
175{
176	vaddr_t eva;
177
178	eva = va + sz;
179	while (va != eva) {
180		cache(HitInvalidate_D, va);
181		va += line;
182	}
183}
184
185static __inline__ void
186mips64r2_hitinv_secondary(vaddr_t va, vsize_t sz, vsize_t line)
187{
188	vaddr_t eva;
189
190	eva = va + sz;
191	while (va != eva) {
192		cache(HitInvalidate_S, va);
193		va += line;
194	}
195}
196
197static __inline__ void
198mips64r2_hitinv_ternary(vaddr_t va, vsize_t sz, vsize_t line)
199{
200	vaddr_t eva;
201
202	eva = va + sz;
203	while (va != eva) {
204		cache(HitInvalidate_T, va);
205		va += line;
206	}
207}
208
209/*
210 * Writeback and invalidate all caches.
211 */
212void
213mips64r2_SyncCache(struct cpu_info *ci)
214{
215	vaddr_t sva, eva;
216
217	sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
218	eva = sva + ci->ci_l1inst.linesize;
219	while (sva != eva) {
220		cache(IndexInvalidate_I, sva);
221		sva += ci->ci_l1inst.linesize;
222	}
223
224	sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
225	eva = sva + ci->ci_l1data.linesize;
226	while (sva != eva) {
227		cache(IndexWBInvalidate_D, sva);
228		sva += ci->ci_l1data.linesize;
229	}
230
231	if (ci->ci_l2.size != 0) {
232		sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
233		eva = sva + ci->ci_l2.size;
234		while (sva != eva) {
235			cache(IndexWBInvalidate_S, sva);
236			sva += ci->ci_l2.linesize;
237		}
238	}
239
240	if (ci->ci_l3.size != 0) {
241		sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
242		eva = sva + ci->ci_l3.size;
243		while (sva != eva) {
244			cache(IndexWBInvalidate_T, sva);
245			sva += ci->ci_l3.linesize;
246		}
247	}
248}
249
250/*
251 * Invalidate I$ for the given range.
252 */
253void
254mips64r2_InvalidateICache(struct cpu_info *ci, vaddr_t _va, size_t _sz)
255{
256	vaddr_t va, sva, eva, iva;
257	vsize_t sz, offs;
258	uint set, nsets;
259
260	/* extend the range to integral cache lines */
261	va = _va & ~(ci->ci_l1inst.linesize - 1);
262	sz = ((_va + _sz + ci->ci_l1inst.linesize - 1) & ~(ci->ci_l1inst.linesize - 1)) - va;
263
264	sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
265	offs = ci->ci_l1inst.setsize;
266	nsets = ci->ci_l1inst.sets;
267	/* keep only the index bits */
268	sva |= va & (offs - 1);
269	eva = sva + sz;
270
271	while (sva != eva) {
272		for (set = nsets, iva = sva; set != 0; set--, iva += offs)
273			cache(IndexInvalidate_I, iva);
274		sva += ci->ci_l1inst.linesize;
275	}
276}
277
278/*
279 * Register a given page for I$ invalidation.
280 */
281void
282mips64r2_InvalidateICachePage(struct cpu_info *ci, vaddr_t va)
283{
284	/* this code is too generic to allow for lazy I$ invalidates, yet */
285	mips64r2_InvalidateICache(ci, va, PAGE_SIZE);
286}
287
288/*
289 * Perform postponed I$ invalidation.
290 */
291void
292mips64r2_SyncICache(struct cpu_info *ci)
293{
294}
295
296/*
297 * Writeback D$ for the given page.
298 */
299void
300mips64r2_SyncDCachePage(struct cpu_info *ci, vaddr_t va, paddr_t pa)
301{
302	vaddr_t sva, eva, iva;
303	vsize_t line, offs;
304	uint set, nsets;
305
306	line = ci->ci_l1data.linesize;
307	sva = PHYS_TO_XKPHYS(0, CCA_CACHED);
308	offs = ci->ci_l1data.setsize;
309	nsets = ci->ci_l1data.sets;
310	/* keep only the index bits */
311	sva += va & (offs - 1);
312	eva = sva + PAGE_SIZE;
313	while (sva != eva) {
314		for (set = nsets, iva = sva; set != 0; set--, iva += offs)
315			cache(IndexWBInvalidate_D, iva);
316		sva += ci->ci_l1data.linesize;
317	}
318}
319
320/*
321 * Writeback D$ for the given page, which is expected to be currently
322 * mapped, allowing the use of `Hit' operations. This is less aggressive
323 * than using `Index' operations.
324 */
325
326void
327mips64r2_HitSyncDCachePage(struct cpu_info *ci, vaddr_t va, paddr_t pa)
328{
329	mips64r2_hitwbinv_primary(va, PAGE_SIZE, ci->ci_l1data.linesize);
330}
331
332/*
333 * Writeback D$ for the given range. Range is expected to be currently
334 * mapped, allowing the use of `Hit' operations. This is less aggressive
335 * than using `Index' operations.
336 */
337
338void
339mips64r2_HitSyncDCache(struct cpu_info *ci, vaddr_t _va, size_t _sz)
340{
341	vaddr_t va;
342	vsize_t sz;
343
344	/* extend the range to integral cache lines */
345	va = _va & ~(ci->ci_l1data.linesize - 1);
346	sz = ((_va + _sz + ci->ci_l1data.linesize - 1) & ~(ci->ci_l1data.linesize - 1)) - va;
347	mips64r2_hitwbinv_primary(va, sz, ci->ci_l1data.linesize);
348}
349
350/*
351 * Invalidate D$ for the given range. Range is expected to be currently
352 * mapped, allowing the use of `Hit' operations. This is less aggressive
353 * than using `Index' operations.
354 */
355
356void
357mips64r2_HitInvalidateDCache(struct cpu_info *ci, vaddr_t _va, size_t _sz)
358{
359	vaddr_t va;
360	vsize_t sz;
361
362	/* extend the range to integral cache lines */
363	va = _va & ~(ci->ci_l1data.linesize - 1);
364	sz = ((_va + _sz + ci->ci_l1data.linesize - 1) & ~(ci->ci_l1data.linesize - 1)) - va;
365	mips64r2_hitinv_primary(va, sz, ci->ci_l1data.linesize);
366}
367
368/*
369 * Backend for bus_dmamap_sync(). Enforce coherency of the given range
370 * by performing the necessary cache writeback and/or invalidate
371 * operations.
372 */
373void
374mips64r2_IOSyncDCache(struct cpu_info *ci, vaddr_t _va, size_t _sz, int how)
375{
376	vaddr_t va;
377	vsize_t sz;
378	int partial_start, partial_end;
379
380	/*
381	 * L1
382	 */
383
384	/* extend the range to integral cache lines */
385	va = _va & ~(ci->ci_l1data.linesize - 1);
386	sz = ((_va + _sz + ci->ci_l1data.linesize - 1) & ~(ci->ci_l1data.linesize - 1)) - va;
387
388	switch (how) {
389	case CACHE_SYNC_R:
390		/* writeback partial cachelines */
391		if (((_va | _sz) & (ci->ci_l1data.linesize - 1)) != 0) {
392			partial_start = va != _va;
393			partial_end = va + sz != _va + _sz;
394		} else {
395			partial_start = partial_end = 0;
396		}
397		if (partial_start) {
398			cache(HitWBInvalidate_D, va);
399			va += ci->ci_l1data.linesize;
400			sz -= ci->ci_l1data.linesize;
401		}
402		if (sz != 0 && partial_end) {
403			sz -= ci->ci_l1data.linesize;
404			cache(HitWBInvalidate_D, va + sz);
405		}
406		if (sz != 0)
407			mips64r2_hitinv_primary(va, sz, ci->ci_l1data.linesize);
408		break;
409	case CACHE_SYNC_X:
410	case CACHE_SYNC_W:
411		mips64r2_hitwbinv_primary(va, sz, ci->ci_l1data.linesize);
412		break;
413	}
414
415	/*
416	 * L2
417	 */
418
419	if (ci->ci_l2.size != 0) {
420		/* extend the range to integral cache lines */
421		va = _va & ~(ci->ci_l2.linesize - 1);
422		sz = ((_va + _sz + ci->ci_l2.linesize - 1) & ~(ci->ci_l2.linesize - 1)) - va;
423
424		switch (how) {
425		case CACHE_SYNC_R:
426			/* writeback partial cachelines */
427			if (((_va | _sz) & (ci->ci_l2.linesize - 1)) != 0) {
428				partial_start = va != _va;
429				partial_end = va + sz != _va + _sz;
430			} else {
431				partial_start = partial_end = 0;
432			}
433			if (partial_start) {
434				cache(HitWBInvalidate_S, va);
435				va += ci->ci_l2.linesize;
436				sz -= ci->ci_l2.linesize;
437			}
438			if (sz != 0 && partial_end) {
439				sz -= ci->ci_l2.linesize;
440				cache(HitWBInvalidate_S, va + sz);
441			}
442			if (sz != 0)
443				mips64r2_hitinv_secondary(va, sz, ci->ci_l2.linesize);
444			break;
445		case CACHE_SYNC_X:
446		case CACHE_SYNC_W:
447			mips64r2_hitwbinv_secondary(va, sz, ci->ci_l2.linesize);
448			break;
449		}
450	}
451
452	/*
453	 * L3
454	 */
455
456	if (ci->ci_l3.size != 0) {
457		/* extend the range to integral cache lines */
458		va = _va & ~(ci->ci_l3.linesize - 1);
459		sz = ((_va + _sz + ci->ci_l3.linesize - 1) & ~(ci->ci_l3.linesize - 1)) - va;
460
461		switch (how) {
462		case CACHE_SYNC_R:
463			/* writeback partial cachelines */
464			if (((_va | _sz) & (ci->ci_l3.linesize - 1)) != 0) {
465				partial_start = va != _va;
466				partial_end = va + sz != _va + _sz;
467			} else {
468				partial_start = partial_end = 0;
469			}
470			if (partial_start) {
471				cache(HitWBInvalidate_S, va);
472				va += ci->ci_l3.linesize;
473				sz -= ci->ci_l3.linesize;
474			}
475			if (sz != 0 && partial_end) {
476				sz -= ci->ci_l3.linesize;
477				cache(HitWBInvalidate_S, va + sz);
478			}
479			if (sz != 0)
480				mips64r2_hitinv_ternary(va, sz, ci->ci_l3.linesize);
481			break;
482		case CACHE_SYNC_X:
483		case CACHE_SYNC_W:
484			mips64r2_hitwbinv_ternary(va, sz, ci->ci_l3.linesize);
485			break;
486		}
487	}
488}
489