1/* sun4v_tlb_miss.S: Sun4v TLB miss handlers.
2 *
3 * Copyright (C) 2006 <davem@davemloft.net>
4 */
5
6	.text
7	.align	32
8
9	/* Load ITLB fault information into VADDR and CTX, using BASE.  */
10#define LOAD_ITLB_INFO(BASE, VADDR, CTX) \
11	ldx	[BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \
12	ldx	[BASE + HV_FAULT_I_CTX_OFFSET], CTX;
13
14	/* Load DTLB fault information into VADDR and CTX, using BASE.  */
15#define LOAD_DTLB_INFO(BASE, VADDR, CTX) \
16	ldx	[BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \
17	ldx	[BASE + HV_FAULT_D_CTX_OFFSET], CTX;
18
19	/* DEST = (VADDR >> 22)
20	 *
21	 * Branch to ZERO_CTX_LABEL if context is zero.
22	 */
23#define	COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \
24	srlx	VADDR, 22, DEST; \
25	brz,pn	CTX, ZERO_CTX_LABEL; \
26	 nop;
27
28	/* Create TSB pointer.  This is something like:
29	 *
30	 * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
31	 * tsb_base = tsb_reg & ~0x7UL;
32	 * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask);
33	 * tsb_ptr = tsb_base + (tsb_index * 16);
34	 */
35#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \
36	and	TSB_PTR, 0x7, TMP1;			\
37	mov	512, TMP2;				\
38	andn	TSB_PTR, 0x7, TSB_PTR;			\
39	sllx	TMP2, TMP1, TMP2;			\
40	srlx	VADDR, HASH_SHIFT, TMP1;		\
41	sub	TMP2, 1, TMP2;				\
42	and	TMP1, TMP2, TMP1;			\
43	sllx	TMP1, 4, TMP1;				\
44	add	TSB_PTR, TMP1, TSB_PTR;
45
46sun4v_itlb_miss:
47	/* Load MMU Miss base into %g2.  */
48	ldxa	[%g0] ASI_SCRATCHPAD, %g2
49
50	/* Load UTSB reg into %g1.  */
51	mov	SCRATCHPAD_UTSBREG1, %g1
52	ldxa	[%g1] ASI_SCRATCHPAD, %g1
53
54	LOAD_ITLB_INFO(%g2, %g4, %g5)
55	COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
56	COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
57
58	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
59	ldda	[%g1] ASI_QUAD_LDD_PHYS_4V, %g2
60	cmp	%g2, %g6
61	bne,a,pn %xcc, tsb_miss_page_table_walk
62	 mov	FAULT_CODE_ITLB, %g3
63	andcc	%g3, _PAGE_EXEC_4V, %g0
64	be,a,pn	%xcc, tsb_do_fault
65	 mov	FAULT_CODE_ITLB, %g3
66
67	/* We have a valid entry, make hypervisor call to load
68	 * I-TLB and return from trap.
69	 *
70	 * %g3:	PTE
71	 * %g4:	vaddr
72	 */
73sun4v_itlb_load:
74	ldxa	[%g0] ASI_SCRATCHPAD, %g6
75	mov	%o0, %g1		! save %o0
76	mov	%o1, %g2		! save %o1
77	mov	%o2, %g5		! save %o2
78	mov	%o3, %g7		! save %o3
79	mov	%g4, %o0		! vaddr
80	ldx	[%g6 + HV_FAULT_I_CTX_OFFSET], %o1	! ctx
81	mov	%g3, %o2		! PTE
82	mov	HV_MMU_IMMU, %o3	! flags
83	ta	HV_MMU_MAP_ADDR_TRAP
84	brnz,pn	%o0, sun4v_itlb_error
85	 mov	%g2, %o1		! restore %o1
86	mov	%g1, %o0		! restore %o0
87	mov	%g5, %o2		! restore %o2
88	mov	%g7, %o3		! restore %o3
89
90	retry
91
92sun4v_dtlb_miss:
93	/* Load MMU Miss base into %g2.  */
94	ldxa	[%g0] ASI_SCRATCHPAD, %g2
95
96	/* Load UTSB reg into %g1.  */
97	mov	SCRATCHPAD_UTSBREG1, %g1
98	ldxa	[%g1] ASI_SCRATCHPAD, %g1
99
100	LOAD_DTLB_INFO(%g2, %g4, %g5)
101	COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
102	COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
103
104	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
105	ldda	[%g1] ASI_QUAD_LDD_PHYS_4V, %g2
106	cmp	%g2, %g6
107	bne,a,pn %xcc, tsb_miss_page_table_walk
108	 mov	FAULT_CODE_DTLB, %g3
109
110	/* We have a valid entry, make hypervisor call to load
111	 * D-TLB and return from trap.
112	 *
113	 * %g3:	PTE
114	 * %g4:	vaddr
115	 */
116sun4v_dtlb_load:
117	ldxa	[%g0] ASI_SCRATCHPAD, %g6
118	mov	%o0, %g1		! save %o0
119	mov	%o1, %g2		! save %o1
120	mov	%o2, %g5		! save %o2
121	mov	%o3, %g7		! save %o3
122	mov	%g4, %o0		! vaddr
123	ldx	[%g6 + HV_FAULT_D_CTX_OFFSET], %o1	! ctx
124	mov	%g3, %o2		! PTE
125	mov	HV_MMU_DMMU, %o3	! flags
126	ta	HV_MMU_MAP_ADDR_TRAP
127	brnz,pn	%o0, sun4v_dtlb_error
128	 mov	%g2, %o1		! restore %o1
129	mov	%g1, %o0		! restore %o0
130	mov	%g5, %o2		! restore %o2
131	mov	%g7, %o3		! restore %o3
132
133	retry
134
135sun4v_dtlb_prot:
136	SET_GL(1)
137
138	/* Load MMU Miss base into %g5.  */
139	ldxa	[%g0] ASI_SCRATCHPAD, %g5
140
141	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
142	rdpr	%tl, %g1
143	cmp	%g1, 1
144	bgu,pn	%xcc, winfix_trampoline
145	 mov	FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
146	ba,pt	%xcc, sparc64_realfault_common
147	 nop
148
149	/* Called from trap table:
150	 * %g4:	vaddr
151	 * %g5:	context
152	 * %g6: TAG TARGET
153	 */
154sun4v_itsb_miss:
155	mov	SCRATCHPAD_UTSBREG1, %g1
156	ldxa	[%g1] ASI_SCRATCHPAD, %g1
157	brz,pn	%g5, kvmap_itlb_4v
158	 mov	FAULT_CODE_ITLB, %g3
159	ba,a,pt	%xcc, sun4v_tsb_miss_common
160
161	/* Called from trap table:
162	 * %g4:	vaddr
163	 * %g5:	context
164	 * %g6: TAG TARGET
165	 */
166sun4v_dtsb_miss:
167	mov	SCRATCHPAD_UTSBREG1, %g1
168	ldxa	[%g1] ASI_SCRATCHPAD, %g1
169	brz,pn	%g5, kvmap_dtlb_4v
170	 mov	FAULT_CODE_DTLB, %g3
171
172	/* fallthrough */
173
174sun4v_tsb_miss_common:
175	COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7)
176
177	sub	%g2, TRAP_PER_CPU_FAULT_INFO, %g2
178
179#ifdef CONFIG_HUGETLB_PAGE
180	mov	SCRATCHPAD_UTSBREG2, %g5
181	ldxa	[%g5] ASI_SCRATCHPAD, %g5
182	cmp	%g5, -1
183	be,pt	%xcc, 80f
184	 nop
185	COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7)
186
187	/* That clobbered %g2, reload it.  */
188	ldxa	[%g0] ASI_SCRATCHPAD, %g2
189	sub	%g2, TRAP_PER_CPU_FAULT_INFO, %g2
190
19180:	stx	%g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP]
192#endif
193
194	ba,pt	%xcc, tsb_miss_page_table_walk_sun4v_fastpath
195	 ldx	[%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
196
197sun4v_itlb_error:
198	sethi	%hi(sun4v_err_itlb_vaddr), %g1
199	stx	%g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
200	sethi	%hi(sun4v_err_itlb_ctx), %g1
201	ldxa	[%g0] ASI_SCRATCHPAD, %g6
202	ldx	[%g6 + HV_FAULT_I_CTX_OFFSET], %o1
203	stx	%o1, [%g1 + %lo(sun4v_err_itlb_ctx)]
204	sethi	%hi(sun4v_err_itlb_pte), %g1
205	stx	%g3, [%g1 + %lo(sun4v_err_itlb_pte)]
206	sethi	%hi(sun4v_err_itlb_error), %g1
207	stx	%o0, [%g1 + %lo(sun4v_err_itlb_error)]
208
209	rdpr	%tl, %g4
210	cmp	%g4, 1
211	ble,pt	%icc, 1f
212	 sethi	%hi(2f), %g7
213	ba,pt	%xcc, etraptl1
214	 or	%g7, %lo(2f), %g7
215
2161:	ba,pt	%xcc, etrap
2172:	 or	%g7, %lo(2b), %g7
218	call	sun4v_itlb_error_report
219	 add	%sp, PTREGS_OFF, %o0
220
221	/* NOTREACHED */
222
223sun4v_dtlb_error:
224	sethi	%hi(sun4v_err_dtlb_vaddr), %g1
225	stx	%g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
226	sethi	%hi(sun4v_err_dtlb_ctx), %g1
227	ldxa	[%g0] ASI_SCRATCHPAD, %g6
228	ldx	[%g6 + HV_FAULT_D_CTX_OFFSET], %o1
229	stx	%o1, [%g1 + %lo(sun4v_err_dtlb_ctx)]
230	sethi	%hi(sun4v_err_dtlb_pte), %g1
231	stx	%g3, [%g1 + %lo(sun4v_err_dtlb_pte)]
232	sethi	%hi(sun4v_err_dtlb_error), %g1
233	stx	%o0, [%g1 + %lo(sun4v_err_dtlb_error)]
234
235	rdpr	%tl, %g4
236	cmp	%g4, 1
237	ble,pt	%icc, 1f
238	 sethi	%hi(2f), %g7
239	ba,pt	%xcc, etraptl1
240	 or	%g7, %lo(2f), %g7
241
2421:	ba,pt	%xcc, etrap
2432:	 or	%g7, %lo(2b), %g7
244	call	sun4v_dtlb_error_report
245	 add	%sp, PTREGS_OFF, %o0
246
247	/* NOTREACHED */
248
249	/* Instruction Access Exception, tl0. */
250sun4v_iacc:
251	ldxa	[%g0] ASI_SCRATCHPAD, %g2
252	ldx	[%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
253	ldx	[%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
254	ldx	[%g2 + HV_FAULT_I_CTX_OFFSET], %g5
255	sllx	%g3, 16, %g3
256	or	%g5, %g3, %g5
257	ba,pt	%xcc, etrap
258	 rd	%pc, %g7
259	mov	%l4, %o1
260	mov	%l5, %o2
261	call	sun4v_insn_access_exception
262	 add	%sp, PTREGS_OFF, %o0
263	ba,a,pt	%xcc, rtrap_clr_l6
264
265	/* Instruction Access Exception, tl1. */
266sun4v_iacc_tl1:
267	ldxa	[%g0] ASI_SCRATCHPAD, %g2
268	ldx	[%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
269	ldx	[%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
270	ldx	[%g2 + HV_FAULT_I_CTX_OFFSET], %g5
271	sllx	%g3, 16, %g3
272	or	%g5, %g3, %g5
273	ba,pt	%xcc, etraptl1
274	 rd	%pc, %g7
275	mov	%l4, %o1
276	mov	%l5, %o2
277	call	sun4v_insn_access_exception_tl1
278	 add	%sp, PTREGS_OFF, %o0
279	ba,a,pt	%xcc, rtrap_clr_l6
280
281	/* Data Access Exception, tl0. */
282sun4v_dacc:
283	ldxa	[%g0] ASI_SCRATCHPAD, %g2
284	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
285	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
286	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
287	sllx	%g3, 16, %g3
288	or	%g5, %g3, %g5
289	ba,pt	%xcc, etrap
290	 rd	%pc, %g7
291	mov	%l4, %o1
292	mov	%l5, %o2
293	call	sun4v_data_access_exception
294	 add	%sp, PTREGS_OFF, %o0
295	ba,a,pt	%xcc, rtrap_clr_l6
296
297	/* Data Access Exception, tl1. */
298sun4v_dacc_tl1:
299	ldxa	[%g0] ASI_SCRATCHPAD, %g2
300	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
301	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
302	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
303	sllx	%g3, 16, %g3
304	or	%g5, %g3, %g5
305	ba,pt	%xcc, etraptl1
306	 rd	%pc, %g7
307	mov	%l4, %o1
308	mov	%l5, %o2
309	call	sun4v_data_access_exception_tl1
310	 add	%sp, PTREGS_OFF, %o0
311	ba,a,pt	%xcc, rtrap_clr_l6
312
313	/* Memory Address Unaligned.  */
314sun4v_mna:
315	/* Window fixup? */
316	rdpr	%tl, %g2
317	cmp	%g2, 1
318	ble,pt	%icc, 1f
319	 nop
320
321	SET_GL(1)
322	ldxa	[%g0] ASI_SCRATCHPAD, %g2
323	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
324	mov	HV_FAULT_TYPE_UNALIGNED, %g3
325	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g4
326	sllx	%g3, 16, %g3
327	or	%g4, %g3, %g4
328	ba,pt	%xcc, winfix_mna
329	 rdpr	%tpc, %g3
330	/* not reached */
331
3321:	ldxa	[%g0] ASI_SCRATCHPAD, %g2
333	mov	HV_FAULT_TYPE_UNALIGNED, %g3
334	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
335	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
336	sllx	%g3, 16, %g3
337	or	%g5, %g3, %g5
338
339	ba,pt	%xcc, etrap
340	 rd	%pc, %g7
341	mov	%l4, %o1
342	mov	%l5, %o2
343	call	sun4v_do_mna
344	 add	%sp, PTREGS_OFF, %o0
345	ba,a,pt	%xcc, rtrap_clr_l6
346
347	/* Privileged Action.  */
348sun4v_privact:
349	ba,pt	%xcc, etrap
350	 rd	%pc, %g7
351	call	do_privact
352	 add	%sp, PTREGS_OFF, %o0
353	ba,a,pt	%xcc, rtrap_clr_l6
354
355	/* Unaligned ldd float, tl0. */
356sun4v_lddfmna:
357	ldxa	[%g0] ASI_SCRATCHPAD, %g2
358	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
359	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
360	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
361	sllx	%g3, 16, %g3
362	or	%g5, %g3, %g5
363	ba,pt	%xcc, etrap
364	 rd	%pc, %g7
365	mov	%l4, %o1
366	mov	%l5, %o2
367	call	handle_lddfmna
368	 add	%sp, PTREGS_OFF, %o0
369	ba,a,pt	%xcc, rtrap_clr_l6
370
371	/* Unaligned std float, tl0. */
372sun4v_stdfmna:
373	ldxa	[%g0] ASI_SCRATCHPAD, %g2
374	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
375	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
376	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
377	sllx	%g3, 16, %g3
378	or	%g5, %g3, %g5
379	ba,pt	%xcc, etrap
380	 rd	%pc, %g7
381	mov	%l4, %o1
382	mov	%l5, %o2
383	call	handle_stdfmna
384	 add	%sp, PTREGS_OFF, %o0
385	ba,a,pt	%xcc, rtrap_clr_l6
386
387#define BRANCH_ALWAYS	0x10680000
388#define NOP		0x01000000
389#define SUN4V_DO_PATCH(OLD, NEW)	\
390	sethi	%hi(NEW), %g1; \
391	or	%g1, %lo(NEW), %g1; \
392	sethi	%hi(OLD), %g2; \
393	or	%g2, %lo(OLD), %g2; \
394	sub	%g1, %g2, %g1; \
395	sethi	%hi(BRANCH_ALWAYS), %g3; \
396	sll	%g1, 11, %g1; \
397	srl	%g1, 11 + 2, %g1; \
398	or	%g3, %lo(BRANCH_ALWAYS), %g3; \
399	or	%g3, %g1, %g3; \
400	stw	%g3, [%g2]; \
401	sethi	%hi(NOP), %g3; \
402	or	%g3, %lo(NOP), %g3; \
403	stw	%g3, [%g2 + 0x4]; \
404	flush	%g2;
405
406	.globl	sun4v_patch_tlb_handlers
407	.type	sun4v_patch_tlb_handlers,#function
408sun4v_patch_tlb_handlers:
409	SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss)
410	SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss)
411	SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss)
412	SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss)
413	SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot)
414	SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot)
415	SUN4V_DO_PATCH(tl0_iax, sun4v_iacc)
416	SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1)
417	SUN4V_DO_PATCH(tl0_dax, sun4v_dacc)
418	SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1)
419	SUN4V_DO_PATCH(tl0_mna, sun4v_mna)
420	SUN4V_DO_PATCH(tl1_mna, sun4v_mna)
421	SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna)
422	SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna)
423	SUN4V_DO_PATCH(tl0_privact, sun4v_privact)
424	retl
425	 nop
426	.size	sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers
427