1/*-
2 * Copyright (c) 2004-2010 Juli Mallett <jmallett@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29#include "opt_ddb.h"
30
31#include <sys/param.h>
32#include <sys/kernel.h>
33#include <sys/systm.h>
34#include <sys/pcpu.h>
35#include <sys/smp.h>
36
37#include <vm/vm.h>
38#include <vm/vm_page.h>
39
40#include <machine/pte.h>
41#include <machine/tlb.h>
42
43struct tlb_state {
44	unsigned wired;
45	struct tlb_entry {
46		register_t entryhi;
47		register_t entrylo0;
48		register_t entrylo1;
49	} entry[MIPS_MAX_TLB_ENTRIES];
50};
51
52static struct tlb_state tlb_state[MAXCPU];
53
54#if 0
55/*
56 * PageMask must increment in steps of 2 bits.
57 */
58COMPILE_TIME_ASSERT(POPCNT(TLBMASK_MASK) % 2 == 0);
59#endif
60
61static inline void
62tlb_probe(void)
63{
64	__asm __volatile ("tlbp" : : : "memory");
65	mips_cp0_sync();
66}
67
68static inline void
69tlb_read(void)
70{
71	__asm __volatile ("tlbr" : : : "memory");
72	mips_cp0_sync();
73}
74
75static inline void
76tlb_write_indexed(void)
77{
78	__asm __volatile ("tlbwi" : : : "memory");
79	mips_cp0_sync();
80}
81
82static inline void
83tlb_write_random(void)
84{
85	__asm __volatile ("tlbwr" : : : "memory");
86	mips_cp0_sync();
87}
88
89static void tlb_invalidate_one(unsigned);
90
91void
92tlb_insert_wired(unsigned i, vm_offset_t va, pt_entry_t pte0, pt_entry_t pte1)
93{
94	register_t asid;
95	register_t s;
96
97	va &= ~PAGE_MASK;
98
99	s = intr_disable();
100	asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
101
102	mips_wr_index(i);
103	mips_wr_pagemask(0);
104	mips_wr_entryhi(TLBHI_ENTRY(va, 0));
105	mips_wr_entrylo0(pte0);
106	mips_wr_entrylo1(pte1);
107	tlb_write_indexed();
108
109	mips_wr_entryhi(asid);
110	intr_restore(s);
111}
112
113void
114tlb_invalidate_address(struct pmap *pmap, vm_offset_t va)
115{
116	register_t asid;
117	register_t s;
118	int i;
119
120	va &= ~PAGE_MASK;
121
122	s = intr_disable();
123	asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
124
125	mips_wr_pagemask(0);
126	mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
127	tlb_probe();
128	i = mips_rd_index();
129	if (i >= 0)
130		tlb_invalidate_one(i);
131
132	mips_wr_entryhi(asid);
133	intr_restore(s);
134}
135
136void
137tlb_invalidate_all(void)
138{
139	register_t asid;
140	register_t s;
141	unsigned i;
142
143	s = intr_disable();
144	asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
145
146	for (i = mips_rd_wired(); i < num_tlbentries; i++)
147		tlb_invalidate_one(i);
148
149	mips_wr_entryhi(asid);
150	intr_restore(s);
151}
152
153void
154tlb_invalidate_all_user(struct pmap *pmap)
155{
156	register_t asid;
157	register_t s;
158	unsigned i;
159
160	s = intr_disable();
161	asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
162
163	for (i = mips_rd_wired(); i < num_tlbentries; i++) {
164		register_t uasid;
165
166		mips_wr_index(i);
167		tlb_read();
168
169		uasid = mips_rd_entryhi() & TLBHI_ASID_MASK;
170		if (pmap == NULL) {
171			/*
172			 * Invalidate all non-kernel entries.
173			 */
174			if (uasid == 0)
175				continue;
176		} else {
177			/*
178			 * Invalidate this pmap's entries.
179			 */
180			if (uasid != pmap_asid(pmap))
181				continue;
182		}
183		tlb_invalidate_one(i);
184	}
185
186	mips_wr_entryhi(asid);
187	intr_restore(s);
188}
189
190/* XXX Only if DDB?  */
191void
192tlb_save(void)
193{
194	unsigned i, cpu;
195
196	cpu = PCPU_GET(cpuid);
197
198	tlb_state[cpu].wired = mips_rd_wired();
199	for (i = 0; i < num_tlbentries; i++) {
200		mips_wr_index(i);
201		tlb_read();
202
203		tlb_state[cpu].entry[i].entryhi = mips_rd_entryhi();
204		tlb_state[cpu].entry[i].entrylo0 = mips_rd_entrylo0();
205		tlb_state[cpu].entry[i].entrylo1 = mips_rd_entrylo1();
206	}
207}
208
209void
210tlb_update(struct pmap *pmap, vm_offset_t va, pt_entry_t pte)
211{
212	register_t asid;
213	register_t s;
214	int i;
215
216	va &= ~PAGE_MASK;
217	pte &= ~TLBLO_SWBITS_MASK;
218
219	s = intr_disable();
220	asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
221
222	mips_wr_pagemask(0);
223	mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
224	tlb_probe();
225	i = mips_rd_index();
226	if (i >= 0) {
227		tlb_read();
228
229		if ((va & PAGE_SIZE) == 0) {
230			mips_wr_entrylo0(pte);
231		} else {
232			mips_wr_entrylo1(pte);
233		}
234		tlb_write_indexed();
235	}
236
237	mips_wr_entryhi(asid);
238	intr_restore(s);
239}
240
241static void
242tlb_invalidate_one(unsigned i)
243{
244	/* XXX an invalid ASID? */
245	mips_wr_entryhi(TLBHI_ENTRY(MIPS_KSEG0_START + (2 * i * PAGE_SIZE), 0));
246	mips_wr_entrylo0(0);
247	mips_wr_entrylo1(0);
248	mips_wr_pagemask(0);
249	mips_wr_index(i);
250	tlb_write_indexed();
251}
252
253#ifdef DDB
254#include <ddb/ddb.h>
255
256DB_SHOW_COMMAND(tlb, ddb_dump_tlb)
257{
258	register_t ehi, elo0, elo1;
259	unsigned i, cpu;
260
261	/*
262	 * XXX
263	 * The worst conversion from hex to decimal ever.
264	 */
265	if (have_addr)
266		cpu = ((addr >> 4) % 16) * 10 + (addr % 16);
267	else
268		cpu = PCPU_GET(cpuid);
269
270	if (cpu < 0 || cpu >= mp_ncpus) {
271		db_printf("Invalid CPU %u\n", cpu);
272		return;
273	}
274
275	if (cpu == PCPU_GET(cpuid))
276		tlb_save();
277
278	db_printf("Beginning TLB dump for CPU %u...\n", cpu);
279	for (i = 0; i < num_tlbentries; i++) {
280		if (i == tlb_state[cpu].wired) {
281			if (i != 0)
282				db_printf("^^^ WIRED ENTRIES ^^^\n");
283			else
284				db_printf("(No wired entries.)\n");
285		}
286
287		/* XXX PageMask.  */
288		ehi = tlb_state[cpu].entry[i].entryhi;
289		elo0 = tlb_state[cpu].entry[i].entrylo0;
290		elo1 = tlb_state[cpu].entry[i].entrylo1;
291
292		if (elo0 == 0 && elo1 == 0)
293			continue;
294
295		db_printf("#%u\t=> %jx\n", i, (intmax_t)ehi);
296		db_printf(" Lo0\t%jx\t(%#jx)\n", (intmax_t)elo0, (intmax_t)TLBLO_PTE_TO_PA(elo0));
297		db_printf(" Lo1\t%jx\t(%#jx)\n", (intmax_t)elo1, (intmax_t)TLBLO_PTE_TO_PA(elo1));
298	}
299	db_printf("Finished.\n");
300}
301#endif
302