1/*-
2 * Copyright (c) 2013 Ian Lepore <ian@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD$");
29
30/* Routines for mapping device memory. */
31
32#include "opt_ddb.h"
33
34#include <sys/param.h>
35#include <sys/systm.h>
36#include <sys/devmap.h>
37#include <vm/vm.h>
38#include <vm/vm_extern.h>
39#include <vm/pmap.h>
40#include <machine/vmparam.h>
41
42static const struct devmap_entry *devmap_table;
43static boolean_t devmap_bootstrap_done = false;
44
45/*
46 * The allocated-kva (akva) devmap table and metadata.  Platforms can call
47 * devmap_add_entry() to add static device mappings to this table using
48 * automatically allocated virtual addresses carved out of the top of kva space.
49 * Allocation begins immediately below the max kernel virtual address.
50 */
51#define	AKVA_DEVMAP_MAX_ENTRIES	32
52static struct devmap_entry	akva_devmap_entries[AKVA_DEVMAP_MAX_ENTRIES];
53static u_int			akva_devmap_idx;
54static vm_offset_t		akva_devmap_vaddr = DEVMAP_MAX_VADDR;
55
56#if defined(__aarch64__) || defined(__riscv)
57extern int early_boot;
58#endif
59
60/*
61 * Print the contents of the static mapping table using the provided printf-like
62 * output function (which will be either printf or db_printf).
63 */
64static void
65devmap_dump_table(int (*prfunc)(const char *, ...))
66{
67	const struct devmap_entry *pd;
68
69	if (devmap_table == NULL || devmap_table[0].pd_size == 0) {
70		prfunc("No static device mappings.\n");
71		return;
72	}
73
74	prfunc("Static device mappings:\n");
75	for (pd = devmap_table; pd->pd_size != 0; ++pd) {
76		prfunc("  0x%08jx - 0x%08jx mapped at VA 0x%08jx\n",
77		    (uintmax_t)pd->pd_pa,
78		    (uintmax_t)(pd->pd_pa + pd->pd_size - 1),
79		    (uintmax_t)pd->pd_va);
80	}
81}
82
83/*
84 * Print the contents of the static mapping table.  Used for bootverbose.
85 */
86void
87devmap_print_table()
88{
89	devmap_dump_table(printf);
90}
91
92/*
93 * Return the "last" kva address used by the registered devmap table.  It's
94 * actually the lowest address used by the static mappings, i.e., the address of
95 * the first unusable byte of KVA.
96 */
97vm_offset_t
98devmap_lastaddr()
99{
100	const struct devmap_entry *pd;
101	vm_offset_t lowaddr;
102
103	if (akva_devmap_idx > 0)
104		return (akva_devmap_vaddr);
105
106	lowaddr = DEVMAP_MAX_VADDR;
107	for (pd = devmap_table; pd != NULL && pd->pd_size != 0; ++pd) {
108		if (lowaddr > pd->pd_va)
109			lowaddr = pd->pd_va;
110	}
111
112	return (lowaddr);
113}
114
115/*
116 * Add an entry to the internal "akva" static devmap table using the given
117 * physical address and size and a virtual address allocated from the top of
118 * kva.  This automatically registers the akva table on the first call, so all a
119 * platform has to do is call this routine to install as many mappings as it
120 * needs and when the platform-specific init function calls devmap_bootstrap()
121 * it will pick up all the entries in the akva table automatically.
122 */
123void
124devmap_add_entry(vm_paddr_t pa, vm_size_t sz)
125{
126	struct devmap_entry *m;
127
128	if (devmap_bootstrap_done)
129		panic("devmap_add_entry() after devmap_bootstrap()");
130
131	if (akva_devmap_idx == (AKVA_DEVMAP_MAX_ENTRIES - 1))
132		panic("AKVA_DEVMAP_MAX_ENTRIES is too small");
133
134	if (akva_devmap_idx == 0)
135		devmap_register_table(akva_devmap_entries);
136
137	 /* Allocate virtual address space from the top of kva downwards. */
138#ifdef __arm__
139	/*
140	 * If the range being mapped is aligned and sized to 1MB boundaries then
141	 * also align the virtual address to the next-lower 1MB boundary so that
142	 * we end with a nice efficient section mapping.
143	 */
144	if ((pa & 0x000fffff) == 0 && (sz & 0x000fffff) == 0) {
145		akva_devmap_vaddr = trunc_1mpage(akva_devmap_vaddr - sz);
146	} else
147#endif
148	{
149		akva_devmap_vaddr = trunc_page(akva_devmap_vaddr - sz);
150	}
151	m = &akva_devmap_entries[akva_devmap_idx++];
152	m->pd_va    = akva_devmap_vaddr;
153	m->pd_pa    = pa;
154	m->pd_size  = sz;
155}
156
157/*
158 * Register the given table as the one to use in devmap_bootstrap().
159 */
160void
161devmap_register_table(const struct devmap_entry *table)
162{
163
164	devmap_table = table;
165}
166
167/*
168 * Map all of the static regions in the devmap table, and remember the devmap
169 * table so the mapdev, ptov, and vtop functions can do lookups later.
170 *
171 * If a non-NULL table pointer is given it is used unconditionally, otherwise
172 * the previously-registered table is used.  This smooths transition from legacy
173 * code that fills in a local table then calls this function passing that table,
174 * and newer code that uses devmap_register_table() in platform-specific
175 * code, then lets the common platform-specific init function call this function
176 * with a NULL pointer.
177 */
178void
179devmap_bootstrap(vm_offset_t l1pt, const struct devmap_entry *table)
180{
181	const struct devmap_entry *pd;
182
183	devmap_bootstrap_done = true;
184
185	/*
186	 * If given a table pointer, use it.  Otherwise, if a table was
187	 * previously registered, use it.  Otherwise, no work to do.
188	 */
189	if (table != NULL)
190		devmap_table = table;
191	else if (devmap_table == NULL)
192		return;
193
194	for (pd = devmap_table; pd->pd_size != 0; ++pd) {
195#if defined(__arm__)
196#if __ARM_ARCH >= 6
197		pmap_preboot_map_attr(pd->pd_pa, pd->pd_va, pd->pd_size,
198		    VM_PROT_READ | VM_PROT_WRITE, VM_MEMATTR_DEVICE);
199#else
200		pmap_map_chunk(l1pt, pd->pd_va, pd->pd_pa, pd->pd_size,
201		    VM_PROT_READ | VM_PROT_WRITE, PTE_DEVICE);
202#endif
203#elif defined(__aarch64__) || defined(__riscv)
204		pmap_kenter_device(pd->pd_va, pd->pd_size, pd->pd_pa);
205#endif
206	}
207}
208
209/*
210 * Look up the given physical address in the static mapping data and return the
211 * corresponding virtual address, or NULL if not found.
212 */
213void *
214devmap_ptov(vm_paddr_t pa, vm_size_t size)
215{
216	const struct devmap_entry *pd;
217
218	if (devmap_table == NULL)
219		return (NULL);
220
221	for (pd = devmap_table; pd->pd_size != 0; ++pd) {
222		if (pa >= pd->pd_pa && pa + size <= pd->pd_pa + pd->pd_size)
223			return ((void *)(pd->pd_va + (pa - pd->pd_pa)));
224	}
225
226	return (NULL);
227}
228
229/*
230 * Look up the given virtual address in the static mapping data and return the
231 * corresponding physical address, or DEVMAP_PADDR_NOTFOUND if not found.
232 */
233vm_paddr_t
234devmap_vtop(void * vpva, vm_size_t size)
235{
236	const struct devmap_entry *pd;
237	vm_offset_t va;
238
239	if (devmap_table == NULL)
240		return (DEVMAP_PADDR_NOTFOUND);
241
242	va = (vm_offset_t)vpva;
243	for (pd = devmap_table; pd->pd_size != 0; ++pd) {
244		if (va >= pd->pd_va && va + size <= pd->pd_va + pd->pd_size)
245			return ((vm_paddr_t)(pd->pd_pa + (va - pd->pd_va)));
246	}
247
248	return (DEVMAP_PADDR_NOTFOUND);
249}
250
251/*
252 * Map a set of physical memory pages into the kernel virtual address space.
253 * Return a pointer to where it is mapped.
254 *
255 * This uses a pre-established static mapping if one exists for the requested
256 * range, otherwise it allocates kva space and maps the physical pages into it.
257 *
258 * This routine is intended to be used for mapping device memory, NOT real
259 * memory; the mapping type is inherently VM_MEMATTR_DEVICE in
260 * pmap_kenter_device().
261 */
262void *
263pmap_mapdev(vm_offset_t pa, vm_size_t size)
264{
265	vm_offset_t va, offset;
266	void * rva;
267
268	/* First look in the static mapping table. */
269	if ((rva = devmap_ptov(pa, size)) != NULL)
270		return (rva);
271
272	offset = pa & PAGE_MASK;
273	pa = trunc_page(pa);
274	size = round_page(size + offset);
275
276#if defined(__aarch64__) || defined(__riscv)
277	if (early_boot) {
278		akva_devmap_vaddr = trunc_page(akva_devmap_vaddr - size);
279		va = akva_devmap_vaddr;
280		KASSERT(va >= VM_MAX_KERNEL_ADDRESS - PMAP_MAPDEV_EARLY_SIZE,
281		    ("Too many early devmap mappings"));
282	} else
283#endif
284		va = kva_alloc(size);
285	if (!va)
286		panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
287
288	pmap_kenter_device(va, size, pa);
289
290	return ((void *)(va + offset));
291}
292
293#if defined(__aarch64__)
294void *
295pmap_mapdev_attr(vm_offset_t pa, vm_size_t size, vm_memattr_t ma)
296{
297	vm_offset_t va, offset;
298	void * rva;
299
300	/* First look in the static mapping table. */
301	if ((rva = devmap_ptov(pa, size)) != NULL)
302		return (rva);
303
304	offset = pa & PAGE_MASK;
305	pa = trunc_page(pa);
306	size = round_page(size + offset);
307
308	if (early_boot) {
309		akva_devmap_vaddr = trunc_page(akva_devmap_vaddr - size);
310		va = akva_devmap_vaddr;
311		KASSERT(va >= (VM_MAX_KERNEL_ADDRESS - (PMAP_MAPDEV_EARLY_SIZE)),
312		    ("Too many early devmap mappings 2"));
313	} else
314		va = kva_alloc(size);
315	if (!va)
316		panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
317
318	pmap_kenter(va, size, pa, ma);
319
320	return ((void *)(va + offset));
321}
322#endif
323
324/*
325 * Unmap device memory and free the kva space.
326 */
327void
328pmap_unmapdev(vm_offset_t va, vm_size_t size)
329{
330	vm_offset_t offset;
331
332	/* Nothing to do if we find the mapping in the static table. */
333	if (devmap_vtop((void*)va, size) != DEVMAP_PADDR_NOTFOUND)
334		return;
335
336	offset = va & PAGE_MASK;
337	va = trunc_page(va);
338	size = round_page(size + offset);
339
340	pmap_kremove_device(va, size);
341	kva_free(va, size);
342}
343
344#ifdef DDB
345#include <ddb/ddb.h>
346
347DB_SHOW_COMMAND(devmap, db_show_devmap)
348{
349	devmap_dump_table(db_printf);
350}
351
352#endif /* DDB */
353