1/* $OpenBSD: machdep.c,v 1.89 2024/04/29 13:01:54 jsg Exp $ */
2/*
3 * Copyright (c) 2014 Patrick Wildt <patrick@blueri.se>
4 * Copyright (c) 2021 Mark Kettenis <kettenis@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/sched.h>
22#include <sys/proc.h>
23#include <sys/sysctl.h>
24#include <sys/reboot.h>
25#include <sys/mount.h>
26#include <sys/exec.h>
27#include <sys/user.h>
28#include <sys/conf.h>
29#include <sys/kcore.h>
30#include <sys/core.h>
31#include <sys/msgbuf.h>
32#include <sys/buf.h>
33#include <sys/termios.h>
34#include <sys/sensors.h>
35#include <sys/malloc.h>
36
37#include <net/if.h>
38#include <uvm/uvm.h>
39#include <dev/cons.h>
40#include <dev/ofw/fdt.h>
41#include <dev/ofw/openfirm.h>
42#include <machine/param.h>
43#include <machine/kcore.h>
44#include <machine/bootconfig.h>
45#include <machine/bus.h>
46#include <machine/fpu.h>
47#include <arm64/arm64/arm64var.h>
48
49#include <machine/db_machdep.h>
50#include <ddb/db_extern.h>
51
52#include <dev/efi/efi.h>
53
54#include "softraid.h"
55#if NSOFTRAID > 0
56#include <dev/softraidvar.h>
57#endif
58
59extern vaddr_t virtual_avail;
60extern uint64_t esym;
61
62extern char _start[];
63
64char *boot_args = NULL;
65uint8_t *bootmac = NULL;
66
67int stdout_node;
68int stdout_speed;
69
70void (*cpuresetfn)(void);
71void (*powerdownfn)(void);
72
73int cold = 1;
74int lid_action = 1;
75
76struct vm_map *exec_map = NULL;
77struct vm_map *phys_map = NULL;
78
79int physmem;
80
81struct consdev *cn_tab;
82
83caddr_t msgbufaddr;
84paddr_t msgbufphys;
85
86struct user *proc0paddr;
87
88struct uvm_constraint_range  dma_constraint = { 0x0, (paddr_t)-1 };
89struct uvm_constraint_range *uvm_md_constraints[] = {
90	&dma_constraint,
91	NULL,
92};
93
94/* the following is used externally (sysctl_hw) */
95char    machine[] = MACHINE;            /* from <machine/param.h> */
96
97int safepri = 0;
98
99struct cpu_info cpu_info_primary;
100struct cpu_info *cpu_info[MAXCPUS] = { &cpu_info_primary };
101
102struct fdt_reg memreg[VM_PHYSSEG_MAX];
103int nmemreg;
104
105void memreg_add(const struct fdt_reg *);
106void memreg_remove(const struct fdt_reg *);
107
108static int
109atoi(const char *s)
110{
111	int n, neg;
112
113	n = 0;
114	neg = 0;
115
116	while (*s == '-') {
117		s++;
118		neg = !neg;
119	}
120
121	while (*s != '\0') {
122		if (*s < '0' || *s > '9')
123			break;
124
125		n = (10 * n) + (*s - '0');
126		s++;
127	}
128
129	return (neg ? -n : n);
130}
131
132void *
133fdt_find_cons(const char *name)
134{
135	char *alias = "serial0";
136	char buf[128];
137	char *stdout = NULL;
138	char *p;
139	void *node;
140
141	/* First check if "stdout-path" is set. */
142	node = fdt_find_node("/chosen");
143	if (node) {
144		if (fdt_node_property(node, "stdout-path", &stdout) > 0) {
145			if (strchr(stdout, ':') != NULL) {
146				strlcpy(buf, stdout, sizeof(buf));
147				if ((p = strchr(buf, ':')) != NULL) {
148					*p++ = '\0';
149					stdout_speed = atoi(p);
150				}
151				stdout = buf;
152			}
153			if (stdout[0] != '/') {
154				/* It's an alias. */
155				alias = stdout;
156				stdout = NULL;
157			}
158		}
159	}
160
161	/* Perform alias lookup if necessary. */
162	if (stdout == NULL) {
163		node = fdt_find_node("/aliases");
164		if (node)
165			fdt_node_property(node, alias, &stdout);
166	}
167
168	/* Lookup the physical address of the interface. */
169	if (stdout) {
170		node = fdt_find_node(stdout);
171		if (node && fdt_is_compatible(node, name)) {
172			stdout_node = OF_finddevice(stdout);
173			return (node);
174		}
175	}
176
177	return (NULL);
178}
179
180void	amluart_init_cons(void);
181void	cduart_init_cons(void);
182void	com_fdt_init_cons(void);
183void	exuart_init_cons(void);
184void	imxuart_init_cons(void);
185void	mvuart_init_cons(void);
186void	pluart_init_cons(void);
187void	simplefb_init_cons(bus_space_tag_t);
188
189void
190consinit(void)
191{
192	static int consinit_called = 0;
193
194	if (consinit_called != 0)
195		return;
196
197	consinit_called = 1;
198
199	amluart_init_cons();
200	cduart_init_cons();
201	com_fdt_init_cons();
202	exuart_init_cons();
203	imxuart_init_cons();
204	mvuart_init_cons();
205	pluart_init_cons();
206	simplefb_init_cons(&arm64_bs_tag);
207}
208
209void
210cpu_idle_enter(void)
211{
212	disable_irq_daif();
213}
214
215void (*cpu_idle_cycle_fcn)(void) = cpu_wfi;
216
217void
218cpu_idle_cycle(void)
219{
220	cpu_idle_cycle_fcn();
221	enable_irq_daif();
222	disable_irq_daif();
223}
224
225void
226cpu_idle_leave(void)
227{
228	enable_irq_daif();
229}
230
231/* Dummy trapframe for proc0. */
232struct trapframe proc0tf;
233
234void
235cpu_startup(void)
236{
237	u_int loop;
238	paddr_t minaddr;
239	paddr_t maxaddr;
240
241	proc0.p_addr = proc0paddr;
242
243	/*
244	 * Give pmap a chance to set up a few more things now the vm
245	 * is initialised
246	 */
247	pmap_postinit();
248
249	/*
250	 * Initialize error message buffer (at end of core).
251	 */
252
253	/* msgbufphys was setup during the secondary boot strap */
254	for (loop = 0; loop < atop(MSGBUFSIZE); ++loop)
255		pmap_kenter_pa((vaddr_t)msgbufaddr + loop * PAGE_SIZE,
256		    msgbufphys + loop * PAGE_SIZE, PROT_READ | PROT_WRITE);
257	pmap_update(pmap_kernel());
258	initmsgbuf(msgbufaddr, round_page(MSGBUFSIZE));
259
260	/*
261	 * Identify ourselves for the msgbuf (everything printed earlier will
262	 * not be buffered).
263	 */
264	printf("%s", version);
265
266	printf("real mem  = %lu (%luMB)\n", ptoa(physmem),
267	    ptoa(physmem) / 1024 / 1024);
268
269	/*
270	 * Allocate a submap for exec arguments.  This map effectively
271	 * limits the number of processes exec'ing at any time.
272	 */
273	minaddr = vm_map_min(kernel_map);
274	exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
275	    16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL);
276
277	/*
278	 * Allocate a submap for physio
279	 */
280	phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
281	    VM_PHYS_SIZE, 0, FALSE, NULL);
282
283	/*
284	 * Set up buffers, so they can be used to read disk labels.
285	 */
286	bufinit();
287
288	printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free),
289	    ptoa(uvmexp.free) / 1024 / 1024);
290
291	curpcb = &proc0.p_addr->u_pcb;
292	curpcb->pcb_flags = 0;
293	curpcb->pcb_tf = &proc0tf;
294
295	if (boothowto & RB_CONFIG) {
296#ifdef BOOT_CONFIG
297		user_config();
298#else
299		printf("kernel does not support -c; continuing..\n");
300#endif
301	}
302}
303
304void    cpu_switchto_asm(struct proc *, struct proc *);
305
306void
307cpu_switchto(struct proc *old, struct proc *new)
308{
309	if (old) {
310		struct pcb *pcb = &old->p_addr->u_pcb;
311
312		if (pcb->pcb_flags & PCB_FPU)
313			fpu_save(old);
314
315		fpu_drop();
316	}
317
318	cpu_switchto_asm(old, new);
319}
320
321/*
322 * machine dependent system variables.
323 */
324
325const struct sysctl_bounded_args cpuctl_vars[] = {
326	{ CPU_LIDACTION, &lid_action, 0, 2 },
327};
328
329int
330cpu_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
331    size_t newlen, struct proc *p)
332{
333	char *compatible;
334	int node, len, error;
335	uint64_t value;
336
337	/* all sysctl names at this level are terminal */
338	if (namelen != 1)
339		return (ENOTDIR);		/* overloaded */
340
341	switch (name[0]) {
342	case CPU_COMPATIBLE:
343		node = OF_finddevice("/");
344		len = OF_getproplen(node, "compatible");
345		if (len <= 0)
346			return (EOPNOTSUPP);
347		compatible = malloc(len, M_TEMP, M_WAITOK | M_ZERO);
348		OF_getprop(node, "compatible", compatible, len);
349		compatible[len - 1] = 0;
350		error = sysctl_rdstring(oldp, oldlenp, newp, compatible);
351		free(compatible, M_TEMP, len);
352		return error;
353	case CPU_ID_AA64ISAR0:
354		value = cpu_id_aa64isar0 & ID_AA64ISAR0_MASK;
355		value &= ~ID_AA64ISAR0_TLB_MASK;
356		return sysctl_rdquad(oldp, oldlenp, newp, value);
357	case CPU_ID_AA64ISAR1:
358		value = cpu_id_aa64isar1 & ID_AA64ISAR1_MASK;
359		value &= ~ID_AA64ISAR1_SPECRES_MASK;
360		return sysctl_rdquad(oldp, oldlenp, newp, value);
361	case CPU_ID_AA64PFR0:
362		value = 0;
363		value |= cpu_id_aa64pfr0 & ID_AA64PFR0_FP_MASK;
364		value |= cpu_id_aa64pfr0 & ID_AA64PFR0_ADV_SIMD_MASK;
365		value |= cpu_id_aa64pfr0 & ID_AA64PFR0_DIT_MASK;
366		return sysctl_rdquad(oldp, oldlenp, newp, value);
367	case CPU_ID_AA64PFR1:
368		value = 0;
369		value |= cpu_id_aa64pfr1 & ID_AA64PFR1_BT_MASK;
370		value |= cpu_id_aa64pfr1 & ID_AA64PFR1_SSBS_MASK;
371		return sysctl_rdquad(oldp, oldlenp, newp, value);
372	case CPU_ID_AA64ISAR2:
373	case CPU_ID_AA64MMFR0:
374	case CPU_ID_AA64MMFR1:
375	case CPU_ID_AA64MMFR2:
376	case CPU_ID_AA64SMFR0:
377	case CPU_ID_AA64ZFR0:
378		return sysctl_rdquad(oldp, oldlenp, newp, 0);
379	default:
380		return (sysctl_bounded_arr(cpuctl_vars, nitems(cpuctl_vars),
381		    name, namelen, oldp, oldlenp, newp, newlen));
382	}
383	/* NOTREACHED */
384}
385
386void dumpsys(void);
387
388int	waittime = -1;
389
390__dead void
391boot(int howto)
392{
393	if ((howto & RB_RESET) != 0)
394		goto doreset;
395
396	if (cold) {
397		if ((howto & RB_USERREQ) == 0)
398			howto |= RB_HALT;
399		goto haltsys;
400	}
401
402	boothowto = howto;
403	if ((howto & RB_NOSYNC) == 0 && waittime < 0) {
404		waittime = 0;
405		vfs_shutdown(curproc);
406
407		if ((howto & RB_TIMEBAD) == 0) {
408			resettodr();
409		} else {
410			printf("WARNING: not updating battery clock\n");
411		}
412	}
413	if_downall();
414
415	uvm_shutdown();
416	splhigh();
417	cold = 1;
418
419	if ((howto & RB_DUMP) != 0)
420		dumpsys();
421
422haltsys:
423	config_suspend_all(DVACT_POWERDOWN);
424
425	if ((howto & RB_HALT) != 0) {
426		if ((howto & RB_POWERDOWN) != 0) {
427			printf("\nAttempting to power down...\n");
428			delay(500000);
429			if (powerdownfn)
430				(*powerdownfn)();
431		}
432
433		printf("\n");
434		printf("The operating system has halted.\n");
435		printf("Please press any key to reboot.\n\n");
436		cngetc();
437	}
438
439doreset:
440	printf("rebooting...\n");
441	delay(500000);
442	if (cpuresetfn)
443		(*cpuresetfn)();
444	printf("reboot failed; spinning\n");
445	for (;;)
446		continue;
447	/* NOTREACHED */
448}
449
450void
451setregs(struct proc *p, struct exec_package *pack, u_long stack,
452    struct ps_strings *arginfo)
453{
454	struct pmap *pm = p->p_vmspace->vm_map.pmap;
455	struct pcb *pcb = &p->p_addr->u_pcb;
456	struct trapframe *tf = pcb->pcb_tf;
457
458	if (pack->ep_flags & EXEC_NOBTCFI)
459		pm->pm_guarded = 0;
460	else
461		pm->pm_guarded = ATTR_GP;
462
463	arc4random_buf(&pm->pm_apiakey, sizeof(pm->pm_apiakey));
464	arc4random_buf(&pm->pm_apdakey, sizeof(pm->pm_apdakey));
465	arc4random_buf(&pm->pm_apibkey, sizeof(pm->pm_apibkey));
466	arc4random_buf(&pm->pm_apdbkey, sizeof(pm->pm_apdbkey));
467	arc4random_buf(&pm->pm_apgakey, sizeof(pm->pm_apgakey));
468	pmap_setpauthkeys(pm);
469
470	/* If we were using the FPU, forget about it. */
471	memset(&pcb->pcb_fpstate, 0, sizeof(pcb->pcb_fpstate));
472	pcb->pcb_flags &= ~PCB_FPU;
473	fpu_drop();
474
475	memset(tf, 0, sizeof *tf);
476	tf->tf_sp = stack;
477	tf->tf_lr = pack->ep_entry;
478	tf->tf_elr = pack->ep_entry; /* ??? */
479	tf->tf_spsr = PSR_M_EL0t | PSR_DIT;
480}
481
482void
483need_resched(struct cpu_info *ci)
484{
485	ci->ci_want_resched = 1;
486
487	/* There's a risk we'll be called before the idle threads start */
488	if (ci->ci_curproc) {
489		aston(ci->ci_curproc);
490		cpu_kick(ci);
491	}
492}
493
494int	cpu_dumpsize(void);
495u_long	cpu_dump_mempagecnt(void);
496
497paddr_t dumpmem_paddr;
498vaddr_t dumpmem_vaddr;
499psize_t dumpmem_sz;
500
501/*
502 * These variables are needed by /sbin/savecore
503 */
504u_long	dumpmag = 0x8fca0101;	/* magic number */
505int 	dumpsize = 0;		/* pages */
506long	dumplo = 0; 		/* blocks */
507
508/*
509 * cpu_dump: dump the machine-dependent kernel core dump headers.
510 */
511int
512cpu_dump(void)
513{
514	int (*dump)(dev_t, daddr_t, caddr_t, size_t);
515	char buf[dbtob(1)];
516	kcore_seg_t *segp;
517	cpu_kcore_hdr_t *cpuhdrp;
518	phys_ram_seg_t *memsegp;
519#if 0
520	caddr_t va;
521	int i;
522#endif
523
524	dump = bdevsw[major(dumpdev)].d_dump;
525
526	memset(buf, 0, sizeof buf);
527	segp = (kcore_seg_t *)buf;
528	cpuhdrp = (cpu_kcore_hdr_t *)&buf[ALIGN(sizeof(*segp))];
529	memsegp = (phys_ram_seg_t *)&buf[ALIGN(sizeof(*segp)) +
530	    ALIGN(sizeof(*cpuhdrp))];
531
532	/*
533	 * Generate a segment header.
534	 */
535	CORE_SETMAGIC(*segp, KCORE_MAGIC, MID_MACHINE, CORE_CPU);
536	segp->c_size = dbtob(1) - ALIGN(sizeof(*segp));
537
538	/*
539	 * Add the machine-dependent header info.
540	 */
541	cpuhdrp->kernelbase = KERNEL_BASE;
542	cpuhdrp->kerneloffs = 0;
543	cpuhdrp->staticsize = 0;
544	cpuhdrp->pmap_kernel_l1 = 0;
545	cpuhdrp->pmap_kernel_l2 = 0;
546
547#if 0
548	/*
549	 * Fill in the memory segment descriptors.
550	 */
551	for (i = 0; i < mem_cluster_cnt; i++) {
552		memsegp[i].start = mem_clusters[i].start;
553		memsegp[i].size = mem_clusters[i].size & PMAP_PA_MASK;
554	}
555
556	/*
557	 * If we have dump memory then assume the kernel stack is in high
558	 * memory and bounce
559	 */
560	if (dumpmem_vaddr != 0) {
561		memcpy((char *)dumpmem_vaddr, buf, sizeof(buf));
562		va = (caddr_t)dumpmem_vaddr;
563	} else {
564		va = (caddr_t)buf;
565	}
566	return (dump(dumpdev, dumplo, va, dbtob(1)));
567#else
568	return ENOSYS;
569#endif
570}
571
572/*
573 * This is called by main to set dumplo and dumpsize.
574 * Dumps always skip the first PAGE_SIZE of disk space
575 * in case there might be a disk label stored there.
576 * If there is extra space, put dump at the end to
577 * reduce the chance that swapping trashes it.
578 */
579void
580dumpconf(void)
581{
582	int nblks, dumpblks;	/* size of dump area */
583
584	if (dumpdev == NODEV ||
585	    (nblks = (bdevsw[major(dumpdev)].d_psize)(dumpdev)) == 0)
586		return;
587	if (nblks <= ctod(1))
588		return;
589
590	dumpblks = cpu_dumpsize();
591	if (dumpblks < 0)
592		return;
593	dumpblks += ctod(cpu_dump_mempagecnt());
594
595	/* If dump won't fit (incl. room for possible label), punt. */
596	if (dumpblks > (nblks - ctod(1)))
597		return;
598
599	/* Put dump at end of partition */
600	dumplo = nblks - dumpblks;
601
602	/* dumpsize is in page units, and doesn't include headers. */
603	dumpsize = cpu_dump_mempagecnt();
604}
605
606/*
607 * Doadump comes here after turning off memory management and
608 * getting on the dump stack, either when called above, or by
609 * the auto-restart code.
610 */
611#define BYTES_PER_DUMP  MAXPHYS /* must be a multiple of pagesize */
612
613void
614dumpsys(void)
615{
616	u_long totalbytesleft, bytes, i, n, memseg;
617	u_long maddr;
618	daddr_t blkno;
619	void *va;
620	int (*dump)(dev_t, daddr_t, caddr_t, size_t);
621	int error;
622
623	if (dumpdev == NODEV)
624		return;
625
626	/*
627	 * For dumps during autoconfiguration,
628	 * if dump device has already configured...
629	 */
630	if (dumpsize == 0)
631		dumpconf();
632	if (dumplo <= 0 || dumpsize == 0) {
633		printf("\ndump to dev %u,%u not possible\n", major(dumpdev),
634		    minor(dumpdev));
635		return;
636	}
637	printf("\ndumping to dev %u,%u offset %ld\n", major(dumpdev),
638	    minor(dumpdev), dumplo);
639
640#ifdef UVM_SWAP_ENCRYPT
641	uvm_swap_finicrypt_all();
642#endif
643
644	error = (*bdevsw[major(dumpdev)].d_psize)(dumpdev);
645	printf("dump ");
646	if (error == -1) {
647		printf("area unavailable\n");
648		return;
649	}
650
651	if ((error = cpu_dump()) != 0)
652		goto err;
653
654	totalbytesleft = ptoa(cpu_dump_mempagecnt());
655	blkno = dumplo + cpu_dumpsize();
656	dump = bdevsw[major(dumpdev)].d_dump;
657	error = 0;
658
659	bytes = n = i = memseg = 0;
660	maddr = 0;
661	va = 0;
662#if 0
663	for (memseg = 0; memseg < mem_cluster_cnt; memseg++) {
664		maddr = mem_clusters[memseg].start;
665		bytes = mem_clusters[memseg].size;
666
667		for (i = 0; i < bytes; i += n, totalbytesleft -= n) {
668			/* Print out how many MBs we have left to go. */
669			if ((totalbytesleft % (1024*1024)) < BYTES_PER_DUMP)
670				printf("%ld ", totalbytesleft / (1024 * 1024));
671
672			/* Limit size for next transfer. */
673			n = bytes - i;
674			if (n > BYTES_PER_DUMP)
675				n = BYTES_PER_DUMP;
676			if (maddr > 0xffffffff) {
677				va = (void *)dumpmem_vaddr;
678				if (n > dumpmem_sz)
679					n = dumpmem_sz;
680				memcpy(va, (void *)PMAP_DIRECT_MAP(maddr), n);
681			} else {
682				va = (void *)PMAP_DIRECT_MAP(maddr);
683			}
684
685			error = (*dump)(dumpdev, blkno, va, n);
686			if (error)
687				goto err;
688			maddr += n;
689			blkno += btodb(n);		/* XXX? */
690
691#if 0	/* XXX this doesn't work.  grr. */
692			/* operator aborting dump? */
693			if (sget() != NULL) {
694				error = EINTR;
695				break;
696			}
697#endif
698		}
699	}
700#endif
701
702 err:
703	switch (error) {
704
705	case ENXIO:
706		printf("device bad\n");
707		break;
708
709	case EFAULT:
710		printf("device not ready\n");
711		break;
712
713	case EINVAL:
714		printf("area improper\n");
715		break;
716
717	case EIO:
718		printf("i/o error\n");
719		break;
720
721	case EINTR:
722		printf("aborted from console\n");
723		break;
724
725	case 0:
726		printf("succeeded\n");
727		break;
728
729	default:
730		printf("error %d\n", error);
731		break;
732	}
733	printf("\n\n");
734	delay(5000000);		/* 5 seconds */
735}
736
737
738/*
739 * Size of memory segments, before any memory is stolen.
740 */
741phys_ram_seg_t mem_clusters[VM_PHYSSEG_MAX];
742int     mem_cluster_cnt;
743
744/*
745 * cpu_dumpsize: calculate size of machine-dependent kernel core dump headers.
746 */
747int
748cpu_dumpsize(void)
749{
750	int size;
751
752	size = ALIGN(sizeof(kcore_seg_t)) +
753	    ALIGN(mem_cluster_cnt * sizeof(phys_ram_seg_t));
754	if (roundup(size, dbtob(1)) != dbtob(1))
755		return (-1);
756
757	return (1);
758}
759
760u_long
761cpu_dump_mempagecnt(void)
762{
763	return 0;
764}
765
766int64_t dcache_line_size;	/* The minimum D cache line size */
767int64_t icache_line_size;	/* The minimum I cache line size */
768int64_t idcache_line_size;	/* The minimum cache line size */
769int64_t dczva_line_size;	/* The size of cache line the dc zva zeroes */
770
771void
772cache_setup(void)
773{
774	int dcache_line_shift, icache_line_shift, dczva_line_shift;
775	uint32_t ctr_el0;
776	uint32_t dczid_el0;
777
778	ctr_el0 = READ_SPECIALREG(ctr_el0);
779
780	/* Read the log2 words in each D cache line */
781	dcache_line_shift = CTR_DLINE_SIZE(ctr_el0);
782	/* Get the D cache line size */
783	dcache_line_size = sizeof(int) << dcache_line_shift;
784
785	/* And the same for the I cache */
786	icache_line_shift = CTR_ILINE_SIZE(ctr_el0);
787	icache_line_size = sizeof(int) << icache_line_shift;
788
789	idcache_line_size = MIN(dcache_line_size, icache_line_size);
790
791	dczid_el0 = READ_SPECIALREG(dczid_el0);
792
793	/* Check if dc zva is not prohibited */
794	if (dczid_el0 & DCZID_DZP)
795		dczva_line_size = 0;
796	else {
797		/* Same as with above calculations */
798		dczva_line_shift = DCZID_BS_SIZE(dczid_el0);
799		dczva_line_size = sizeof(int) << dczva_line_shift;
800	}
801}
802
803uint64_t mmap_start;
804uint32_t mmap_size;
805uint32_t mmap_desc_size;
806uint32_t mmap_desc_ver;
807
808EFI_MEMORY_DESCRIPTOR *mmap;
809
810void	collect_kernel_args(const char *);
811void	process_kernel_args(void);
812
813int	pmap_bootstrap_bs_map(bus_space_tag_t, bus_addr_t,
814	    bus_size_t, int, bus_space_handle_t *);
815
816void
817initarm(struct arm64_bootparams *abp)
818{
819	long kernbase = (long)_start & ~PAGE_MASK;
820	long kvo = abp->kern_delta;
821	paddr_t memstart, memend;
822	paddr_t startpa, endpa, pa;
823	vaddr_t vstart, va;
824	struct fdt_head *fh;
825	void *config = abp->arg2;
826	void *fdt = NULL;
827	struct fdt_reg reg;
828	void *node;
829	EFI_PHYSICAL_ADDRESS system_table = 0;
830	int (*map_func_save)(bus_space_tag_t, bus_addr_t, bus_size_t, int,
831	    bus_space_handle_t *);
832	int i;
833
834	/*
835	 * Set the per-CPU pointer with a backup in tpidr_el1 to be
836	 * loaded when entering the kernel from userland.
837	 */
838	__asm volatile("mov x18, %0\n"
839	    "msr tpidr_el1, %0" :: "r"(&cpu_info_primary));
840
841	cache_setup();
842
843	/* The bootloader has loaded us into a 64MB block. */
844	memstart = KERNBASE + kvo;
845	memend = memstart + 64 * 1024 * 1024;
846
847	/* Bootstrap enough of pmap to enter the kernel proper. */
848	vstart = pmap_bootstrap(kvo, abp->kern_l1pt,
849	    kernbase, esym, memstart, memend);
850
851	/* Map the FDT header to determine its size. */
852	va = vstart;
853	startpa = trunc_page((paddr_t)config);
854	endpa = round_page((paddr_t)config + sizeof(struct fdt_head));
855	for (pa = startpa; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE)
856		pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB);
857	fh = (void *)(vstart + ((paddr_t)config - startpa));
858	if (betoh32(fh->fh_magic) != FDT_MAGIC || betoh32(fh->fh_size) == 0)
859		panic("%s: no FDT", __func__);
860
861	/* Map the remainder of the FDT. */
862	endpa = round_page((paddr_t)config + betoh32(fh->fh_size));
863	for (; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE)
864		pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB);
865	config = (void *)(vstart + ((paddr_t)config - startpa));
866	vstart = va;
867
868	if (!fdt_init(config))
869		panic("%s: corrupt FDT", __func__);
870
871	node = fdt_find_node("/chosen");
872	if (node != NULL) {
873		char *prop;
874		int len;
875		static uint8_t lladdr[6];
876
877		len = fdt_node_property(node, "bootargs", &prop);
878		if (len > 0)
879			collect_kernel_args(prop);
880
881		len = fdt_node_property(node, "openbsd,boothowto", &prop);
882		if (len == sizeof(boothowto))
883			boothowto = bemtoh32((uint32_t *)prop);
884
885		len = fdt_node_property(node, "openbsd,bootduid", &prop);
886		if (len == sizeof(bootduid))
887			memcpy(bootduid, prop, sizeof(bootduid));
888
889		len = fdt_node_property(node, "openbsd,bootmac", &prop);
890		if (len == sizeof(lladdr)) {
891			memcpy(lladdr, prop, sizeof(lladdr));
892			bootmac = lladdr;
893		}
894
895		len = fdt_node_property(node, "openbsd,sr-bootuuid", &prop);
896#if NSOFTRAID > 0
897		if (len == sizeof(sr_bootuuid))
898			memcpy(&sr_bootuuid, prop, sizeof(sr_bootuuid));
899#endif
900		if (len > 0)
901			explicit_bzero(prop, len);
902
903		len = fdt_node_property(node, "openbsd,sr-bootkey", &prop);
904#if NSOFTRAID > 0
905		if (len == sizeof(sr_bootkey))
906			memcpy(&sr_bootkey, prop, sizeof(sr_bootkey));
907#endif
908		if (len > 0)
909			explicit_bzero(prop, len);
910
911		len = fdt_node_property(node, "openbsd,uefi-mmap-start", &prop);
912		if (len == sizeof(mmap_start))
913			mmap_start = bemtoh64((uint64_t *)prop);
914		len = fdt_node_property(node, "openbsd,uefi-mmap-size", &prop);
915		if (len == sizeof(mmap_size))
916			mmap_size = bemtoh32((uint32_t *)prop);
917		len = fdt_node_property(node, "openbsd,uefi-mmap-desc-size", &prop);
918		if (len == sizeof(mmap_desc_size))
919			mmap_desc_size = bemtoh32((uint32_t *)prop);
920		len = fdt_node_property(node, "openbsd,uefi-mmap-desc-ver", &prop);
921		if (len == sizeof(mmap_desc_ver))
922			mmap_desc_ver = bemtoh32((uint32_t *)prop);
923
924		len = fdt_node_property(node, "openbsd,uefi-system-table", &prop);
925		if (len == sizeof(system_table))
926			system_table = bemtoh64((uint64_t *)prop);
927
928		len = fdt_node_property(node, "openbsd,dma-constraint", &prop);
929		if (len == sizeof(dma_constraint)) {
930			dma_constraint.ucr_low = bemtoh64((uint64_t *)prop);
931			dma_constraint.ucr_high = bemtoh64((uint64_t *)prop + 1);
932		}
933	}
934
935	process_kernel_args();
936
937	proc0paddr = (struct user *)abp->kern_stack;
938
939	msgbufaddr = (caddr_t)vstart;
940	msgbufphys = pmap_steal_avail(round_page(MSGBUFSIZE), PAGE_SIZE, NULL);
941	vstart += round_page(MSGBUFSIZE);
942
943	zero_page = vstart;
944	vstart += MAXCPUS * PAGE_SIZE;
945	copy_src_page = vstart;
946	vstart += MAXCPUS * PAGE_SIZE;
947	copy_dst_page = vstart;
948	vstart += MAXCPUS * PAGE_SIZE;
949
950	/* Relocate the FDT to safe memory. */
951	if (fdt_get_size(config) != 0) {
952		uint32_t csize, size = round_page(fdt_get_size(config));
953		paddr_t pa;
954		vaddr_t va;
955
956		pa = pmap_steal_avail(size, PAGE_SIZE, NULL);
957		memcpy((void *)pa, config, size); /* copy to physical */
958		for (va = vstart, csize = size; csize > 0;
959		    csize -= PAGE_SIZE, va += PAGE_SIZE, pa += PAGE_SIZE)
960			pmap_kenter_cache(va, pa, PROT_READ, PMAP_CACHE_WB);
961
962		fdt = (void *)vstart;
963		vstart += size;
964	}
965
966	/* Relocate the EFI memory map too. */
967	if (mmap_start != 0) {
968		uint32_t csize, size = round_page(mmap_size);
969		paddr_t pa, startpa, endpa;
970		vaddr_t va;
971
972		startpa = trunc_page(mmap_start);
973		endpa = round_page(mmap_start + mmap_size);
974		for (pa = startpa, va = vstart; pa < endpa;
975		    pa += PAGE_SIZE, va += PAGE_SIZE)
976			pmap_kenter_cache(va, pa, PROT_READ, PMAP_CACHE_WB);
977		pa = pmap_steal_avail(size, PAGE_SIZE, NULL);
978		memcpy((void *)pa, (caddr_t)vstart + (mmap_start - startpa),
979		    mmap_size); /* copy to physical */
980		pmap_kremove(vstart, endpa - startpa);
981
982		for (va = vstart, csize = size; csize > 0;
983		    csize -= PAGE_SIZE, va += PAGE_SIZE, pa += PAGE_SIZE)
984			pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB);
985
986		mmap = (void *)vstart;
987		vstart += size;
988	}
989
990	/* No more KVA stealing after this point. */
991	virtual_avail = vstart;
992
993	/* Now we can reinit the FDT, using the virtual address. */
994	if (fdt)
995		fdt_init(fdt);
996
997	map_func_save = arm64_bs_tag._space_map;
998	arm64_bs_tag._space_map = pmap_bootstrap_bs_map;
999
1000	consinit();
1001
1002	arm64_bs_tag._space_map = map_func_save;
1003
1004	pmap_avail_fixup();
1005
1006	uvmexp.pagesize = PAGE_SIZE;
1007	uvm_setpagesize();
1008
1009	/* Make what's left of the initial 64MB block available to UVM. */
1010	pmap_physload_avail();
1011
1012	/* Make all other physical memory available to UVM. */
1013	if (mmap && mmap_desc_ver == EFI_MEMORY_DESCRIPTOR_VERSION) {
1014		EFI_MEMORY_DESCRIPTOR *desc = mmap;
1015
1016		/*
1017		 * Load all memory marked as EfiConventionalMemory,
1018		 * EfiBootServicesCode or EfiBootServicesData.
1019		 * The initial 64MB memory block should be marked as
1020		 * EfiLoaderData so it won't be added here.
1021		 */
1022		for (i = 0; i < mmap_size / mmap_desc_size; i++) {
1023#ifdef MMAP_DEBUG
1024			printf("type 0x%x pa 0x%llx va 0x%llx pages 0x%llx attr 0x%llx\n",
1025			    desc->Type, desc->PhysicalStart,
1026			    desc->VirtualStart, desc->NumberOfPages,
1027			    desc->Attribute);
1028#endif
1029			if (desc->Type == EfiConventionalMemory ||
1030			    desc->Type == EfiBootServicesCode ||
1031			    desc->Type == EfiBootServicesData) {
1032				reg.addr = desc->PhysicalStart;
1033				reg.size = ptoa(desc->NumberOfPages);
1034				memreg_add(&reg);
1035			}
1036			desc = NextMemoryDescriptor(desc, mmap_desc_size);
1037		}
1038	} else {
1039		node = fdt_find_node("/memory");
1040		if (node == NULL)
1041			panic("%s: no memory specified", __func__);
1042
1043		for (i = 0; nmemreg < nitems(memreg); i++) {
1044			if (fdt_get_reg(node, i, &reg))
1045				break;
1046			if (reg.size == 0)
1047				continue;
1048			memreg_add(&reg);
1049		}
1050	}
1051
1052	/* Remove reserved memory. */
1053	node = fdt_find_node("/reserved-memory");
1054	if (node) {
1055		for (node = fdt_child_node(node); node;
1056		    node = fdt_next_node(node)) {
1057			char *no_map;
1058			if (fdt_node_property(node, "no-map", &no_map) < 0)
1059				continue;
1060			if (fdt_get_reg(node, 0, &reg))
1061				continue;
1062			if (reg.size == 0)
1063				continue;
1064			memreg_remove(&reg);
1065		}
1066	}
1067
1068	/* Remove the initial 64MB block. */
1069	reg.addr = memstart;
1070	reg.size = memend - memstart;
1071	memreg_remove(&reg);
1072
1073	for (i = 0; i < nmemreg; i++) {
1074		paddr_t start = memreg[i].addr;
1075		paddr_t end = start + memreg[i].size;
1076
1077		uvm_page_physload(atop(start), atop(end),
1078		    atop(start), atop(end), 0);
1079		physmem += atop(end - start);
1080	}
1081
1082	kmeminit_nkmempages();
1083
1084	/*
1085	 * Make sure that we have enough KVA to initialize UVM.  In
1086	 * particular, we need enough KVA to be able to allocate the
1087	 * vm_page structures and nkmempages for malloc(9).
1088	 */
1089	pmap_growkernel(VM_MIN_KERNEL_ADDRESS + 1024 * 1024 * 1024 +
1090	    physmem * sizeof(struct vm_page) + ptoa(nkmempages));
1091
1092#ifdef DDB
1093	db_machine_init();
1094
1095	/* Firmware doesn't load symbols. */
1096	ddb_init();
1097
1098	if (boothowto & RB_KDB)
1099		db_enter();
1100#endif
1101
1102	softintr_init();
1103	splraise(IPL_IPI);
1104}
1105
1106char bootargs[256];
1107
1108void
1109collect_kernel_args(const char *args)
1110{
1111	/* Make a local copy of the bootargs */
1112	strlcpy(bootargs, args, sizeof(bootargs));
1113}
1114
1115void
1116process_kernel_args(void)
1117{
1118	char *cp = bootargs;
1119
1120	if (*cp == 0)
1121		return;
1122
1123	/* Skip the kernel image filename */
1124	while (*cp != ' ' && *cp != 0)
1125		cp++;
1126
1127	if (*cp != 0)
1128		*cp++ = 0;
1129
1130	while (*cp == ' ')
1131		cp++;
1132
1133	boot_args = cp;
1134
1135	printf("bootargs: %s\n", boot_args);
1136
1137	/* Setup pointer to boot flags */
1138	while (*cp != '-')
1139		if (*cp++ == '\0')
1140			return;
1141
1142	while (*cp != 0) {
1143		switch (*cp) {
1144		case 'a':
1145			boothowto |= RB_ASKNAME;
1146			break;
1147		case 'c':
1148			boothowto |= RB_CONFIG;
1149			break;
1150		case 'd':
1151			boothowto |= RB_KDB;
1152			break;
1153		case 's':
1154			boothowto |= RB_SINGLE;
1155			break;
1156		default:
1157			printf("unknown option `%c'\n", *cp);
1158			break;
1159		}
1160		cp++;
1161	}
1162}
1163
1164/*
1165 * Allow bootstrap to steal KVA after machdep has given it back to pmap.
1166 */
1167int
1168pmap_bootstrap_bs_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size,
1169    int flags, bus_space_handle_t *bshp)
1170{
1171	u_long startpa, pa, endpa;
1172	vaddr_t va;
1173
1174	va = virtual_avail;	/* steal memory from virtual avail. */
1175
1176	startpa = trunc_page(bpa);
1177	endpa = round_page((bpa + size));
1178
1179	*bshp = (bus_space_handle_t)(va + (bpa - startpa));
1180
1181	for (pa = startpa; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE)
1182		pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE,
1183		    PMAP_CACHE_DEV_NGNRNE);
1184
1185	virtual_avail = va;
1186
1187	return 0;
1188}
1189
1190void
1191memreg_add(const struct fdt_reg *reg)
1192{
1193	int i;
1194
1195	for (i = 0; i < nmemreg; i++) {
1196		if (reg->addr == memreg[i].addr + memreg[i].size) {
1197			memreg[i].size += reg->size;
1198			return;
1199		}
1200		if (reg->addr + reg->size == memreg[i].addr) {
1201			memreg[i].addr = reg->addr;
1202			memreg[i].size += reg->size;
1203			return;
1204		}
1205	}
1206
1207	if (nmemreg >= nitems(memreg))
1208		return;
1209
1210	memreg[nmemreg++] = *reg;
1211}
1212
1213void
1214memreg_remove(const struct fdt_reg *reg)
1215{
1216	uint64_t start = reg->addr;
1217	uint64_t end = reg->addr + reg->size;
1218	int i, j;
1219
1220	for (i = 0; i < nmemreg; i++) {
1221		uint64_t memstart = memreg[i].addr;
1222		uint64_t memend = memreg[i].addr + memreg[i].size;
1223
1224		if (end <= memstart)
1225			continue;
1226		if (start >= memend)
1227			continue;
1228
1229		if (start <= memstart)
1230			memstart = MIN(end, memend);
1231		if (end >= memend)
1232			memend = MAX(start, memstart);
1233
1234		if (start > memstart && end < memend) {
1235			if (nmemreg < nitems(memreg)) {
1236				memreg[nmemreg].addr = end;
1237				memreg[nmemreg].size = memend - end;
1238				nmemreg++;
1239			}
1240			memend = start;
1241		}
1242		memreg[i].addr = memstart;
1243		memreg[i].size = memend - memstart;
1244	}
1245
1246	/* Remove empty slots. */
1247	for (i = nmemreg - 1; i >= 0; i--) {
1248		if (memreg[i].size == 0) {
1249			for (j = i; (j + 1) < nmemreg; j++)
1250				memreg[j] = memreg[j + 1];
1251			nmemreg--;
1252		}
1253	}
1254}
1255