riscv/riscv/pmap.c

312 static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva,
460 	pt_entry_t *l3;
463 	l3 = (pd_entry_t *)PHYS_TO_DMAP(phys);
465 	return (&l3[pmap_l3_index(va)]);
1077 	pt_entry_t *l3p, l3;
1083 	if (l3p != NULL && (l3 = pmap_load(l3p)) != 0) {
1084 		if ((l3 & PTE_W) != 0 || (prot & VM_PROT_WRITE) == 0) {
1085 			m = PTE_TO_VM_PAGE(l3);
1104 	pt_entry_t *l3;
1129 		l3 = pmap_l2_to_l3(&l2e, va);
1130 		if (l3 == NULL)
1131 			panic("pmap_kextract: No l3...");
1132 		pa = PTE_TO_PHYS(pmap_load(l3));
1146 	pt_entry_t *l3;
1159 		l3 = pmap_l3(kernel_pmap, va);
1160 		KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va));
1165 		pmap_store(l3, entry);
1187 	pt_entry_t *l3;
1189 	l3 = pmap_l3(kernel_pmap, va);
1190 	KASSERT(l3 != NULL, ("pmap_kremove: Invalid address"));
1192 	pmap_clear(l3);
1199 	pt_entry_t *l3;
1209 		l3 = pmap_l3(kernel_pmap, va);
1210 		KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va));
1211 		pmap_clear(l3);
1251 	pt_entry_t *l3, pa;
1263 		l3 = pmap_l3(kernel_pmap, va);
1267 		pmap_store(l3, entry);
1282 	pt_entry_t *l3;
1288 		l3 = pmap_l3(kernel_pmap, va);
1289 		KASSERT(l3 != NULL, ("pmap_kremove: Invalid address"));
1290 		pmap_clear(l3);
2400 			    ("pmap_remove_l2: l3 page not promoted"));
2403 			    ("pmap_remove_l2: l3 page ref count error"));
2416 pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va,
2424 	old_l3 = pmap_load_clear(l3);
2461 	pt_entry_t *l3;
2535 		for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++,
2537 			if (pmap_load(l3) == 0) {
2546 			if (pmap_remove_l3(pmap, l3, sva, l2e, &free, &lock)) {
2580 	pt_entry_t *l3, l3e;
2611 		l3 = pmap_l2_to_l3(l2, pv->pv_va);
2612 		l3e = pmap_load_clear(l3);
2643 	pt_entry_t *l3, l3e, mask;
2732 		for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++,
2734 			l3e = pmap_load(l3);
2744 			if (!atomic_fcmpset_long(l3, &l3e, l3e & ~mask))
2918 	pt_entry_t all_l3e_PTE_A, *firstl3, firstl3e, *l3, l3e;
2967 	for (l3 = firstl3 + Ln_ENTRIES - 1; l3 > firstl3; l3--) {
2968 		l3e = pmap_load(l3);
2977 			if (atomic_fcmpset_64(l3, &l3e, l3e & ~PTE_W)) {
3048 	pt_entry_t *l3;
3106 		l3 = pmap_l2_to_l3(l2, va);
3122 		l3 = pmap_l3(pmap, va);
3127 	orig_l3 = pmap_load(l3);
3181 		orig_l3 = pmap_load_clear(l3);
3245 		orig_l3 = pmap_load_store(l3, new_l3);
3253 		pmap_store(l3, new_l3);
3358 	pd_entry_t *l2, *l3, oldl2;
3399 				l3 = pmap_l2_to_l3(l2, sva);
3400 				if ((pmap_load(l3) & PTE_V) != 0 &&
3401 				    pmap_remove_l3(pmap, l3, sva, oldl2, &free,
3562 	pt_entry_t *l3, newl3;
3612 		l3 = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte));
3613 		l3 = &l3[pmap_l3_index(va)];
3616 		l3 = pmap_l3(kernel_pmap, va);
3618 	if (l3 == NULL)
3619 		panic("pmap_enter_quick_locked: No l3");
3620 	if (pmap_load(l3) != 0) {
3660 	pmap_store(l3, newl3);
3715 	pt_entry_t *l3, l3e;
3773 		for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++,
3775 			if ((l3e = pmap_load(l3)) == 0)
3778 				panic("pmap_unwire: l3 %#jx is missing "
3786 			pmap_clear_bits(l3, PTE_SW_WIRED);
3965 	pt_entry_t *l3;
3991 		l3 = pmap_l2_to_l3(l2, pv->pv_va);
3992 		if ((pmap_load(l3) & PTE_SW_WIRED) != 0)
4213 	pt_entry_t *l3, mask;
4245 		l3 = pmap_l2_to_l3(l2, pv->pv_va);
4246 		rv = (pmap_load(l3) & mask) == mask;
4310 	pt_entry_t *l3;
4319 	l3 = pmap_l3(pmap, addr);
4320 	if (l3 != NULL && pmap_load(l3) == 0) {
4352 	pt_entry_t *l3, oldl3, newl3;
4408 		l3 = pmap_l2_to_l3(l2, pv->pv_va);
4409 		oldl3 = pmap_load(l3);
4413 			if (!atomic_fcmpset_long(l3, &oldl3, newl3))
4451 	pt_entry_t *l3, l3e;
4554 		l3 = pmap_l2_to_l3(l2, pv->pv_va);
4555 		l3e = pmap_load(l3);
4566 				pmap_clear_bits(l3, PTE_A);
4609 	pt_entry_t *l3;
4657 			l3 = pmap_l2_to_l3(l2, va);
4658 			pmap_clear_bits(l3, PTE_D | PTE_W);
4680 		l3 = pmap_l2_to_l3(l2, pv->pv_va);
4681 		if ((pmap_load(l3) & (PTE_D | PTE_W)) == (PTE_D | PTE_W)) {
4682 			pmap_clear_bits(l3, PTE_D | PTE_W);
4752 	pt_entry_t *l3, l3e;
4792 		l3 = pmap_l2_to_l3(l2, tmpva);
4793 		if (l3 == NULL || ((l3e = pmap_load(l3)) & PTE_V) == 0)
4813 	pt_entry_t *l2, *l3, tpte;
4825 			l3 = pmap_l2_to_l3(l2, addr);
4826 			tpte = pmap_load(l3);
5044     pt_entry_t **l3)
5057 		*l3 = NULL;
5069 		*l3 = NULL;
5073 	/* Get l3 page table entry. */
5074 	*l3 = pmap_l2_to_l3(l2p, va);
5166 	pt_entry_t *l3, l3e;
5222 			l3 = (pd_entry_t *)PHYS_TO_DMAP(pa);
5226 				l3e = l3[k];