mm/pat/set_memory.c

67  * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
294 static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx)
296 	if (cpa->flags & CPA_PAGES_ARRAY) {
297 		struct page *page = cpa->pages[idx];
305 	if (cpa->flags & CPA_ARRAY)
306 		return cpa->vaddr[idx];
308 	return *cpa->vaddr + idx * PAGE_SIZE;
392 	struct cpa_data *cpa = data;
395 	for (i = 0; i < cpa->numpages; i++)
396 		flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i)));
401 	struct cpa_data *cpa = data;
411 	if (cpa->force_flush_all || cpa->numpages > tlb_single_page_flush_ceiling)
414 		on_each_cpu(__cpa_flush_tlb, cpa, 1);
420 	for (i = 0; i < cpa->numpages; i++) {
421 		unsigned long addr = __cpa_addr(cpa, i);
749 static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
754 	if (!cpa->pgd)
757 		pgd = cpa->pgd + pgd_index(address);
881 				     struct cpa_data *cpa)
893 	tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw);
921 	if (numpages < cpa->numpages)
922 		cpa->numpages = numpages;
926 	 * Convert protection attributes to 4k-format, as cpa->mask* are set
933 	pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
934 	pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
951 	cpa->pfn = pfn;
973 		cpa->force_static_prot = 1;
994 	if (address != lpaddr || cpa->numpages != numpages)
1022 	cpa->flags |= CPA_FLUSHTLB;
1028 				   struct cpa_data *cpa)
1032 	if (cpa->force_split)
1036 	do_split = __should_split_large_page(kpte, address, cpa);
1042 static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,
1053 	if (!cpa->force_static_prot)
1079 __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
1094 	tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw);
1142 		split_set_pte(cpa, pbase + i, pfn, ref_prot, lpaddr, lpinc);
1184 static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
1197 	if (__split_large_page(cpa, kpte, address, base))
1360 static void populate_pte(struct cpa_data *cpa,
1371 		set_pte(pte, pfn_pte(cpa->pfn, pgprot));
1374 		cpa->pfn++;
1379 static long populate_pmd(struct cpa_data *cpa,
1406 		populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);
1430 		set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn,
1434 		cpa->pfn  += PMD_SIZE >> PAGE_SHIFT;
1447 		populate_pte(cpa, start, end, num_pages - cur_pages,
1453 static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d,
1461 	end = start + (cpa->numpages << PAGE_SHIFT);
1473 		cur_pages = min_t(int, (int)cpa->numpages, cur_pages);
1484 		cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,
1493 	if (cpa->numpages == cur_pages)
1503 		set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn,
1507 		cpa->pfn  += PUD_SIZE >> PAGE_SHIFT;
1521 		tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,
1535 static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
1543 	pgd_entry = cpa->pgd + pgd_index(addr);
1565 	pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr);
1566 	pgprot_val(pgprot) |=  pgprot_val(cpa->mask_set);
1568 	ret = populate_pud(cpa, addr, p4d, pgprot);
1576 				addr + (cpa->numpages << PAGE_SHIFT));
1580 	cpa->numpages = ret;
1584 static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
1587 	if (cpa->pgd) {
1593 		return populate_pgd(cpa, vaddr);
1600 		cpa->numpages = 1;
1607 	 * Also set numpages to '1' indicating that we processed cpa req for
1613 		cpa->numpages = 1;
1614 		cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
1617 	} else if (__cpa_pfn_in_highmap(cpa->pfn)) {
1622 			"vaddr = %lx cpa->vaddr = %lx\n", vaddr,
1623 			*cpa->vaddr);
1629 static int __change_page_attr(struct cpa_data *cpa, int primary)
1637 	address = __cpa_addr(cpa, cpa->curpage);
1639 	kpte = _lookup_address_cpa(cpa, address, &level, &nx, &rw);
1641 		return __cpa_process_fault(cpa, address, primary);
1645 		return __cpa_process_fault(cpa, address, primary);
1653 		pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
1654 		pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
1672 		cpa->pfn = pfn;
1678 			cpa->flags |= CPA_FLUSHTLB;
1680 		cpa->numpages = 1;
1688 	do_split = should_split_large_page(kpte, address, cpa);
1691 	 * return. cp->numpages and cpa->tlbflush have been updated in
1700 	err = split_large_page(cpa, kpte, address);
1707 static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary);
1712 static int cpa_process_alias(struct cpa_data *cpa)
1715 	unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
1719 	if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
1726 	vaddr = __cpa_addr(cpa, cpa->curpage);
1730 		alias_cpa = *cpa;
1741 		cpa->force_flush_all = 1;
1755 	    __cpa_pfn_in_highmap(cpa->pfn)) {
1756 		unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
1758 		alias_cpa = *cpa;
1772 		cpa->force_flush_all = 1;
1784 static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary)
1786 	unsigned long numpages = cpa->numpages;
1793 	if (!(pgprot_val(cpa->mask_set) | pgprot_val(cpa->mask_clr)) &&
1794 	    !cpa->force_split)
1802 		cpa->numpages = rempages;
1804 		if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
1805 			cpa->numpages = 1;
1809 		ret = __change_page_attr(cpa, primary);
1815 		if (primary && !(cpa->flags & CPA_NO_CHECK_ALIAS)) {
1816 			ret = cpa_process_alias(cpa);
1826 		BUG_ON(cpa->numpages > rempages || !cpa->numpages);
1827 		rempages -= cpa->numpages;
1828 		cpa->curpage += cpa->numpages;
1833 	cpa->numpages = numpages;
1842 	struct cpa_data cpa;
1845 	memset(&cpa, 0, sizeof(cpa));
1884 	cpa.vaddr = addr;
1885 	cpa.pages = pages;
1886 	cpa.numpages = numpages;
1887 	cpa.mask_set = mask_set;
1888 	cpa.mask_clr = mask_clr;
1889 	cpa.flags = in_flag;
1890 	cpa.curpage = 0;
1891 	cpa.force_split = force_split;
1893 	ret = __change_page_attr_set_clr(&cpa, 1);
1898 	if (!(cpa.flags & CPA_FLUSHTLB))
1915 	cpa_flush(&cpa, cache);
2176 	struct cpa_data cpa;
2183 	memset(&cpa, 0, sizeof(cpa));
2184 	cpa.vaddr = &addr;
2185 	cpa.numpages = numpages;
2186 	cpa.mask_set = enc ? pgprot_encrypted(empty) : pgprot_decrypted(empty);
2187 	cpa.mask_clr = enc ? pgprot_decrypted(empty) : pgprot_encrypted(empty);
2188 	cpa.pgd = init_mm.pgd;
2196 		cpa_flush(&cpa, x86_platform.guest.enc_cache_flush_required());
2202 	ret = __change_page_attr_set_clr(&cpa, 1);
2211 	cpa_flush(&cpa, 0);
2365 	struct cpa_data cpa = { .vaddr = &tempaddr,
2378 	return __change_page_attr_set_clr(&cpa, 1);
2384 	struct cpa_data cpa = { .vaddr = &tempaddr,
2397 	return __change_page_attr_set_clr(&cpa, 1);
2461 	struct cpa_data cpa = {
2477 		cpa.mask_clr = pgprot_encrypted(cpa.mask_clr);
2479 	cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags);
2481 	retval = __change_page_attr_set_clr(&cpa, 1);
2504 	struct cpa_data cpa = {
2516 	retval = __change_page_attr_set_clr(&cpa, 1);
2527 #include "cpa-test.c"