linux-master/mm/gup.c

580 		unsigned long address, pmd_t *pmd, unsigned int flags,
594 	ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
696 	pmd_t *pmd, pmdval;
701 	pmd = pmd_offset(pudp, address);
702 	pmdval = pmdp_get_lockless(pmd);
708 		ptl = pmd_lock(mm, pmd);
709 		page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
716 		return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
721 	ptl = pmd_lock(mm, pmd);
722 	if (unlikely(!pmd_present(*pmd))) {
726 	if (unlikely(!pmd_trans_huge(*pmd))) {
728 		return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
732 		split_huge_pmd(vma, pmd, address);
733 		/* If pmd was left empty, stuff a page table in there quickly */
734 		return pte_alloc(mm, pmd) ? ERR_PTR(-ENOMEM) :
735 			follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
737 	page = follow_trans_huge_pmd(vma, address, pmd, flags);
866 	pmd_t *pmd;
886 	pmd = pmd_offset(pud, address);
887 	if (!pmd_present(*pmd))
889 	pte = pte_offset_map(pmd, address);
2574  * walking a pgtable page that is being freed (pte is still valid but pmd
2576  * also check pmd here to make sure pmd doesn't change (corresponds to
2579 static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
2587 	ptem = ptep = pte_offset_map(&pmd, addr);
2632 		if (unlikely(pmd_val(pmd) != pmd_val(*pmdp)) ||
2685 static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
3013 		pmd_t pmd = pmdp_get_lockless(pmdp);
3016 		if (!pmd_present(pmd))
3019 		if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
3020 			     pmd_devmap(pmd))) {
3022 			if (pmd_protnone(pmd))
3025 			if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
3029 		} else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
3032 			 * pmd format and THP pmd format
3034 			if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
3037 		} else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr))