linux-master/mm/gup.c

3067 static int gup_fast_devmap_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
3074 	fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
3078 	if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
3085 static int gup_fast_devmap_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
3092 	fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
3096 	if (unlikely(pud_val(orig) != pud_val(*pudp))) {
3103 static int gup_fast_devmap_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
3120 static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
3128 	if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
3131 	if (pmd_devmap(orig)) {
3134 		return gup_fast_devmap_pmd_leaf(orig, pmdp, addr, end, flags,
3138 	page = pmd_page(orig);
3145 	if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
3154 	if (!pmd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) {
3164 static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
3172 	if (!pud_access_permitted(orig, flags & FOLL_WRITE))
3175 	if (pud_devmap(orig)) {
3178 		return gup_fast_devmap_pud_leaf(orig, pudp, addr, end, flags,
3182 	page = pud_page(orig);
3189 	if (unlikely(pud_val(orig) != pud_val(*pudp))) {
3199 	if (!pud_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) {
3209 static int gup_fast_pgd_leaf(pgd_t orig, pgd_t *pgdp, unsigned long addr,
3217 	if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
3220 	BUILD_BUG_ON(pgd_devmap(orig));
3222 	page = pgd_page(orig);
3229 	if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
3234 	if (!pgd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) {