Lines Matching defs:pgmap
46 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)
48 if (pgmap->type == MEMORY_DEVICE_FS_DAX)
52 static void devmap_managed_enable_get(struct dev_pagemap *pgmap)
54 if (pgmap->type == MEMORY_DEVICE_FS_DAX)
58 static void devmap_managed_enable_get(struct dev_pagemap *pgmap)
61 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)
73 static unsigned long pfn_first(struct dev_pagemap *pgmap, int range_id)
75 struct range *range = &pgmap->ranges[range_id];
80 return pfn + vmem_altmap_offset(pgmap_altmap(pgmap));
83 bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn)
87 for (i = 0; i < pgmap->nr_range; i++) {
88 struct range *range = &pgmap->ranges[i];
92 return pfn >= pfn_first(pgmap, i);
98 static unsigned long pfn_end(struct dev_pagemap *pgmap, int range_id)
100 const struct range *range = &pgmap->ranges[range_id];
105 static unsigned long pfn_len(struct dev_pagemap *pgmap, unsigned long range_id)
107 return (pfn_end(pgmap, range_id) -
108 pfn_first(pgmap, range_id)) >> pgmap->vmemmap_shift;
111 static void pageunmap_range(struct dev_pagemap *pgmap, int range_id)
113 struct range *range = &pgmap->ranges[range_id];
117 first_page = pfn_to_page(pfn_first(pgmap, range_id));
123 if (pgmap->type == MEMORY_DEVICE_PRIVATE) {
128 pgmap_altmap(pgmap));
137 void memunmap_pages(struct dev_pagemap *pgmap)
141 percpu_ref_kill(&pgmap->ref);
142 if (pgmap->type != MEMORY_DEVICE_PRIVATE &&
143 pgmap->type != MEMORY_DEVICE_COHERENT)
144 for (i = 0; i < pgmap->nr_range; i++)
145 percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i));
147 wait_for_completion(&pgmap->done);
149 for (i = 0; i < pgmap->nr_range; i++)
150 pageunmap_range(pgmap, i);
151 percpu_ref_exit(&pgmap->ref);
153 WARN_ONCE(pgmap->altmap.alloc, "failed to free all reserved pages\n");
154 devmap_managed_enable_put(pgmap);
165 struct dev_pagemap *pgmap = container_of(ref, struct dev_pagemap, ref);
167 complete(&pgmap->done);
170 static int pagemap_range(struct dev_pagemap *pgmap, struct mhp_params *params,
173 const bool is_private = pgmap->type == MEMORY_DEVICE_PRIVATE;
174 struct range *range = &pgmap->ranges[range_id];
178 if (WARN_ONCE(pgmap_altmap(pgmap) && range_id > 0,
207 PHYS_PFN(range->end), pgmap, GFP_KERNEL));
270 PHYS_PFN(range_len(range)), pgmap);
271 if (pgmap->type != MEMORY_DEVICE_PRIVATE &&
272 pgmap->type != MEMORY_DEVICE_COHERENT)
273 percpu_ref_get_many(&pgmap->ref, pfn_len(pgmap, range_id));
292 void *memremap_pages(struct dev_pagemap *pgmap, int nid)
295 .altmap = pgmap_altmap(pgmap),
296 .pgmap = pgmap,
299 const int nr_range = pgmap->nr_range;
305 switch (pgmap->type) {
311 if (!pgmap->ops || !pgmap->ops->migrate_to_ram) {
315 if (!pgmap->ops->page_free) {
319 if (!pgmap->owner) {
325 if (!pgmap->ops->page_free) {
329 if (!pgmap->owner) {
347 WARN(1, "Invalid pgmap type %d\n", pgmap->type);
351 init_completion(&pgmap->done);
352 error = percpu_ref_init(&pgmap->ref, dev_pagemap_percpu_release, 0,
357 devmap_managed_enable_get(pgmap);
360 * Clear the pgmap nr_range as it will be incremented for each
364 pgmap->nr_range = 0;
367 error = pagemap_range(pgmap, ¶ms, i, nid);
370 pgmap->nr_range++;
374 memunmap_pages(pgmap);
375 pgmap->nr_range = nr_range;
379 return __va(pgmap->ranges[0].start);
386 * @pgmap: pointer to a struct dev_pagemap
389 * 1/ At a minimum the range and type members of @pgmap must be initialized
393 * PGMAP_ALTMAP_VALID must be set in pgmap->flags.
395 * 3/ The ref field may optionally be provided, in which pgmap->ref must be
403 void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)
408 ret = memremap_pages(pgmap, dev_to_node(dev));
413 pgmap);
420 void devm_memunmap_pages(struct device *dev, struct dev_pagemap *pgmap)
422 devm_release_action(dev, devm_memremap_pages_release, pgmap);
429 * @pgmap: optional known pgmap that already has a reference
431 * If @pgmap is non-NULL and covers @pfn it will be returned as-is. If @pgmap
435 struct dev_pagemap *pgmap)
442 if (pgmap) {
443 if (phys >= pgmap->range.start && phys <= pgmap->range.end)
444 return pgmap;
445 put_dev_pagemap(pgmap);
450 pgmap = xa_load(&pgmap_array, PHYS_PFN(phys));
451 if (pgmap && !percpu_ref_tryget_live_rcu(&pgmap->ref))
452 pgmap = NULL;
455 return pgmap;
461 if (WARN_ON_ONCE(!page->pgmap->ops || !page->pgmap->ops->page_free))
487 page->pgmap->ops->page_free(page);
489 if (page->pgmap->type != MEMORY_DEVICE_PRIVATE &&
490 page->pgmap->type != MEMORY_DEVICE_COHERENT)
497 put_dev_pagemap(page->pgmap);
506 WARN_ON_ONCE(!percpu_ref_tryget_live(&page->pgmap->ref));
515 if (page->pgmap->type != MEMORY_DEVICE_FS_DAX)