iommu/iommufd/iova_bitmap.c

16  * Main data structure for tracking mapped user pages of bitmap data.
90  * which u64 word of the bitmap is mapped, up to @mapped_total_index.
94  * The IOVA bitmap is usually located on what tracks DMA mapped ranges or
99 	/* IOVA range representing the currently mapped bitmap data */
100 	struct iova_bitmap_map mapped;
105 	/* u64 index that @mapped points to */
125  * Relative IOVA means relative to the bitmap::mapped base IOVA
126  * (stored in mapped::iova). All computations in this file are done using
127  * relative IOVAs and thus avoid an extra subtraction against mapped::iova.
133 	unsigned long pgsize = 1 << bitmap->mapped.pgshift;
144 	unsigned long pgshift = bitmap->mapped.pgshift;
150  * Returns the base IOVA of the mapped range.
166 	struct iova_bitmap_map *mapped = &bitmap->mapped;
172 	 * @mapped_base_index is the index of the currently mapped u64 words
174 	 * mapped. The range @mapped_base_index .. @mapped_total_index-1 is
175 	 * mapped but capped at a maximum number of pages.
195 				  FOLL_WRITE, mapped->pages);
199 	mapped->npages = (unsigned long)ret;
201 	mapped->iova = iova_bitmap_mapped_iova(bitmap);
208 	mapped->pgoff = offset_in_page(addr);
219 	struct iova_bitmap_map *mapped = &bitmap->mapped;
221 	if (mapped->npages) {
222 		unpin_user_pages(mapped->pages, mapped->npages);
223 		mapped->npages = 0;
244 	struct iova_bitmap_map *mapped;
252 	mapped = &bitmap->mapped;
253 	mapped->pgshift = __ffs(page_size);
259 	mapped->iova = iova;
260 	mapped->pages = (struct page **)__get_free_page(GFP_KERNEL);
261 	if (!mapped->pages) {
286 	struct iova_bitmap_map *mapped = &bitmap->mapped;
290 	if (mapped->pages) {
291 		free_page((unsigned long)mapped->pages);
292 		mapped->pages = NULL;
307 	bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff;
317  * Returns the length of the mapped IOVA range.
411  * mapped bitmap user pages into IOVA ranges to process.
446 	struct iova_bitmap_map *mapped = &bitmap->mapped;
447 	unsigned long cur_bit = ((iova - mapped->iova) >>
448 			mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;
449 	unsigned long last_bit = (((iova + length - 1) - mapped->iova) >>
450 			mapped->pgshift) + mapped->pgoff * BITS_PER_BYTE;
451 	unsigned long last_page_idx = mapped->npages - 1;
463 		kaddr = kmap_local_page(mapped->pages[page_idx]);
471 			((last_bit - cur_bit + 1) << bitmap->mapped.pgshift);