1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Copyright (c) by Jaroslav Kysela <perex@perex.cz> 4 * Takashi Iwai <tiwai@suse.de> 5 * 6 * Generic memory allocators 7 */ 8 9#include <linux/slab.h> 10#include <linux/mm.h> 11#include <linux/dma-mapping.h> 12#include <linux/dma-map-ops.h> 13#include <linux/genalloc.h> 14#include <linux/highmem.h> 15#include <linux/vmalloc.h> 16#ifdef CONFIG_X86 17#include <asm/set_memory.h> 18#endif 19#include <sound/memalloc.h> 20#include "memalloc_local.h" 21 22#define DEFAULT_GFP \ 23 (GFP_KERNEL | \ 24 __GFP_RETRY_MAYFAIL | /* don't trigger OOM-killer */ \ 25 __GFP_NOWARN) /* no stack trace print - this call is non-critical */ 26 27static const struct snd_malloc_ops *snd_dma_get_ops(struct snd_dma_buffer *dmab); 28 29#ifdef CONFIG_SND_DMA_SGBUF 30static void *snd_dma_sg_fallback_alloc(struct snd_dma_buffer *dmab, size_t size); 31#endif 32 33static void *__snd_dma_alloc_pages(struct snd_dma_buffer *dmab, size_t size) 34{ 35 const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab); 36 37 if (WARN_ON_ONCE(!ops || !ops->alloc)) 38 return NULL; 39 return ops->alloc(dmab, size); 40} 41 42/** 43 * snd_dma_alloc_dir_pages - allocate the buffer area according to the given 44 * type and direction 45 * @type: the DMA buffer type 46 * @device: the device pointer 47 * @dir: DMA direction 48 * @size: the buffer size to allocate 49 * @dmab: buffer allocation record to store the allocated data 50 * 51 * Calls the memory-allocator function for the corresponding 52 * buffer type. 53 * 54 * Return: Zero if the buffer with the given size is allocated successfully, 55 * otherwise a negative value on error. 56 */ 57int snd_dma_alloc_dir_pages(int type, struct device *device, 58 enum dma_data_direction dir, size_t size, 59 struct snd_dma_buffer *dmab) 60{ 61 if (WARN_ON(!size)) 62 return -ENXIO; 63 if (WARN_ON(!dmab)) 64 return -ENXIO; 65 66 size = PAGE_ALIGN(size); 67 dmab->dev.type = type; 68 dmab->dev.dev = device; 69 dmab->dev.dir = dir; 70 dmab->bytes = 0; 71 dmab->addr = 0; 72 dmab->private_data = NULL; 73 dmab->area = __snd_dma_alloc_pages(dmab, size); 74 if (!dmab->area) 75 return -ENOMEM; 76 dmab->bytes = size; 77 return 0; 78} 79EXPORT_SYMBOL(snd_dma_alloc_dir_pages); 80 81/** 82 * snd_dma_alloc_pages_fallback - allocate the buffer area according to the given type with fallback 83 * @type: the DMA buffer type 84 * @device: the device pointer 85 * @size: the buffer size to allocate 86 * @dmab: buffer allocation record to store the allocated data 87 * 88 * Calls the memory-allocator function for the corresponding 89 * buffer type. When no space is left, this function reduces the size and 90 * tries to allocate again. The size actually allocated is stored in 91 * res_size argument. 92 * 93 * Return: Zero if the buffer with the given size is allocated successfully, 94 * otherwise a negative value on error. 95 */ 96int snd_dma_alloc_pages_fallback(int type, struct device *device, size_t size, 97 struct snd_dma_buffer *dmab) 98{ 99 int err; 100 101 while ((err = snd_dma_alloc_pages(type, device, size, dmab)) < 0) { 102 if (err != -ENOMEM) 103 return err; 104 if (size <= PAGE_SIZE) 105 return -ENOMEM; 106 size >>= 1; 107 size = PAGE_SIZE << get_order(size); 108 } 109 if (! dmab->area) 110 return -ENOMEM; 111 return 0; 112} 113EXPORT_SYMBOL(snd_dma_alloc_pages_fallback); 114 115/** 116 * snd_dma_free_pages - release the allocated buffer 117 * @dmab: the buffer allocation record to release 118 * 119 * Releases the allocated buffer via snd_dma_alloc_pages(). 120 */ 121void snd_dma_free_pages(struct snd_dma_buffer *dmab) 122{ 123 const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab); 124 125 if (ops && ops->free) 126 ops->free(dmab); 127} 128EXPORT_SYMBOL(snd_dma_free_pages); 129 130/* called by devres */ 131static void __snd_release_pages(struct device *dev, void *res) 132{ 133 snd_dma_free_pages(res); 134} 135 136/** 137 * snd_devm_alloc_dir_pages - allocate the buffer and manage with devres 138 * @dev: the device pointer 139 * @type: the DMA buffer type 140 * @dir: DMA direction 141 * @size: the buffer size to allocate 142 * 143 * Allocate buffer pages depending on the given type and manage using devres. 144 * The pages will be released automatically at the device removal. 145 * 146 * Unlike snd_dma_alloc_pages(), this function requires the real device pointer, 147 * hence it can't work with SNDRV_DMA_TYPE_CONTINUOUS or 148 * SNDRV_DMA_TYPE_VMALLOC type. 149 * 150 * Return: the snd_dma_buffer object at success, or NULL if failed 151 */ 152struct snd_dma_buffer * 153snd_devm_alloc_dir_pages(struct device *dev, int type, 154 enum dma_data_direction dir, size_t size) 155{ 156 struct snd_dma_buffer *dmab; 157 int err; 158 159 if (WARN_ON(type == SNDRV_DMA_TYPE_CONTINUOUS || 160 type == SNDRV_DMA_TYPE_VMALLOC)) 161 return NULL; 162 163 dmab = devres_alloc(__snd_release_pages, sizeof(*dmab), GFP_KERNEL); 164 if (!dmab) 165 return NULL; 166 167 err = snd_dma_alloc_dir_pages(type, dev, dir, size, dmab); 168 if (err < 0) { 169 devres_free(dmab); 170 return NULL; 171 } 172 173 devres_add(dev, dmab); 174 return dmab; 175} 176EXPORT_SYMBOL_GPL(snd_devm_alloc_dir_pages); 177 178/** 179 * snd_dma_buffer_mmap - perform mmap of the given DMA buffer 180 * @dmab: buffer allocation information 181 * @area: VM area information 182 * 183 * Return: zero if successful, or a negative error code 184 */ 185int snd_dma_buffer_mmap(struct snd_dma_buffer *dmab, 186 struct vm_area_struct *area) 187{ 188 const struct snd_malloc_ops *ops; 189 190 if (!dmab) 191 return -ENOENT; 192 ops = snd_dma_get_ops(dmab); 193 if (ops && ops->mmap) 194 return ops->mmap(dmab, area); 195 else 196 return -ENOENT; 197} 198EXPORT_SYMBOL(snd_dma_buffer_mmap); 199 200#ifdef CONFIG_HAS_DMA 201/** 202 * snd_dma_buffer_sync - sync DMA buffer between CPU and device 203 * @dmab: buffer allocation information 204 * @mode: sync mode 205 */ 206void snd_dma_buffer_sync(struct snd_dma_buffer *dmab, 207 enum snd_dma_sync_mode mode) 208{ 209 const struct snd_malloc_ops *ops; 210 211 if (!dmab || !dmab->dev.need_sync) 212 return; 213 ops = snd_dma_get_ops(dmab); 214 if (ops && ops->sync) 215 ops->sync(dmab, mode); 216} 217EXPORT_SYMBOL_GPL(snd_dma_buffer_sync); 218#endif /* CONFIG_HAS_DMA */ 219 220/** 221 * snd_sgbuf_get_addr - return the physical address at the corresponding offset 222 * @dmab: buffer allocation information 223 * @offset: offset in the ring buffer 224 * 225 * Return: the physical address 226 */ 227dma_addr_t snd_sgbuf_get_addr(struct snd_dma_buffer *dmab, size_t offset) 228{ 229 const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab); 230 231 if (ops && ops->get_addr) 232 return ops->get_addr(dmab, offset); 233 else 234 return dmab->addr + offset; 235} 236EXPORT_SYMBOL(snd_sgbuf_get_addr); 237 238/** 239 * snd_sgbuf_get_page - return the physical page at the corresponding offset 240 * @dmab: buffer allocation information 241 * @offset: offset in the ring buffer 242 * 243 * Return: the page pointer 244 */ 245struct page *snd_sgbuf_get_page(struct snd_dma_buffer *dmab, size_t offset) 246{ 247 const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab); 248 249 if (ops && ops->get_page) 250 return ops->get_page(dmab, offset); 251 else 252 return virt_to_page(dmab->area + offset); 253} 254EXPORT_SYMBOL(snd_sgbuf_get_page); 255 256/** 257 * snd_sgbuf_get_chunk_size - compute the max chunk size with continuous pages 258 * on sg-buffer 259 * @dmab: buffer allocation information 260 * @ofs: offset in the ring buffer 261 * @size: the requested size 262 * 263 * Return: the chunk size 264 */ 265unsigned int snd_sgbuf_get_chunk_size(struct snd_dma_buffer *dmab, 266 unsigned int ofs, unsigned int size) 267{ 268 const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab); 269 270 if (ops && ops->get_chunk_size) 271 return ops->get_chunk_size(dmab, ofs, size); 272 else 273 return size; 274} 275EXPORT_SYMBOL(snd_sgbuf_get_chunk_size); 276 277/* 278 * Continuous pages allocator 279 */ 280static void *do_alloc_pages(struct device *dev, size_t size, dma_addr_t *addr, 281 bool wc) 282{ 283 void *p; 284 gfp_t gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; 285 286 again: 287 p = alloc_pages_exact(size, gfp); 288 if (!p) 289 return NULL; 290 *addr = page_to_phys(virt_to_page(p)); 291 if (!dev) 292 return p; 293 if ((*addr + size - 1) & ~dev->coherent_dma_mask) { 294 if (IS_ENABLED(CONFIG_ZONE_DMA32) && !(gfp & GFP_DMA32)) { 295 gfp |= GFP_DMA32; 296 goto again; 297 } 298 if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) { 299 gfp = (gfp & ~GFP_DMA32) | GFP_DMA; 300 goto again; 301 } 302 } 303#ifdef CONFIG_X86 304 if (wc) 305 set_memory_wc((unsigned long)(p), size >> PAGE_SHIFT); 306#endif 307 return p; 308} 309 310static void do_free_pages(void *p, size_t size, bool wc) 311{ 312#ifdef CONFIG_X86 313 if (wc) 314 set_memory_wb((unsigned long)(p), size >> PAGE_SHIFT); 315#endif 316 free_pages_exact(p, size); 317} 318 319 320static void *snd_dma_continuous_alloc(struct snd_dma_buffer *dmab, size_t size) 321{ 322 return do_alloc_pages(dmab->dev.dev, size, &dmab->addr, false); 323} 324 325static void snd_dma_continuous_free(struct snd_dma_buffer *dmab) 326{ 327 do_free_pages(dmab->area, dmab->bytes, false); 328} 329 330static int snd_dma_continuous_mmap(struct snd_dma_buffer *dmab, 331 struct vm_area_struct *area) 332{ 333 return remap_pfn_range(area, area->vm_start, 334 dmab->addr >> PAGE_SHIFT, 335 area->vm_end - area->vm_start, 336 area->vm_page_prot); 337} 338 339static const struct snd_malloc_ops snd_dma_continuous_ops = { 340 .alloc = snd_dma_continuous_alloc, 341 .free = snd_dma_continuous_free, 342 .mmap = snd_dma_continuous_mmap, 343}; 344 345/* 346 * VMALLOC allocator 347 */ 348static void *snd_dma_vmalloc_alloc(struct snd_dma_buffer *dmab, size_t size) 349{ 350 return vmalloc(size); 351} 352 353static void snd_dma_vmalloc_free(struct snd_dma_buffer *dmab) 354{ 355 vfree(dmab->area); 356} 357 358static int snd_dma_vmalloc_mmap(struct snd_dma_buffer *dmab, 359 struct vm_area_struct *area) 360{ 361 return remap_vmalloc_range(area, dmab->area, 0); 362} 363 364#define get_vmalloc_page_addr(dmab, offset) \ 365 page_to_phys(vmalloc_to_page((dmab)->area + (offset))) 366 367static dma_addr_t snd_dma_vmalloc_get_addr(struct snd_dma_buffer *dmab, 368 size_t offset) 369{ 370 return get_vmalloc_page_addr(dmab, offset) + offset % PAGE_SIZE; 371} 372 373static struct page *snd_dma_vmalloc_get_page(struct snd_dma_buffer *dmab, 374 size_t offset) 375{ 376 return vmalloc_to_page(dmab->area + offset); 377} 378 379static unsigned int 380snd_dma_vmalloc_get_chunk_size(struct snd_dma_buffer *dmab, 381 unsigned int ofs, unsigned int size) 382{ 383 unsigned int start, end; 384 unsigned long addr; 385 386 start = ALIGN_DOWN(ofs, PAGE_SIZE); 387 end = ofs + size - 1; /* the last byte address */ 388 /* check page continuity */ 389 addr = get_vmalloc_page_addr(dmab, start); 390 for (;;) { 391 start += PAGE_SIZE; 392 if (start > end) 393 break; 394 addr += PAGE_SIZE; 395 if (get_vmalloc_page_addr(dmab, start) != addr) 396 return start - ofs; 397 } 398 /* ok, all on continuous pages */ 399 return size; 400} 401 402static const struct snd_malloc_ops snd_dma_vmalloc_ops = { 403 .alloc = snd_dma_vmalloc_alloc, 404 .free = snd_dma_vmalloc_free, 405 .mmap = snd_dma_vmalloc_mmap, 406 .get_addr = snd_dma_vmalloc_get_addr, 407 .get_page = snd_dma_vmalloc_get_page, 408 .get_chunk_size = snd_dma_vmalloc_get_chunk_size, 409}; 410 411#ifdef CONFIG_HAS_DMA 412/* 413 * IRAM allocator 414 */ 415#ifdef CONFIG_GENERIC_ALLOCATOR 416static void *snd_dma_iram_alloc(struct snd_dma_buffer *dmab, size_t size) 417{ 418 struct device *dev = dmab->dev.dev; 419 struct gen_pool *pool; 420 void *p; 421 422 if (dev->of_node) { 423 pool = of_gen_pool_get(dev->of_node, "iram", 0); 424 /* Assign the pool into private_data field */ 425 dmab->private_data = pool; 426 427 p = gen_pool_dma_alloc_align(pool, size, &dmab->addr, PAGE_SIZE); 428 if (p) 429 return p; 430 } 431 432 /* Internal memory might have limited size and no enough space, 433 * so if we fail to malloc, try to fetch memory traditionally. 434 */ 435 dmab->dev.type = SNDRV_DMA_TYPE_DEV; 436 return __snd_dma_alloc_pages(dmab, size); 437} 438 439static void snd_dma_iram_free(struct snd_dma_buffer *dmab) 440{ 441 struct gen_pool *pool = dmab->private_data; 442 443 if (pool && dmab->area) 444 gen_pool_free(pool, (unsigned long)dmab->area, dmab->bytes); 445} 446 447static int snd_dma_iram_mmap(struct snd_dma_buffer *dmab, 448 struct vm_area_struct *area) 449{ 450 area->vm_page_prot = pgprot_writecombine(area->vm_page_prot); 451 return remap_pfn_range(area, area->vm_start, 452 dmab->addr >> PAGE_SHIFT, 453 area->vm_end - area->vm_start, 454 area->vm_page_prot); 455} 456 457static const struct snd_malloc_ops snd_dma_iram_ops = { 458 .alloc = snd_dma_iram_alloc, 459 .free = snd_dma_iram_free, 460 .mmap = snd_dma_iram_mmap, 461}; 462#endif /* CONFIG_GENERIC_ALLOCATOR */ 463 464/* 465 * Coherent device pages allocator 466 */ 467static void *snd_dma_dev_alloc(struct snd_dma_buffer *dmab, size_t size) 468{ 469 return dma_alloc_coherent(dmab->dev.dev, size, &dmab->addr, DEFAULT_GFP); 470} 471 472static void snd_dma_dev_free(struct snd_dma_buffer *dmab) 473{ 474 dma_free_coherent(dmab->dev.dev, dmab->bytes, dmab->area, dmab->addr); 475} 476 477static int snd_dma_dev_mmap(struct snd_dma_buffer *dmab, 478 struct vm_area_struct *area) 479{ 480 return dma_mmap_coherent(dmab->dev.dev, area, 481 dmab->area, dmab->addr, dmab->bytes); 482} 483 484static const struct snd_malloc_ops snd_dma_dev_ops = { 485 .alloc = snd_dma_dev_alloc, 486 .free = snd_dma_dev_free, 487 .mmap = snd_dma_dev_mmap, 488}; 489 490/* 491 * Write-combined pages 492 */ 493/* x86-specific allocations */ 494#ifdef CONFIG_SND_DMA_SGBUF 495static void *snd_dma_wc_alloc(struct snd_dma_buffer *dmab, size_t size) 496{ 497 return do_alloc_pages(dmab->dev.dev, size, &dmab->addr, true); 498} 499 500static void snd_dma_wc_free(struct snd_dma_buffer *dmab) 501{ 502 do_free_pages(dmab->area, dmab->bytes, true); 503} 504 505static int snd_dma_wc_mmap(struct snd_dma_buffer *dmab, 506 struct vm_area_struct *area) 507{ 508 area->vm_page_prot = pgprot_writecombine(area->vm_page_prot); 509 return snd_dma_continuous_mmap(dmab, area); 510} 511#else 512static void *snd_dma_wc_alloc(struct snd_dma_buffer *dmab, size_t size) 513{ 514 return dma_alloc_wc(dmab->dev.dev, size, &dmab->addr, DEFAULT_GFP); 515} 516 517static void snd_dma_wc_free(struct snd_dma_buffer *dmab) 518{ 519 dma_free_wc(dmab->dev.dev, dmab->bytes, dmab->area, dmab->addr); 520} 521 522static int snd_dma_wc_mmap(struct snd_dma_buffer *dmab, 523 struct vm_area_struct *area) 524{ 525 return dma_mmap_wc(dmab->dev.dev, area, 526 dmab->area, dmab->addr, dmab->bytes); 527} 528#endif /* CONFIG_SND_DMA_SGBUF */ 529 530static const struct snd_malloc_ops snd_dma_wc_ops = { 531 .alloc = snd_dma_wc_alloc, 532 .free = snd_dma_wc_free, 533 .mmap = snd_dma_wc_mmap, 534}; 535 536/* 537 * Non-contiguous pages allocator 538 */ 539static void *snd_dma_noncontig_alloc(struct snd_dma_buffer *dmab, size_t size) 540{ 541 struct sg_table *sgt; 542 void *p; 543 544#ifdef CONFIG_SND_DMA_SGBUF 545 if (cpu_feature_enabled(X86_FEATURE_XENPV)) 546 return snd_dma_sg_fallback_alloc(dmab, size); 547#endif 548 sgt = dma_alloc_noncontiguous(dmab->dev.dev, size, dmab->dev.dir, 549 DEFAULT_GFP, 0); 550#ifdef CONFIG_SND_DMA_SGBUF 551 if (!sgt && !get_dma_ops(dmab->dev.dev)) 552 return snd_dma_sg_fallback_alloc(dmab, size); 553#endif 554 if (!sgt) 555 return NULL; 556 557 dmab->dev.need_sync = dma_need_sync(dmab->dev.dev, 558 sg_dma_address(sgt->sgl)); 559 p = dma_vmap_noncontiguous(dmab->dev.dev, size, sgt); 560 if (p) { 561 dmab->private_data = sgt; 562 /* store the first page address for convenience */ 563 dmab->addr = snd_sgbuf_get_addr(dmab, 0); 564 } else { 565 dma_free_noncontiguous(dmab->dev.dev, size, sgt, dmab->dev.dir); 566 } 567 return p; 568} 569 570static void snd_dma_noncontig_free(struct snd_dma_buffer *dmab) 571{ 572 dma_vunmap_noncontiguous(dmab->dev.dev, dmab->area); 573 dma_free_noncontiguous(dmab->dev.dev, dmab->bytes, dmab->private_data, 574 dmab->dev.dir); 575} 576 577static int snd_dma_noncontig_mmap(struct snd_dma_buffer *dmab, 578 struct vm_area_struct *area) 579{ 580 return dma_mmap_noncontiguous(dmab->dev.dev, area, 581 dmab->bytes, dmab->private_data); 582} 583 584static void snd_dma_noncontig_sync(struct snd_dma_buffer *dmab, 585 enum snd_dma_sync_mode mode) 586{ 587 if (mode == SNDRV_DMA_SYNC_CPU) { 588 if (dmab->dev.dir == DMA_TO_DEVICE) 589 return; 590 invalidate_kernel_vmap_range(dmab->area, dmab->bytes); 591 dma_sync_sgtable_for_cpu(dmab->dev.dev, dmab->private_data, 592 dmab->dev.dir); 593 } else { 594 if (dmab->dev.dir == DMA_FROM_DEVICE) 595 return; 596 flush_kernel_vmap_range(dmab->area, dmab->bytes); 597 dma_sync_sgtable_for_device(dmab->dev.dev, dmab->private_data, 598 dmab->dev.dir); 599 } 600} 601 602static inline void snd_dma_noncontig_iter_set(struct snd_dma_buffer *dmab, 603 struct sg_page_iter *piter, 604 size_t offset) 605{ 606 struct sg_table *sgt = dmab->private_data; 607 608 __sg_page_iter_start(piter, sgt->sgl, sgt->orig_nents, 609 offset >> PAGE_SHIFT); 610} 611 612static dma_addr_t snd_dma_noncontig_get_addr(struct snd_dma_buffer *dmab, 613 size_t offset) 614{ 615 struct sg_dma_page_iter iter; 616 617 snd_dma_noncontig_iter_set(dmab, &iter.base, offset); 618 __sg_page_iter_dma_next(&iter); 619 return sg_page_iter_dma_address(&iter) + offset % PAGE_SIZE; 620} 621 622static struct page *snd_dma_noncontig_get_page(struct snd_dma_buffer *dmab, 623 size_t offset) 624{ 625 struct sg_page_iter iter; 626 627 snd_dma_noncontig_iter_set(dmab, &iter, offset); 628 __sg_page_iter_next(&iter); 629 return sg_page_iter_page(&iter); 630} 631 632static unsigned int 633snd_dma_noncontig_get_chunk_size(struct snd_dma_buffer *dmab, 634 unsigned int ofs, unsigned int size) 635{ 636 struct sg_dma_page_iter iter; 637 unsigned int start, end; 638 unsigned long addr; 639 640 start = ALIGN_DOWN(ofs, PAGE_SIZE); 641 end = ofs + size - 1; /* the last byte address */ 642 snd_dma_noncontig_iter_set(dmab, &iter.base, start); 643 if (!__sg_page_iter_dma_next(&iter)) 644 return 0; 645 /* check page continuity */ 646 addr = sg_page_iter_dma_address(&iter); 647 for (;;) { 648 start += PAGE_SIZE; 649 if (start > end) 650 break; 651 addr += PAGE_SIZE; 652 if (!__sg_page_iter_dma_next(&iter) || 653 sg_page_iter_dma_address(&iter) != addr) 654 return start - ofs; 655 } 656 /* ok, all on continuous pages */ 657 return size; 658} 659 660static const struct snd_malloc_ops snd_dma_noncontig_ops = { 661 .alloc = snd_dma_noncontig_alloc, 662 .free = snd_dma_noncontig_free, 663 .mmap = snd_dma_noncontig_mmap, 664 .sync = snd_dma_noncontig_sync, 665 .get_addr = snd_dma_noncontig_get_addr, 666 .get_page = snd_dma_noncontig_get_page, 667 .get_chunk_size = snd_dma_noncontig_get_chunk_size, 668}; 669 670/* x86-specific SG-buffer with WC pages */ 671#ifdef CONFIG_SND_DMA_SGBUF 672#define sg_wc_address(it) ((unsigned long)page_address(sg_page_iter_page(it))) 673 674static void *snd_dma_sg_wc_alloc(struct snd_dma_buffer *dmab, size_t size) 675{ 676 void *p = snd_dma_noncontig_alloc(dmab, size); 677 struct sg_table *sgt = dmab->private_data; 678 struct sg_page_iter iter; 679 680 if (!p) 681 return NULL; 682 if (dmab->dev.type != SNDRV_DMA_TYPE_DEV_WC_SG) 683 return p; 684 for_each_sgtable_page(sgt, &iter, 0) 685 set_memory_wc(sg_wc_address(&iter), 1); 686 return p; 687} 688 689static void snd_dma_sg_wc_free(struct snd_dma_buffer *dmab) 690{ 691 struct sg_table *sgt = dmab->private_data; 692 struct sg_page_iter iter; 693 694 for_each_sgtable_page(sgt, &iter, 0) 695 set_memory_wb(sg_wc_address(&iter), 1); 696 snd_dma_noncontig_free(dmab); 697} 698 699static int snd_dma_sg_wc_mmap(struct snd_dma_buffer *dmab, 700 struct vm_area_struct *area) 701{ 702 area->vm_page_prot = pgprot_writecombine(area->vm_page_prot); 703 return dma_mmap_noncontiguous(dmab->dev.dev, area, 704 dmab->bytes, dmab->private_data); 705} 706 707static const struct snd_malloc_ops snd_dma_sg_wc_ops = { 708 .alloc = snd_dma_sg_wc_alloc, 709 .free = snd_dma_sg_wc_free, 710 .mmap = snd_dma_sg_wc_mmap, 711 .sync = snd_dma_noncontig_sync, 712 .get_addr = snd_dma_noncontig_get_addr, 713 .get_page = snd_dma_noncontig_get_page, 714 .get_chunk_size = snd_dma_noncontig_get_chunk_size, 715}; 716 717/* Fallback SG-buffer allocations for x86 */ 718struct snd_dma_sg_fallback { 719 bool use_dma_alloc_coherent; 720 size_t count; 721 struct page **pages; 722 /* DMA address array; the first page contains #pages in ~PAGE_MASK */ 723 dma_addr_t *addrs; 724}; 725 726static void __snd_dma_sg_fallback_free(struct snd_dma_buffer *dmab, 727 struct snd_dma_sg_fallback *sgbuf) 728{ 729 size_t i, size; 730 731 if (sgbuf->pages && sgbuf->addrs) { 732 i = 0; 733 while (i < sgbuf->count) { 734 if (!sgbuf->pages[i] || !sgbuf->addrs[i]) 735 break; 736 size = sgbuf->addrs[i] & ~PAGE_MASK; 737 if (WARN_ON(!size)) 738 break; 739 if (sgbuf->use_dma_alloc_coherent) 740 dma_free_coherent(dmab->dev.dev, size << PAGE_SHIFT, 741 page_address(sgbuf->pages[i]), 742 sgbuf->addrs[i] & PAGE_MASK); 743 else 744 do_free_pages(page_address(sgbuf->pages[i]), 745 size << PAGE_SHIFT, false); 746 i += size; 747 } 748 } 749 kvfree(sgbuf->pages); 750 kvfree(sgbuf->addrs); 751 kfree(sgbuf); 752} 753 754static void *snd_dma_sg_fallback_alloc(struct snd_dma_buffer *dmab, size_t size) 755{ 756 struct snd_dma_sg_fallback *sgbuf; 757 struct page **pagep, *curp; 758 size_t chunk, npages; 759 dma_addr_t *addrp; 760 dma_addr_t addr; 761 void *p; 762 763 /* correct the type */ 764 if (dmab->dev.type == SNDRV_DMA_TYPE_DEV_SG) 765 dmab->dev.type = SNDRV_DMA_TYPE_DEV_SG_FALLBACK; 766 else if (dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG) 767 dmab->dev.type = SNDRV_DMA_TYPE_DEV_WC_SG_FALLBACK; 768 769 sgbuf = kzalloc(sizeof(*sgbuf), GFP_KERNEL); 770 if (!sgbuf) 771 return NULL; 772 sgbuf->use_dma_alloc_coherent = cpu_feature_enabled(X86_FEATURE_XENPV); 773 size = PAGE_ALIGN(size); 774 sgbuf->count = size >> PAGE_SHIFT; 775 sgbuf->pages = kvcalloc(sgbuf->count, sizeof(*sgbuf->pages), GFP_KERNEL); 776 sgbuf->addrs = kvcalloc(sgbuf->count, sizeof(*sgbuf->addrs), GFP_KERNEL); 777 if (!sgbuf->pages || !sgbuf->addrs) 778 goto error; 779 780 pagep = sgbuf->pages; 781 addrp = sgbuf->addrs; 782 chunk = (PAGE_SIZE - 1) << PAGE_SHIFT; /* to fit in low bits in addrs */ 783 while (size > 0) { 784 chunk = min(size, chunk); 785 if (sgbuf->use_dma_alloc_coherent) 786 p = dma_alloc_coherent(dmab->dev.dev, chunk, &addr, DEFAULT_GFP); 787 else 788 p = do_alloc_pages(dmab->dev.dev, chunk, &addr, false); 789 if (!p) { 790 if (chunk <= PAGE_SIZE) 791 goto error; 792 chunk >>= 1; 793 chunk = PAGE_SIZE << get_order(chunk); 794 continue; 795 } 796 797 size -= chunk; 798 /* fill pages */ 799 npages = chunk >> PAGE_SHIFT; 800 *addrp = npages; /* store in lower bits */ 801 curp = virt_to_page(p); 802 while (npages--) { 803 *pagep++ = curp++; 804 *addrp++ |= addr; 805 addr += PAGE_SIZE; 806 } 807 } 808 809 p = vmap(sgbuf->pages, sgbuf->count, VM_MAP, PAGE_KERNEL); 810 if (!p) 811 goto error; 812 813 if (dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG_FALLBACK) 814 set_pages_array_wc(sgbuf->pages, sgbuf->count); 815 816 dmab->private_data = sgbuf; 817 /* store the first page address for convenience */ 818 dmab->addr = sgbuf->addrs[0] & PAGE_MASK; 819 return p; 820 821 error: 822 __snd_dma_sg_fallback_free(dmab, sgbuf); 823 return NULL; 824} 825 826static void snd_dma_sg_fallback_free(struct snd_dma_buffer *dmab) 827{ 828 struct snd_dma_sg_fallback *sgbuf = dmab->private_data; 829 830 if (dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG_FALLBACK) 831 set_pages_array_wb(sgbuf->pages, sgbuf->count); 832 vunmap(dmab->area); 833 __snd_dma_sg_fallback_free(dmab, dmab->private_data); 834} 835 836static dma_addr_t snd_dma_sg_fallback_get_addr(struct snd_dma_buffer *dmab, 837 size_t offset) 838{ 839 struct snd_dma_sg_fallback *sgbuf = dmab->private_data; 840 size_t index = offset >> PAGE_SHIFT; 841 842 return (sgbuf->addrs[index] & PAGE_MASK) | (offset & ~PAGE_MASK); 843} 844 845static int snd_dma_sg_fallback_mmap(struct snd_dma_buffer *dmab, 846 struct vm_area_struct *area) 847{ 848 struct snd_dma_sg_fallback *sgbuf = dmab->private_data; 849 850 if (dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG_FALLBACK) 851 area->vm_page_prot = pgprot_writecombine(area->vm_page_prot); 852 return vm_map_pages(area, sgbuf->pages, sgbuf->count); 853} 854 855static const struct snd_malloc_ops snd_dma_sg_fallback_ops = { 856 .alloc = snd_dma_sg_fallback_alloc, 857 .free = snd_dma_sg_fallback_free, 858 .mmap = snd_dma_sg_fallback_mmap, 859 .get_addr = snd_dma_sg_fallback_get_addr, 860 /* reuse vmalloc helpers */ 861 .get_page = snd_dma_vmalloc_get_page, 862 .get_chunk_size = snd_dma_vmalloc_get_chunk_size, 863}; 864#endif /* CONFIG_SND_DMA_SGBUF */ 865 866/* 867 * Non-coherent pages allocator 868 */ 869static void *snd_dma_noncoherent_alloc(struct snd_dma_buffer *dmab, size_t size) 870{ 871 void *p; 872 873 p = dma_alloc_noncoherent(dmab->dev.dev, size, &dmab->addr, 874 dmab->dev.dir, DEFAULT_GFP); 875 if (p) 876 dmab->dev.need_sync = dma_need_sync(dmab->dev.dev, dmab->addr); 877 return p; 878} 879 880static void snd_dma_noncoherent_free(struct snd_dma_buffer *dmab) 881{ 882 dma_free_noncoherent(dmab->dev.dev, dmab->bytes, dmab->area, 883 dmab->addr, dmab->dev.dir); 884} 885 886static int snd_dma_noncoherent_mmap(struct snd_dma_buffer *dmab, 887 struct vm_area_struct *area) 888{ 889 area->vm_page_prot = vm_get_page_prot(area->vm_flags); 890 return dma_mmap_pages(dmab->dev.dev, area, 891 area->vm_end - area->vm_start, 892 virt_to_page(dmab->area)); 893} 894 895static void snd_dma_noncoherent_sync(struct snd_dma_buffer *dmab, 896 enum snd_dma_sync_mode mode) 897{ 898 if (mode == SNDRV_DMA_SYNC_CPU) { 899 if (dmab->dev.dir != DMA_TO_DEVICE) 900 dma_sync_single_for_cpu(dmab->dev.dev, dmab->addr, 901 dmab->bytes, dmab->dev.dir); 902 } else { 903 if (dmab->dev.dir != DMA_FROM_DEVICE) 904 dma_sync_single_for_device(dmab->dev.dev, dmab->addr, 905 dmab->bytes, dmab->dev.dir); 906 } 907} 908 909static const struct snd_malloc_ops snd_dma_noncoherent_ops = { 910 .alloc = snd_dma_noncoherent_alloc, 911 .free = snd_dma_noncoherent_free, 912 .mmap = snd_dma_noncoherent_mmap, 913 .sync = snd_dma_noncoherent_sync, 914}; 915 916#endif /* CONFIG_HAS_DMA */ 917 918/* 919 * Entry points 920 */ 921static const struct snd_malloc_ops *snd_dma_ops[] = { 922 [SNDRV_DMA_TYPE_CONTINUOUS] = &snd_dma_continuous_ops, 923 [SNDRV_DMA_TYPE_VMALLOC] = &snd_dma_vmalloc_ops, 924#ifdef CONFIG_HAS_DMA 925 [SNDRV_DMA_TYPE_DEV] = &snd_dma_dev_ops, 926 [SNDRV_DMA_TYPE_DEV_WC] = &snd_dma_wc_ops, 927 [SNDRV_DMA_TYPE_NONCONTIG] = &snd_dma_noncontig_ops, 928 [SNDRV_DMA_TYPE_NONCOHERENT] = &snd_dma_noncoherent_ops, 929#ifdef CONFIG_SND_DMA_SGBUF 930 [SNDRV_DMA_TYPE_DEV_WC_SG] = &snd_dma_sg_wc_ops, 931#endif 932#ifdef CONFIG_GENERIC_ALLOCATOR 933 [SNDRV_DMA_TYPE_DEV_IRAM] = &snd_dma_iram_ops, 934#endif /* CONFIG_GENERIC_ALLOCATOR */ 935#ifdef CONFIG_SND_DMA_SGBUF 936 [SNDRV_DMA_TYPE_DEV_SG_FALLBACK] = &snd_dma_sg_fallback_ops, 937 [SNDRV_DMA_TYPE_DEV_WC_SG_FALLBACK] = &snd_dma_sg_fallback_ops, 938#endif 939#endif /* CONFIG_HAS_DMA */ 940}; 941 942static const struct snd_malloc_ops *snd_dma_get_ops(struct snd_dma_buffer *dmab) 943{ 944 if (WARN_ON_ONCE(!dmab)) 945 return NULL; 946 if (WARN_ON_ONCE(dmab->dev.type <= SNDRV_DMA_TYPE_UNKNOWN || 947 dmab->dev.type >= ARRAY_SIZE(snd_dma_ops))) 948 return NULL; 949 return snd_dma_ops[dmab->dev.type]; 950} 951