1/*- 2 * Copyright (c) 1997, 1998 Justin T. Gibbs. 3 * Copyright (c) 2015-2016 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship of the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by Semihalf 10 * under sponsorship of the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions, and the following disclaimer, 17 * without modification, immediately at the beginning of the file. 18 * 2. The name of the author may not be used to endorse or promote products 19 * derived from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 25 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34#include <sys/param.h> 35#include <sys/systm.h> 36#include <sys/domainset.h> 37#include <sys/malloc.h> 38#include <sys/bus.h> 39#include <sys/interrupt.h> 40#include <sys/kernel.h> 41#include <sys/ktr.h> 42#include <sys/lock.h> 43#include <sys/memdesc.h> 44#include <sys/msan.h> 45#include <sys/mutex.h> 46#include <sys/proc.h> 47#include <sys/sysctl.h> 48#include <sys/uio.h> 49 50#include <vm/vm.h> 51#include <vm/vm_extern.h> 52#include <vm/vm_kern.h> 53#include <vm/vm_page.h> 54#include <vm/vm_map.h> 55 56#include <machine/atomic.h> 57#include <machine/bus.h> 58#include <machine/md_var.h> 59#include <arm64/include/bus_dma_impl.h> 60 61#define MAX_BPAGES 4096 62 63enum { 64 BF_COULD_BOUNCE = 0x01, 65 BF_MIN_ALLOC_COMP = 0x02, 66 BF_KMEM_ALLOC = 0x04, 67 BF_COHERENT = 0x10, 68}; 69 70struct bounce_page; 71struct bounce_zone; 72 73struct bus_dma_tag { 74 struct bus_dma_tag_common common; 75 size_t alloc_size; 76 size_t alloc_alignment; 77 int map_count; 78 int bounce_flags; 79 bus_dma_segment_t *segments; 80 struct bounce_zone *bounce_zone; 81}; 82 83static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 84 "Busdma parameters"); 85 86struct sync_list { 87 vm_offset_t vaddr; /* kva of client data */ 88 bus_addr_t paddr; /* physical address */ 89 vm_page_t pages; /* starting page of client data */ 90 bus_size_t datacount; /* client data count */ 91}; 92 93struct bus_dmamap { 94 STAILQ_HEAD(, bounce_page) bpages; 95 int pagesneeded; 96 int pagesreserved; 97 bus_dma_tag_t dmat; 98 struct memdesc mem; 99 bus_dmamap_callback_t *callback; 100 void *callback_arg; 101 __sbintime_t queued_time; 102 STAILQ_ENTRY(bus_dmamap) links; 103 u_int flags; 104#define DMAMAP_COHERENT (1 << 0) 105#define DMAMAP_FROM_DMAMEM (1 << 1) 106#define DMAMAP_MBUF (1 << 2) 107 int sync_count; 108#ifdef KMSAN 109 struct memdesc kmsan_mem; 110#endif 111 struct sync_list slist[]; 112}; 113 114static bool _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, 115 vm_paddr_t buf, bus_size_t buflen, int *pagesneeded); 116static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 117 pmap_t pmap, void *buf, bus_size_t buflen, int flags); 118static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 119 vm_paddr_t buf, bus_size_t buflen, int flags); 120 121static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); 122 123#define dmat_alignment(dmat) ((dmat)->common.alignment) 124#define dmat_bounce_flags(dmat) ((dmat)->bounce_flags) 125#define dmat_boundary(dmat) ((dmat)->common.boundary) 126#define dmat_domain(dmat) ((dmat)->common.domain) 127#define dmat_flags(dmat) ((dmat)->common.flags) 128#define dmat_highaddr(dmat) ((dmat)->common.highaddr) 129#define dmat_lowaddr(dmat) ((dmat)->common.lowaddr) 130#define dmat_lockfunc(dmat) ((dmat)->common.lockfunc) 131#define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg) 132#define dmat_maxsegsz(dmat) ((dmat)->common.maxsegsz) 133#define dmat_nsegments(dmat) ((dmat)->common.nsegments) 134 135#include "../../kern/subr_busdma_bounce.c" 136 137static int 138bounce_bus_dma_zone_setup(bus_dma_tag_t dmat) 139{ 140 struct bounce_zone *bz; 141 bus_size_t maxsize; 142 int error; 143 144 /* 145 * Round size up to a full page, and add one more page because 146 * there can always be one more boundary crossing than the 147 * number of pages in a transfer. 148 */ 149 maxsize = roundup2(dmat->common.maxsize, PAGE_SIZE) + PAGE_SIZE; 150 151 /* Must bounce */ 152 if ((error = alloc_bounce_zone(dmat)) != 0) 153 return (error); 154 bz = dmat->bounce_zone; 155 156 if (ptoa(bz->total_bpages) < maxsize) { 157 int pages; 158 159 pages = atop(maxsize) + 1 - bz->total_bpages; 160 161 /* Add pages to our bounce pool */ 162 if (alloc_bounce_pages(dmat, pages) < pages) 163 return (ENOMEM); 164 } 165 /* Performed initial allocation */ 166 dmat->bounce_flags |= BF_MIN_ALLOC_COMP; 167 168 return (error); 169} 170 171/* 172 * Return true if the DMA should bounce because the start or end does not fall 173 * on a cacheline boundary (which would require a partial cacheline flush). 174 * COHERENT memory doesn't trigger cacheline flushes. Memory allocated by 175 * bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a 176 * strict rule that such memory cannot be accessed by the CPU while DMA is in 177 * progress (or by multiple DMA engines at once), so that it's always safe to do 178 * full cacheline flushes even if that affects memory outside the range of a 179 * given DMA operation that doesn't involve the full allocated buffer. If we're 180 * mapping an mbuf, that follows the same rules as a buffer we allocated. 181 */ 182static bool 183cacheline_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 184 bus_size_t size) 185{ 186 187#define DMAMAP_CACHELINE_FLAGS \ 188 (DMAMAP_FROM_DMAMEM | DMAMAP_COHERENT | DMAMAP_MBUF) 189 if ((dmat->bounce_flags & BF_COHERENT) != 0) 190 return (false); 191 if (map != NULL && (map->flags & DMAMAP_CACHELINE_FLAGS) != 0) 192 return (false); 193 return (((paddr | size) & (dcache_line_size - 1)) != 0); 194#undef DMAMAP_CACHELINE_FLAGS 195} 196 197/* 198 * Return true if the given address does not fall on the alignment boundary. 199 */ 200static bool 201alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr) 202{ 203 204 return (!vm_addr_align_ok(addr, dmat->common.alignment)); 205} 206 207static bool 208might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 209 bus_size_t size) 210{ 211 212 /* Memory allocated by bounce_bus_dmamem_alloc won't bounce */ 213 if (map && (map->flags & DMAMAP_FROM_DMAMEM) != 0) 214 return (false); 215 216 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) 217 return (true); 218 219 if (cacheline_bounce(dmat, map, paddr, size)) 220 return (true); 221 222 if (alignment_bounce(dmat, paddr)) 223 return (true); 224 225 return (false); 226} 227 228static bool 229must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 230 bus_size_t size) 231{ 232 233 if (cacheline_bounce(dmat, map, paddr, size)) 234 return (true); 235 236 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0 && 237 addr_needs_bounce(dmat, paddr)) 238 return (true); 239 240 return (false); 241} 242 243/* 244 * Allocate a device specific dma_tag. 245 */ 246static int 247bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 248 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 249 bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, 250 bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) 251{ 252 bus_dma_tag_t newtag; 253 int error; 254 255 *dmat = NULL; 256 error = common_bus_dma_tag_create(parent != NULL ? &parent->common : 257 NULL, alignment, boundary, lowaddr, highaddr, maxsize, nsegments, 258 maxsegsz, flags, lockfunc, lockfuncarg, 259 sizeof (struct bus_dma_tag), (void **)&newtag); 260 if (error != 0) 261 return (error); 262 263 newtag->common.impl = &bus_dma_bounce_impl; 264 newtag->map_count = 0; 265 newtag->segments = NULL; 266 267 if ((flags & BUS_DMA_COHERENT) != 0) { 268 newtag->bounce_flags |= BF_COHERENT; 269 } 270 271 if (parent != NULL) { 272 if ((parent->bounce_flags & BF_COULD_BOUNCE) != 0) 273 newtag->bounce_flags |= BF_COULD_BOUNCE; 274 275 /* Copy some flags from the parent */ 276 newtag->bounce_flags |= parent->bounce_flags & BF_COHERENT; 277 } 278 279 if ((newtag->bounce_flags & BF_COHERENT) != 0) { 280 newtag->alloc_alignment = newtag->common.alignment; 281 newtag->alloc_size = newtag->common.maxsize; 282 } else { 283 /* 284 * Ensure the buffer is aligned to a cacheline when allocating 285 * a non-coherent buffer. This is so we don't have any data 286 * that another CPU may be accessing around DMA buffer 287 * causing the cache to become dirty. 288 */ 289 newtag->alloc_alignment = MAX(newtag->common.alignment, 290 dcache_line_size); 291 newtag->alloc_size = roundup2(newtag->common.maxsize, 292 dcache_line_size); 293 } 294 295 if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) || 296 newtag->common.alignment > 1) 297 newtag->bounce_flags |= BF_COULD_BOUNCE; 298 299 if ((flags & BUS_DMA_ALLOCNOW) != 0) 300 error = bounce_bus_dma_zone_setup(newtag); 301 else 302 error = 0; 303 304 if (error != 0) 305 free(newtag, M_DEVBUF); 306 else 307 *dmat = newtag; 308 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 309 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), 310 error); 311 return (error); 312} 313 314static int 315bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat) 316{ 317 int error = 0; 318 319 if (dmat != NULL) { 320 if (dmat->map_count != 0) { 321 error = EBUSY; 322 goto out; 323 } 324 if (dmat->segments != NULL) 325 free(dmat->segments, M_DEVBUF); 326 free(dmat, M_DEVBUF); 327 } 328out: 329 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat, error); 330 return (error); 331} 332 333/* 334 * Update the domain for the tag. We may need to reallocate the zone and 335 * bounce pages. 336 */ 337static int 338bounce_bus_dma_tag_set_domain(bus_dma_tag_t dmat) 339{ 340 341 KASSERT(dmat->map_count == 0, 342 ("bounce_bus_dma_tag_set_domain: Domain set after use.\n")); 343 if ((dmat->bounce_flags & BF_COULD_BOUNCE) == 0 || 344 dmat->bounce_zone == NULL) 345 return (0); 346 dmat->bounce_flags &= ~BF_MIN_ALLOC_COMP; 347 return (bounce_bus_dma_zone_setup(dmat)); 348} 349 350static bool 351bounce_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen) 352{ 353 354 if (!might_bounce(dmat, NULL, buf, buflen)) 355 return (true); 356 return (!_bus_dmamap_pagesneeded(dmat, NULL, buf, buflen, NULL)); 357} 358 359static bus_dmamap_t 360alloc_dmamap(bus_dma_tag_t dmat, int flags) 361{ 362 u_long mapsize; 363 bus_dmamap_t map; 364 365 mapsize = sizeof(*map); 366 mapsize += sizeof(struct sync_list) * dmat->common.nsegments; 367 map = malloc_domainset(mapsize, M_DEVBUF, 368 DOMAINSET_PREF(dmat->common.domain), flags | M_ZERO); 369 if (map == NULL) 370 return (NULL); 371 372 /* Initialize the new map */ 373 STAILQ_INIT(&map->bpages); 374 375 return (map); 376} 377 378/* 379 * Allocate a handle for mapping from kva/uva/physical 380 * address space into bus device space. 381 */ 382static int 383bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 384{ 385 struct bounce_zone *bz; 386 int error, maxpages, pages; 387 388 error = 0; 389 390 if (dmat->segments == NULL) { 391 dmat->segments = mallocarray_domainset(dmat->common.nsegments, 392 sizeof(bus_dma_segment_t), M_DEVBUF, 393 DOMAINSET_PREF(dmat->common.domain), M_NOWAIT); 394 if (dmat->segments == NULL) { 395 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 396 __func__, dmat, ENOMEM); 397 return (ENOMEM); 398 } 399 } 400 401 *mapp = alloc_dmamap(dmat, M_NOWAIT); 402 if (*mapp == NULL) { 403 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 404 __func__, dmat, ENOMEM); 405 return (ENOMEM); 406 } 407 408 /* 409 * Bouncing might be required if the driver asks for an active 410 * exclusion region, a data alignment that is stricter than 1, and/or 411 * an active address boundary. 412 */ 413 if (dmat->bounce_zone == NULL) { 414 if ((error = alloc_bounce_zone(dmat)) != 0) { 415 free(*mapp, M_DEVBUF); 416 return (error); 417 } 418 } 419 bz = dmat->bounce_zone; 420 421 /* 422 * Attempt to add pages to our pool on a per-instance basis up to a sane 423 * limit. Even if the tag isn't subject of bouncing due to alignment 424 * and boundary constraints, it could still auto-bounce due to 425 * cacheline alignment, which requires at most two bounce pages. 426 */ 427 if (dmat->common.alignment > 1) 428 maxpages = MAX_BPAGES; 429 else 430 maxpages = MIN(MAX_BPAGES, Maxmem - 431 atop(dmat->common.lowaddr)); 432 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 || 433 (bz->map_count > 0 && bz->total_bpages < maxpages)) { 434 pages = atop(roundup2(dmat->common.maxsize, PAGE_SIZE)) + 1; 435 pages = MIN(maxpages - bz->total_bpages, pages); 436 pages = MAX(pages, 2); 437 if (alloc_bounce_pages(dmat, pages) < pages) 438 error = ENOMEM; 439 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0) { 440 if (error == 0) { 441 dmat->bounce_flags |= BF_MIN_ALLOC_COMP; 442 } 443 } else 444 error = 0; 445 } 446 bz->map_count++; 447 448 if (error == 0) { 449 dmat->map_count++; 450 if ((dmat->bounce_flags & BF_COHERENT) != 0) 451 (*mapp)->flags |= DMAMAP_COHERENT; 452 } else { 453 free(*mapp, M_DEVBUF); 454 } 455 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 456 __func__, dmat, dmat->common.flags, error); 457 return (error); 458} 459 460/* 461 * Destroy a handle for mapping from kva/uva/physical 462 * address space into bus device space. 463 */ 464static int 465bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 466{ 467 468 /* Check we are destroying the correct map type */ 469 if ((map->flags & DMAMAP_FROM_DMAMEM) != 0) 470 panic("bounce_bus_dmamap_destroy: Invalid map freed\n"); 471 472 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 473 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY); 474 return (EBUSY); 475 } 476 if (dmat->bounce_zone) 477 dmat->bounce_zone->map_count--; 478 free(map, M_DEVBUF); 479 dmat->map_count--; 480 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 481 return (0); 482} 483 484/* 485 * Allocate a piece of memory that can be efficiently mapped into 486 * bus device space based on the constraints lited in the dma tag. 487 * A dmamap to for use with dmamap_load is also allocated. 488 */ 489static int 490bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 491 bus_dmamap_t *mapp) 492{ 493 vm_memattr_t attr; 494 int mflags; 495 496 if (flags & BUS_DMA_NOWAIT) 497 mflags = M_NOWAIT; 498 else 499 mflags = M_WAITOK; 500 501 if (dmat->segments == NULL) { 502 dmat->segments = mallocarray_domainset(dmat->common.nsegments, 503 sizeof(bus_dma_segment_t), M_DEVBUF, 504 DOMAINSET_PREF(dmat->common.domain), mflags); 505 if (dmat->segments == NULL) { 506 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 507 __func__, dmat, dmat->common.flags, ENOMEM); 508 return (ENOMEM); 509 } 510 } 511 if (flags & BUS_DMA_ZERO) 512 mflags |= M_ZERO; 513 if (flags & BUS_DMA_NOCACHE) 514 attr = VM_MEMATTR_UNCACHEABLE; 515 else if ((flags & BUS_DMA_COHERENT) != 0 && 516 (dmat->bounce_flags & BF_COHERENT) == 0) 517 /* 518 * If we have a non-coherent tag, and are trying to allocate 519 * a coherent block of memory it needs to be uncached. 520 */ 521 attr = VM_MEMATTR_UNCACHEABLE; 522 else 523 attr = VM_MEMATTR_DEFAULT; 524 525 /* 526 * Create the map, but don't set the could bounce flag as 527 * this allocation should never bounce; 528 */ 529 *mapp = alloc_dmamap(dmat, mflags); 530 if (*mapp == NULL) { 531 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 532 __func__, dmat, dmat->common.flags, ENOMEM); 533 return (ENOMEM); 534 } 535 536 /* 537 * Mark the map as coherent if we used uncacheable memory or the 538 * tag was already marked as coherent. 539 */ 540 if (attr == VM_MEMATTR_UNCACHEABLE || 541 (dmat->bounce_flags & BF_COHERENT) != 0) 542 (*mapp)->flags |= DMAMAP_COHERENT; 543 544 (*mapp)->flags |= DMAMAP_FROM_DMAMEM; 545 546 /* 547 * Allocate the buffer from the malloc(9) allocator if... 548 * - It's small enough to fit into a single page. 549 * - Its alignment requirement is also smaller than the page size. 550 * - The low address requirement is fulfilled. 551 * - Default cache attributes are requested (WB). 552 * else allocate non-contiguous pages if... 553 * - The page count that could get allocated doesn't exceed 554 * nsegments also when the maximum segment size is less 555 * than PAGE_SIZE. 556 * - The alignment constraint isn't larger than a page boundary. 557 * - There are no boundary-crossing constraints. 558 * else allocate a block of contiguous pages because one or more of the 559 * constraints is something that only the contig allocator can fulfill. 560 * 561 * NOTE: The (dmat->common.alignment <= dmat->maxsize) check 562 * below is just a quick hack. The exact alignment guarantees 563 * of malloc(9) need to be nailed down, and the code below 564 * should be rewritten to take that into account. 565 * 566 * In the meantime warn the user if malloc gets it wrong. 567 */ 568 if (dmat->alloc_size <= PAGE_SIZE && 569 dmat->alloc_alignment <= PAGE_SIZE && 570 dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && 571 attr == VM_MEMATTR_DEFAULT) { 572 *vaddr = malloc_domainset_aligned(dmat->alloc_size, 573 dmat->alloc_alignment, M_DEVBUF, 574 DOMAINSET_PREF(dmat->common.domain), mflags); 575 } else if (dmat->common.nsegments >= 576 howmany(dmat->alloc_size, MIN(dmat->common.maxsegsz, PAGE_SIZE)) && 577 dmat->alloc_alignment <= PAGE_SIZE && 578 (dmat->common.boundary % PAGE_SIZE) == 0) { 579 /* Page-based multi-segment allocations allowed */ 580 *vaddr = kmem_alloc_attr_domainset( 581 DOMAINSET_PREF(dmat->common.domain), dmat->alloc_size, 582 mflags, 0ul, dmat->common.lowaddr, attr); 583 dmat->bounce_flags |= BF_KMEM_ALLOC; 584 } else { 585 *vaddr = kmem_alloc_contig_domainset( 586 DOMAINSET_PREF(dmat->common.domain), dmat->alloc_size, 587 mflags, 0ul, dmat->common.lowaddr, 588 dmat->alloc_alignment != 0 ? dmat->alloc_alignment : 1ul, 589 dmat->common.boundary, attr); 590 dmat->bounce_flags |= BF_KMEM_ALLOC; 591 } 592 if (*vaddr == NULL) { 593 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 594 __func__, dmat, dmat->common.flags, ENOMEM); 595 free(*mapp, M_DEVBUF); 596 return (ENOMEM); 597 } else if (!vm_addr_align_ok(vtophys(*vaddr), dmat->alloc_alignment)) { 598 printf("bus_dmamem_alloc failed to align memory properly.\n"); 599 } 600 dmat->map_count++; 601 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 602 __func__, dmat, dmat->common.flags, 0); 603 return (0); 604} 605 606/* 607 * Free a piece of memory and it's allociated dmamap, that was allocated 608 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 609 */ 610static void 611bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 612{ 613 614 /* 615 * Check the map came from bounce_bus_dmamem_alloc, so the map 616 * should be NULL and the BF_KMEM_ALLOC flag cleared if malloc() 617 * was used and set if kmem_alloc_contig() was used. 618 */ 619 if ((map->flags & DMAMAP_FROM_DMAMEM) == 0) 620 panic("bus_dmamem_free: Invalid map freed\n"); 621 if ((dmat->bounce_flags & BF_KMEM_ALLOC) == 0) 622 free(vaddr, M_DEVBUF); 623 else 624 kmem_free(vaddr, dmat->alloc_size); 625 free(map, M_DEVBUF); 626 dmat->map_count--; 627 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, 628 dmat->bounce_flags); 629} 630 631static bool 632_bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 633 bus_size_t buflen, int *pagesneeded) 634{ 635 bus_addr_t curaddr; 636 bus_size_t sgsize; 637 int count; 638 639 /* 640 * Count the number of bounce pages needed in order to 641 * complete this transfer 642 */ 643 count = 0; 644 curaddr = buf; 645 while (buflen != 0) { 646 sgsize = buflen; 647 if (must_bounce(dmat, map, curaddr, sgsize)) { 648 sgsize = MIN(sgsize, 649 PAGE_SIZE - (curaddr & PAGE_MASK)); 650 if (pagesneeded == NULL) 651 return (true); 652 count++; 653 } 654 curaddr += sgsize; 655 buflen -= sgsize; 656 } 657 658 if (pagesneeded != NULL) 659 *pagesneeded = count; 660 return (count != 0); 661} 662 663static void 664_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 665 bus_size_t buflen, int flags) 666{ 667 668 if (map->pagesneeded == 0) { 669 _bus_dmamap_pagesneeded(dmat, map, buf, buflen, 670 &map->pagesneeded); 671 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 672 } 673} 674 675static void 676_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 677 void *buf, bus_size_t buflen, int flags) 678{ 679 vm_offset_t vaddr; 680 vm_offset_t vendaddr; 681 bus_addr_t paddr; 682 bus_size_t sg_len; 683 684 if (map->pagesneeded == 0) { 685 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " 686 "alignment= %d", dmat->common.lowaddr, 687 ptoa((vm_paddr_t)Maxmem), 688 dmat->common.boundary, dmat->common.alignment); 689 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, 690 map->pagesneeded); 691 /* 692 * Count the number of bounce pages 693 * needed in order to complete this transfer 694 */ 695 vaddr = (vm_offset_t)buf; 696 vendaddr = (vm_offset_t)buf + buflen; 697 698 while (vaddr < vendaddr) { 699 sg_len = MIN(vendaddr - vaddr, 700 PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); 701 if (pmap == kernel_pmap) 702 paddr = pmap_kextract(vaddr); 703 else 704 paddr = pmap_extract(pmap, vaddr); 705 if (must_bounce(dmat, map, paddr, sg_len) != 0) { 706 sg_len = roundup2(sg_len, 707 dmat->common.alignment); 708 map->pagesneeded++; 709 } 710 vaddr += sg_len; 711 } 712 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 713 } 714} 715 716/* 717 * Utility function to load a physical buffer. segp contains 718 * the starting segment on entrace, and the ending segment on exit. 719 */ 720static int 721bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 722 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, 723 int *segp) 724{ 725 struct sync_list *sl; 726 bus_size_t sgsize; 727 bus_addr_t curaddr, sl_end; 728 int error; 729 730 if (segs == NULL) 731 segs = dmat->segments; 732 733 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 734 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 735 if (map->pagesneeded != 0) { 736 error = _bus_dmamap_reserve_pages(dmat, map, flags); 737 if (error) 738 return (error); 739 } 740 } 741 742 sl = map->slist + map->sync_count - 1; 743 sl_end = 0; 744 745 while (buflen > 0) { 746 curaddr = buf; 747 sgsize = buflen; 748 if (map->pagesneeded != 0 && 749 must_bounce(dmat, map, curaddr, sgsize)) { 750 /* 751 * The attempt to split a physically continuous buffer 752 * seems very controversial, it's unclear whether we 753 * can do this in all cases. Also, memory for bounced 754 * buffers is allocated as pages, so we cannot 755 * guarantee multipage alignment. 756 */ 757 KASSERT(dmat->common.alignment <= PAGE_SIZE, 758 ("bounced buffer cannot have alignment bigger " 759 "than PAGE_SIZE: %lu", dmat->common.alignment)); 760 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 761 curaddr = add_bounce_page(dmat, map, 0, curaddr, 762 sgsize); 763 } else if ((map->flags & DMAMAP_COHERENT) == 0) { 764 if (map->sync_count > 0) 765 sl_end = sl->paddr + sl->datacount; 766 767 if (map->sync_count == 0 || curaddr != sl_end) { 768 if (++map->sync_count > dmat->common.nsegments) 769 break; 770 sl++; 771 sl->vaddr = 0; 772 sl->paddr = curaddr; 773 sl->pages = PHYS_TO_VM_PAGE(curaddr); 774 KASSERT(sl->pages != NULL, 775 ("%s: page at PA:0x%08lx is not in " 776 "vm_page_array", __func__, curaddr)); 777 sl->datacount = sgsize; 778 } else 779 sl->datacount += sgsize; 780 } 781 if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs, 782 segp)) 783 break; 784 buf += sgsize; 785 buflen -= sgsize; 786 } 787 788 /* 789 * Did we fit? 790 */ 791 if (buflen != 0) { 792 bus_dmamap_unload(dmat, map); 793 return (EFBIG); /* XXX better return value here? */ 794 } 795 return (0); 796} 797 798/* 799 * Utility function to load a linear buffer. segp contains 800 * the starting segment on entrace, and the ending segment on exit. 801 */ 802static int 803bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 804 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 805 int *segp) 806{ 807 struct sync_list *sl; 808 bus_size_t sgsize; 809 bus_addr_t curaddr, sl_pend; 810 vm_offset_t kvaddr, vaddr, sl_vend; 811 int error; 812 813 KASSERT((map->flags & DMAMAP_FROM_DMAMEM) != 0 || 814 dmat->common.alignment <= PAGE_SIZE, 815 ("loading user buffer with alignment bigger than PAGE_SIZE is not " 816 "supported")); 817 818 if (segs == NULL) 819 segs = dmat->segments; 820 821 if (flags & BUS_DMA_LOAD_MBUF) 822 map->flags |= DMAMAP_MBUF; 823 824 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 825 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 826 if (map->pagesneeded != 0) { 827 error = _bus_dmamap_reserve_pages(dmat, map, flags); 828 if (error) 829 return (error); 830 } 831 } 832 833 /* 834 * XXX Optimally we should parse input buffer for physically 835 * continuous segments first and then pass these segment into 836 * load loop. 837 */ 838 sl = map->slist + map->sync_count - 1; 839 vaddr = (vm_offset_t)buf; 840 sl_pend = 0; 841 sl_vend = 0; 842 843 while (buflen > 0) { 844 /* 845 * Get the physical address for this segment. 846 */ 847 if (__predict_true(pmap == kernel_pmap)) { 848 curaddr = pmap_kextract(vaddr); 849 kvaddr = vaddr; 850 } else { 851 curaddr = pmap_extract(pmap, vaddr); 852 kvaddr = 0; 853 } 854 855 /* 856 * Compute the segment size, and adjust counts. 857 */ 858 sgsize = buflen; 859 if ((map->flags & DMAMAP_FROM_DMAMEM) == 0) 860 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 861 862 if (map->pagesneeded != 0 && 863 must_bounce(dmat, map, curaddr, sgsize)) { 864 /* See comment in bounce_bus_dmamap_load_phys */ 865 KASSERT(dmat->common.alignment <= PAGE_SIZE, 866 ("bounced buffer cannot have alignment bigger " 867 "than PAGE_SIZE: %lu", dmat->common.alignment)); 868 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 869 sgsize); 870 } else if ((map->flags & DMAMAP_COHERENT) == 0) { 871 if (map->sync_count > 0) { 872 sl_pend = sl->paddr + sl->datacount; 873 sl_vend = sl->vaddr + sl->datacount; 874 } 875 876 if (map->sync_count == 0 || 877 (kvaddr != 0 && kvaddr != sl_vend) || 878 (curaddr != sl_pend)) { 879 if (++map->sync_count > dmat->common.nsegments) 880 break; 881 sl++; 882 sl->vaddr = kvaddr; 883 sl->paddr = curaddr; 884 if (kvaddr != 0) { 885 sl->pages = NULL; 886 } else { 887 sl->pages = PHYS_TO_VM_PAGE(curaddr); 888 KASSERT(sl->pages != NULL, 889 ("%s: page at PA:0x%08lx is not " 890 "in vm_page_array", __func__, 891 curaddr)); 892 } 893 sl->datacount = sgsize; 894 } else 895 sl->datacount += sgsize; 896 } 897 if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs, 898 segp)) 899 break; 900 vaddr += sgsize; 901 buflen -= sgsize; 902 } 903 904 /* 905 * Did we fit? 906 */ 907 if (buflen != 0) { 908 bus_dmamap_unload(dmat, map); 909 return (EFBIG); /* XXX better return value here? */ 910 } 911 return (0); 912} 913 914static void 915bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 916 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 917{ 918 919 map->mem = *mem; 920 map->dmat = dmat; 921 map->callback = callback; 922 map->callback_arg = callback_arg; 923} 924 925static bus_dma_segment_t * 926bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 927 bus_dma_segment_t *segs, int nsegs, int error) 928{ 929 930 if (segs == NULL) 931 segs = dmat->segments; 932 return (segs); 933} 934 935/* 936 * Release the mapping held by map. 937 */ 938static void 939bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 940{ 941 free_bounce_pages(dmat, map); 942 map->sync_count = 0; 943 map->flags &= ~DMAMAP_MBUF; 944} 945 946static void 947dma_preread_safe(char *va, vm_size_t size) 948{ 949 /* 950 * Write back any partial cachelines immediately before and 951 * after the DMA region. 952 */ 953 if (!__is_aligned(va, dcache_line_size)) 954 cpu_dcache_wb_range(va, 1); 955 if (!__is_aligned(va + size, dcache_line_size)) 956 cpu_dcache_wb_range(va + size, 1); 957 958 cpu_dcache_inv_range(va, size); 959} 960 961static void 962dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op) 963{ 964 uint32_t len, offset; 965 vm_page_t m; 966 vm_paddr_t pa; 967 vm_offset_t va, tempva; 968 bus_size_t size; 969 970 offset = sl->paddr & PAGE_MASK; 971 m = sl->pages; 972 size = sl->datacount; 973 pa = sl->paddr; 974 975 for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) { 976 tempva = 0; 977 if (sl->vaddr == 0) { 978 len = min(PAGE_SIZE - offset, size); 979 tempva = pmap_quick_enter_page(m); 980 va = tempva | offset; 981 KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset), 982 ("unexpected vm_page_t phys: 0x%16lx != 0x%16lx", 983 VM_PAGE_TO_PHYS(m) | offset, pa)); 984 } else { 985 len = sl->datacount; 986 va = sl->vaddr; 987 } 988 989 switch (op) { 990 case BUS_DMASYNC_PREWRITE: 991 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: 992 cpu_dcache_wb_range((void *)va, len); 993 break; 994 case BUS_DMASYNC_PREREAD: 995 /* 996 * An mbuf may start in the middle of a cacheline. There 997 * will be no cpu writes to the beginning of that line 998 * (which contains the mbuf header) while dma is in 999 * progress. Handle that case by doing a writeback of 1000 * just the first cacheline before invalidating the 1001 * overall buffer. Any mbuf in a chain may have this 1002 * misalignment. Buffers which are not mbufs bounce if 1003 * they are not aligned to a cacheline. 1004 */ 1005 dma_preread_safe((void *)va, len); 1006 break; 1007 case BUS_DMASYNC_POSTREAD: 1008 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: 1009 cpu_dcache_inv_range((void *)va, len); 1010 break; 1011 default: 1012 panic("unsupported combination of sync operations: " 1013 "0x%08x\n", op); 1014 } 1015 1016 if (tempva != 0) 1017 pmap_quick_remove_page(tempva); 1018 } 1019} 1020 1021static void 1022bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, 1023 bus_dmasync_op_t op) 1024{ 1025 struct bounce_page *bpage; 1026 struct sync_list *sl, *end; 1027 vm_offset_t datavaddr, tempvaddr; 1028 1029 if (op == BUS_DMASYNC_POSTWRITE) 1030 return; 1031 1032 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1033 /* 1034 * Wait for any DMA operations to complete before the bcopy. 1035 */ 1036 dsb(sy); 1037 } 1038 1039 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1040 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1041 "performing bounce", __func__, dmat, dmat->common.flags, 1042 op); 1043 1044 if ((op & BUS_DMASYNC_PREWRITE) != 0) { 1045 while (bpage != NULL) { 1046 tempvaddr = 0; 1047 datavaddr = bpage->datavaddr; 1048 if (datavaddr == 0) { 1049 tempvaddr = pmap_quick_enter_page( 1050 bpage->datapage); 1051 datavaddr = tempvaddr | bpage->dataoffs; 1052 } 1053 1054 bcopy((void *)datavaddr, 1055 (void *)bpage->vaddr, bpage->datacount); 1056 if (tempvaddr != 0) 1057 pmap_quick_remove_page(tempvaddr); 1058 if ((map->flags & DMAMAP_COHERENT) == 0) 1059 cpu_dcache_wb_range((void *)bpage->vaddr, 1060 bpage->datacount); 1061 bpage = STAILQ_NEXT(bpage, links); 1062 } 1063 dmat->bounce_zone->total_bounced++; 1064 } else if ((op & BUS_DMASYNC_PREREAD) != 0) { 1065 while (bpage != NULL) { 1066 if ((map->flags & DMAMAP_COHERENT) == 0) 1067 cpu_dcache_wbinv_range((void *)bpage->vaddr, 1068 bpage->datacount); 1069 bpage = STAILQ_NEXT(bpage, links); 1070 } 1071 } 1072 1073 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1074 while (bpage != NULL) { 1075 if ((map->flags & DMAMAP_COHERENT) == 0) 1076 cpu_dcache_inv_range((void *)bpage->vaddr, 1077 bpage->datacount); 1078 tempvaddr = 0; 1079 datavaddr = bpage->datavaddr; 1080 if (datavaddr == 0) { 1081 tempvaddr = pmap_quick_enter_page( 1082 bpage->datapage); 1083 datavaddr = tempvaddr | bpage->dataoffs; 1084 } 1085 1086 bcopy((void *)bpage->vaddr, 1087 (void *)datavaddr, bpage->datacount); 1088 1089 if (tempvaddr != 0) 1090 pmap_quick_remove_page(tempvaddr); 1091 bpage = STAILQ_NEXT(bpage, links); 1092 } 1093 dmat->bounce_zone->total_bounced++; 1094 } 1095 } 1096 1097 /* 1098 * Cache maintenance for normal (non-COHERENT non-bounce) buffers. 1099 */ 1100 if (map->sync_count != 0) { 1101 sl = &map->slist[0]; 1102 end = &map->slist[map->sync_count]; 1103 CTR3(KTR_BUSDMA, "%s: tag %p op 0x%x " 1104 "performing sync", __func__, dmat, op); 1105 1106 for ( ; sl != end; ++sl) 1107 dma_dcache_sync(sl, op); 1108 } 1109 1110 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) != 0) { 1111 /* 1112 * Wait for the bcopy to complete before any DMA operations. 1113 */ 1114 dsb(sy); 1115 } 1116 1117 kmsan_bus_dmamap_sync(&map->kmsan_mem, op); 1118} 1119 1120#ifdef KMSAN 1121static void 1122bounce_bus_dmamap_load_kmsan(bus_dmamap_t map, struct memdesc *mem) 1123{ 1124 if (map == NULL) 1125 return; 1126 memcpy(&map->kmsan_mem, mem, sizeof(map->kmsan_mem)); 1127} 1128#endif 1129 1130struct bus_dma_impl bus_dma_bounce_impl = { 1131 .tag_create = bounce_bus_dma_tag_create, 1132 .tag_destroy = bounce_bus_dma_tag_destroy, 1133 .tag_set_domain = bounce_bus_dma_tag_set_domain, 1134 .id_mapped = bounce_bus_dma_id_mapped, 1135 .map_create = bounce_bus_dmamap_create, 1136 .map_destroy = bounce_bus_dmamap_destroy, 1137 .mem_alloc = bounce_bus_dmamem_alloc, 1138 .mem_free = bounce_bus_dmamem_free, 1139 .load_phys = bounce_bus_dmamap_load_phys, 1140 .load_buffer = bounce_bus_dmamap_load_buffer, 1141 .load_ma = bus_dmamap_load_ma_triv, 1142 .map_waitok = bounce_bus_dmamap_waitok, 1143 .map_complete = bounce_bus_dmamap_complete, 1144 .map_unload = bounce_bus_dmamap_unload, 1145 .map_sync = bounce_bus_dmamap_sync, 1146#ifdef KMSAN 1147 .load_kmsan = bounce_bus_dmamap_load_kmsan, 1148#endif 1149}; 1150