busdma_machdep.c revision 32516
1/* 2 * Copyright (c) 1997 Justin T. Gibbs. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions, and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $Id$ 27 */ 28 29#include <sys/param.h> 30#include <sys/systm.h> 31#include <sys/malloc.h> 32#include <sys/queue.h> 33 34#include <vm/vm.h> 35#include <vm/vm_prot.h> 36#include <vm/vm_page.h> 37 38#include <machine/bus.h> 39#include <machine/md_var.h> 40 41#define MAX(a,b) (((a) > (b)) ? (a) : (b)) 42#define MIN(a,b) (((a) < (b)) ? (a) : (b)) 43#define MAX_BPAGES 128 44 45struct bus_dma_tag { 46 bus_dma_tag_t parent; 47 bus_size_t boundary; 48 bus_addr_t lowaddr; 49 bus_addr_t highaddr; 50 bus_dma_filter_t *filter; 51 void *filterarg; 52 bus_size_t maxsize; 53 int nsegments; 54 bus_size_t maxsegsz; 55 int flags; 56 int ref_count; 57 int map_count; 58}; 59 60struct bounce_page { 61 vm_offset_t vaddr; /* kva of bounce buffer */ 62 bus_addr_t busaddr; /* Physical address */ 63 vm_offset_t datavaddr; /* kva of client data */ 64 bus_size_t datacount; /* client data count */ 65 STAILQ_ENTRY(bounce_page) links; 66}; 67 68int busdma_swi_pending; 69 70static STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 71static int free_bpages; 72static int reserved_bpages; 73static int active_bpages; 74static int total_bpages; 75static bus_addr_t bounce_lowaddr = BUS_SPACE_MAXADDR; 76 77struct bus_dmamap { 78 struct bp_list bpages; 79 int pagesneeded; 80 int pagesreserved; 81 bus_dma_tag_t dmat; 82 void *buf; /* unmapped buffer pointer */ 83 bus_size_t buflen; /* unmapped buffer length */ 84 bus_dmamap_callback_t *callback; 85 void *callback_arg; 86 STAILQ_ENTRY(bus_dmamap) links; 87}; 88 89static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 90static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 91static struct bus_dmamap nobounce_dmamap; 92 93static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 94static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map); 95static vm_offset_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 96 vm_offset_t vaddr, bus_size_t size); 97static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 98static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 99 100static __inline int 101run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 102{ 103 int retval; 104 105 retval = 0; 106 do { 107 if (paddr > dmat->lowaddr 108 && paddr <= dmat->highaddr 109 && (dmat->filter == NULL 110 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 111 retval = 1; 112 113 dmat = dmat->parent; 114 } while (retval == 0 && dmat != NULL); 115 return (retval); 116} 117 118/* 119 * Allocate a device specific dma_tag. 120 */ 121int 122bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t boundary, 123 bus_addr_t lowaddr, bus_addr_t highaddr, 124 bus_dma_filter_t *filter, void *filterarg, 125 bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, 126 int flags, bus_dma_tag_t *dmat) 127{ 128 bus_dma_tag_t newtag; 129 int error = 0; 130 131 /* Return a NULL tag on failure */ 132 *dmat = NULL; 133 134 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 135 if (newtag == NULL) 136 return (ENOMEM); 137 138 newtag->parent = parent; 139 newtag->boundary = boundary; 140 newtag->lowaddr = trunc_page(lowaddr) + (PAGE_SIZE - 1); 141 newtag->highaddr = trunc_page(highaddr) + (PAGE_SIZE - 1); 142 newtag->filter = filter; 143 newtag->filterarg = filterarg; 144 newtag->maxsize = maxsize; 145 newtag->nsegments = nsegments; 146 newtag->maxsegsz = maxsegsz; 147 newtag->flags = flags; 148 newtag->ref_count = 1; /* Count ourself */ 149 newtag->map_count = 0; 150 151 /* Take into account any restrictions imposed by our parent tag */ 152 if (parent != NULL) { 153 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 154 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 155 /* 156 * XXX Not really correct??? Probably need to honor boundary 157 * all the way up the inheritence chain. 158 */ 159 newtag->boundary = MIN(parent->boundary, newtag->boundary); 160 if (newtag->filter == NULL) { 161 /* 162 * Short circuit looking at our parent directly 163 * since we have encapsulated all of it's information 164 */ 165 newtag->filter = parent->filter; 166 newtag->filterarg = parent->filterarg; 167 newtag->parent = parent->parent; 168 } 169 if (newtag->parent != NULL) { 170 parent->ref_count++; 171 } 172 } 173 174 if (newtag->lowaddr < ptoa(Maxmem)) { 175 /* Must bounce */ 176 177 if (lowaddr > bounce_lowaddr) { 178 /* 179 * Go through the pool and kill any pages 180 * that don't reside below lowaddr. 181 */ 182 panic("bus_dmamap_create: page reallocation " 183 "not implemented"); 184 } 185 if (ptoa(total_bpages) < maxsize) { 186 int pages; 187 188 pages = atop(maxsize) - total_bpages; 189 190 /* Add pages to our bounce pool */ 191 if (alloc_bounce_pages(newtag, pages) < pages) 192 error = ENOMEM; 193 } 194 } 195 196 if (error != 0) { 197 free(newtag, M_DEVBUF); 198 } else { 199 *dmat = newtag; 200 } 201 return (error); 202} 203 204int 205bus_dma_tag_destroy(bus_dma_tag_t dmat) 206{ 207 if (dmat != NULL) { 208 209 if (dmat->map_count != 0) 210 return (EBUSY); 211 212 while (dmat != NULL) { 213 bus_dma_tag_t parent; 214 215 parent = dmat->parent; 216 dmat->ref_count--; 217 if (dmat->ref_count == 0) { 218 free(dmat, M_DEVBUF); 219 } 220 dmat = parent; 221 } 222 } 223 return (0); 224} 225 226/* 227 * Allocate a handle for mapping from kva/uva/physical 228 * address space into bus device space. 229 */ 230int 231bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 232{ 233 int error; 234 235 error = 0; 236 237 if (dmat->lowaddr < ptoa(Maxmem)) { 238 /* Must bounce */ 239 int maxpages; 240 241 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 242 M_NOWAIT); 243 if (*mapp == NULL) { 244 error = ENOMEM; 245 } else { 246 /* Initialize the new map */ 247 bzero(*mapp, sizeof(**mapp)); 248 STAILQ_INIT(&((*mapp)->bpages)); 249 } 250 /* 251 * Attempt to add pages to our pool on a per-instance 252 * basis up to a sane limit. 253 */ 254 maxpages = MIN(MAX_BPAGES, Maxmem - atop(dmat->lowaddr)); 255 if (dmat->map_count > 0 256 && total_bpages < maxpages) { 257 int pages; 258 259 pages = atop(dmat->maxsize); 260 pages = MIN(maxpages - total_bpages, pages); 261 alloc_bounce_pages(dmat, pages); 262 } 263 } else { 264 *mapp = NULL; 265 } 266 if (error == 0) 267 dmat->map_count++; 268 return (error); 269} 270 271/* 272 * Destroy a handle for mapping from kva/uva/physical 273 * address space into bus device space. 274 */ 275int 276bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 277{ 278 if (map != NULL) { 279 if (STAILQ_FIRST(&map->bpages) != NULL) 280 return (EBUSY); 281 free(map, M_DEVBUF); 282 } 283 dmat->map_count--; 284 return (0); 285} 286 287#define BUS_DMAMAP_NSEGS ((BUS_SPACE_MAXSIZE / PAGE_SIZE) + 1) 288 289/* 290 * Map the buffer buf into bus space using the dmamap map. 291 */ 292int 293bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 294 bus_size_t buflen, bus_dmamap_callback_t *callback, 295 void *callback_arg, int flags) 296{ 297 vm_offset_t vaddr; 298 vm_offset_t paddr; 299#ifdef __GNUC__ 300 bus_dma_segment_t dm_segments[dmat->nsegments]; 301#else 302 bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 303#endif 304 bus_dma_segment_t *sg; 305 int seg; 306 int error; 307 308 error = 0; 309 /* 310 * If we are being called during a callback, pagesneeded will 311 * be non-zero, so we can avoid doing the work twice. 312 */ 313 if (dmat->lowaddr < ptoa(Maxmem) && map->pagesneeded == 0) { 314 vm_offset_t vendaddr; 315 316 /* 317 * Count the number of bounce pages 318 * needed in order to complete this transfer 319 */ 320 vaddr = trunc_page(buf); 321 vendaddr = (vm_offset_t)buf + buflen; 322 323 while (vaddr < vendaddr) { 324 paddr = pmap_kextract(vaddr); 325 if (run_filter(dmat, paddr) != 0) { 326 327 map->pagesneeded++; 328 } 329 vaddr += PAGE_SIZE; 330 } 331 } 332 333 if (map == NULL) 334 map = &nobounce_dmamap; 335 336 /* Reserve Necessary Bounce Pages */ 337 if (map->pagesneeded != 0) { 338 int s; 339 340 s = splhigh(); 341 if (reserve_bounce_pages(dmat, map) != 0) { 342 343 /* Queue us for resources */ 344 map->dmat = dmat; 345 map->buf = buf; 346 map->buflen = buflen; 347 map->callback = callback; 348 map->callback_arg = callback_arg; 349 350 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 351 splx(s); 352 353 return (EINPROGRESS); 354 } 355 splx(s); 356 } 357 358 vaddr = (vm_offset_t)buf; 359 sg = &dm_segments[0]; 360 seg = 1; 361 sg->ds_len = 0; 362 363 do { 364 bus_size_t size; 365 vm_offset_t nextpaddr; 366 367 paddr = pmap_kextract(vaddr); 368 size = PAGE_SIZE - (paddr & PAGE_MASK); 369 if (size > buflen) 370 size = buflen; 371 372 if (map->pagesneeded != 0 373 && run_filter(dmat, paddr)) { 374 paddr = add_bounce_page(dmat, map, vaddr, size); 375 } 376 377 if (sg->ds_len == 0) { 378 sg->ds_addr = paddr; 379 sg->ds_len = size; 380 } else if (paddr == nextpaddr) { 381 sg->ds_len += size; 382 } else { 383 /* Go to the next segment */ 384 sg++; 385 seg++; 386 if (seg > dmat->nsegments) 387 break; 388 sg->ds_addr = paddr; 389 sg->ds_len = size; 390 } 391 vaddr += size; 392 nextpaddr = paddr + size; 393 buflen -= size; 394 } while (buflen > 0); 395 396 if (buflen != 0) { 397 printf("bus_dmamap_load: Too many segs!\n"); 398 error = EFBIG; 399 } 400 401 (*callback)(callback_arg, dm_segments, seg, error); 402 403 return (0); 404} 405 406/* 407 * Release the mapping held by map. 408 */ 409void 410_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 411{ 412 struct bounce_page *bpage; 413 414 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 415 STAILQ_REMOVE_HEAD(&map->bpages, links); 416 free_bounce_page(dmat, bpage); 417 } 418} 419 420void 421_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 422{ 423 struct bounce_page *bpage; 424 425 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 426 427 /* 428 * Handle data bouncing. We might also 429 * want to add support for invalidating 430 * the caches on broken hardware 431 */ 432 switch (op) { 433 case BUS_DMASYNC_PREWRITE: 434 while (bpage != NULL) { 435 bcopy((void *)bpage->datavaddr, 436 (void *)bpage->vaddr, 437 bpage->datacount); 438 bpage = STAILQ_NEXT(bpage, links); 439 } 440 break; 441 442 case BUS_DMASYNC_POSTREAD: 443 while (bpage != NULL) { 444 bcopy((void *)bpage->vaddr, 445 (void *)bpage->datavaddr, 446 bpage->datacount); 447 bpage = STAILQ_NEXT(bpage, links); 448 } 449 break; 450 case BUS_DMASYNC_PREREAD: 451 case BUS_DMASYNC_POSTWRITE: 452 /* No-ops */ 453 break; 454 } 455 } 456} 457 458static int 459alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 460{ 461 int count; 462 463 count = 0; 464 if (total_bpages == 0) { 465 STAILQ_INIT(&bounce_page_list); 466 STAILQ_INIT(&bounce_map_waitinglist); 467 STAILQ_INIT(&bounce_map_callbacklist); 468 } 469 470 while (numpages > 0) { 471 struct bounce_page *bpage; 472 int s; 473 474 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 475 M_NOWAIT); 476 477 if (bpage == NULL) 478 break; 479 bzero(bpage, sizeof(*bpage)); 480 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 481 M_NOWAIT, 0ul, 482 dmat->lowaddr, 483 PAGE_SIZE, 0x10000); 484 if (bpage->vaddr == NULL) { 485 free(bpage, M_DEVBUF); 486 break; 487 } 488 bpage->busaddr = pmap_kextract(bpage->vaddr); 489 s = splhigh(); 490 STAILQ_INSERT_TAIL(&bounce_page_list, bpage, links); 491 total_bpages++; 492 free_bpages++; 493 splx(s); 494 count++; 495 numpages--; 496 } 497 return (count); 498} 499 500static int 501reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map) 502{ 503 int pages; 504 505 pages = MIN(free_bpages, map->pagesneeded - map->pagesreserved); 506 free_bpages -= pages; 507 reserved_bpages += pages; 508 map->pagesreserved += pages; 509 pages = map->pagesneeded - map->pagesreserved; 510 511 return (pages); 512} 513 514static vm_offset_t 515add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 516 bus_size_t size) 517{ 518 int s; 519 struct bounce_page *bpage; 520 521 if (map->pagesneeded == 0) 522 panic("add_bounce_page: map doesn't need any pages"); 523 map->pagesneeded--; 524 525 if (map->pagesreserved == 0) 526 panic("add_bounce_page: map doesn't need any pages"); 527 map->pagesreserved--; 528 529 s = splhigh(); 530 bpage = STAILQ_FIRST(&bounce_page_list); 531 if (bpage == NULL) 532 panic("add_bounce_page: free page list is empty"); 533 534 STAILQ_REMOVE_HEAD(&bounce_page_list, links); 535 reserved_bpages--; 536 active_bpages++; 537 splx(s); 538 539 bpage->datavaddr = vaddr; 540 bpage->datacount = size; 541 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 542 return (bpage->busaddr); 543} 544 545static void 546free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 547{ 548 int s; 549 struct bus_dmamap *map; 550 551 bpage->datavaddr = 0; 552 bpage->datacount = 0; 553 554 s = splhigh(); 555 STAILQ_INSERT_HEAD(&bounce_page_list, bpage, links); 556 free_bpages++; 557 active_bpages--; 558 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 559 if (reserve_bounce_pages(map->dmat, map) == 0) { 560 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 561 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 562 map, links); 563 busdma_swi_pending = 1; 564 setsoftvm(); 565 } 566 } 567 splx(s); 568} 569 570void 571busdma_swi() 572{ 573 int s; 574 struct bus_dmamap *map; 575 576 s = splhigh(); 577 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 578 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 579 splx(s); 580 bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 581 map->callback, map->callback_arg, /*flags*/0); 582 s = splhigh(); 583 } 584 splx(s); 585} 586