busdma_machdep.c revision 35256
1/* 2 * Copyright (c) 1997 Justin T. Gibbs. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions, and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $Id: busdma_machdep.c,v 1.4 1998/02/20 13:11:47 bde Exp $ 27 */ 28 29#include <sys/param.h> 30#include <sys/systm.h> 31#include <sys/malloc.h> 32 33#include <vm/vm.h> 34#include <vm/vm_prot.h> 35#include <vm/vm_page.h> 36 37#include <machine/bus.h> 38#include <machine/md_var.h> 39 40#define MAX(a,b) (((a) > (b)) ? (a) : (b)) 41#define MIN(a,b) (((a) < (b)) ? (a) : (b)) 42#define MAX_BPAGES 128 43 44struct bus_dma_tag { 45 bus_dma_tag_t parent; 46 bus_size_t boundary; 47 bus_addr_t lowaddr; 48 bus_addr_t highaddr; 49 bus_dma_filter_t *filter; 50 void *filterarg; 51 bus_size_t maxsize; 52 int nsegments; 53 bus_size_t maxsegsz; 54 int flags; 55 int ref_count; 56 int map_count; 57}; 58 59struct bounce_page { 60 vm_offset_t vaddr; /* kva of bounce buffer */ 61 bus_addr_t busaddr; /* Physical address */ 62 vm_offset_t datavaddr; /* kva of client data */ 63 bus_size_t datacount; /* client data count */ 64 STAILQ_ENTRY(bounce_page) links; 65}; 66 67int busdma_swi_pending; 68 69static STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 70static int free_bpages; 71static int reserved_bpages; 72static int active_bpages; 73static int total_bpages; 74static bus_addr_t bounce_lowaddr = BUS_SPACE_MAXADDR; 75 76struct bus_dmamap { 77 struct bp_list bpages; 78 int pagesneeded; 79 int pagesreserved; 80 bus_dma_tag_t dmat; 81 void *buf; /* unmapped buffer pointer */ 82 bus_size_t buflen; /* unmapped buffer length */ 83 bus_dmamap_callback_t *callback; 84 void *callback_arg; 85 STAILQ_ENTRY(bus_dmamap) links; 86}; 87 88static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 89static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 90static struct bus_dmamap nobounce_dmamap; 91 92static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 93static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map); 94static vm_offset_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 95 vm_offset_t vaddr, bus_size_t size); 96static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 97static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 98 99static __inline int 100run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 101{ 102 int retval; 103 104 retval = 0; 105 do { 106 if (paddr > dmat->lowaddr 107 && paddr <= dmat->highaddr 108 && (dmat->filter == NULL 109 || (*dmat->filter)(dmat->filterarg, paddr) != 0)) 110 retval = 1; 111 112 dmat = dmat->parent; 113 } while (retval == 0 && dmat != NULL); 114 return (retval); 115} 116 117/* 118 * Allocate a device specific dma_tag. 119 */ 120int 121bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t boundary, 122 bus_addr_t lowaddr, bus_addr_t highaddr, 123 bus_dma_filter_t *filter, void *filterarg, 124 bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, 125 int flags, bus_dma_tag_t *dmat) 126{ 127 bus_dma_tag_t newtag; 128 int error = 0; 129 130 /* Return a NULL tag on failure */ 131 *dmat = NULL; 132 133 newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); 134 if (newtag == NULL) 135 return (ENOMEM); 136 137 newtag->parent = parent; 138 newtag->boundary = boundary; 139 newtag->lowaddr = trunc_page(lowaddr) + (PAGE_SIZE - 1); 140 newtag->highaddr = trunc_page(highaddr) + (PAGE_SIZE - 1); 141 newtag->filter = filter; 142 newtag->filterarg = filterarg; 143 newtag->maxsize = maxsize; 144 newtag->nsegments = nsegments; 145 newtag->maxsegsz = maxsegsz; 146 newtag->flags = flags; 147 newtag->ref_count = 1; /* Count ourself */ 148 newtag->map_count = 0; 149 150 /* Take into account any restrictions imposed by our parent tag */ 151 if (parent != NULL) { 152 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 153 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 154 /* 155 * XXX Not really correct??? Probably need to honor boundary 156 * all the way up the inheritence chain. 157 */ 158 newtag->boundary = MIN(parent->boundary, newtag->boundary); 159 if (newtag->filter == NULL) { 160 /* 161 * Short circuit looking at our parent directly 162 * since we have encapsulated all of its information 163 */ 164 newtag->filter = parent->filter; 165 newtag->filterarg = parent->filterarg; 166 newtag->parent = parent->parent; 167 } 168 if (newtag->parent != NULL) { 169 parent->ref_count++; 170 } 171 } 172 173 if (newtag->lowaddr < ptoa(Maxmem)) { 174 /* Must bounce */ 175 176 if (lowaddr > bounce_lowaddr) { 177 /* 178 * Go through the pool and kill any pages 179 * that don't reside below lowaddr. 180 */ 181 panic("bus_dmamap_create: page reallocation " 182 "not implemented"); 183 } 184 if (ptoa(total_bpages) < maxsize) { 185 int pages; 186 187 pages = atop(maxsize) - total_bpages; 188 189 /* Add pages to our bounce pool */ 190 if (alloc_bounce_pages(newtag, pages) < pages) 191 error = ENOMEM; 192 } 193 } 194 195 if (error != 0) { 196 free(newtag, M_DEVBUF); 197 } else { 198 *dmat = newtag; 199 } 200 return (error); 201} 202 203int 204bus_dma_tag_destroy(bus_dma_tag_t dmat) 205{ 206 if (dmat != NULL) { 207 208 if (dmat->map_count != 0) 209 return (EBUSY); 210 211 while (dmat != NULL) { 212 bus_dma_tag_t parent; 213 214 parent = dmat->parent; 215 dmat->ref_count--; 216 if (dmat->ref_count == 0) { 217 free(dmat, M_DEVBUF); 218 } 219 dmat = parent; 220 } 221 } 222 return (0); 223} 224 225/* 226 * Allocate a handle for mapping from kva/uva/physical 227 * address space into bus device space. 228 */ 229int 230bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 231{ 232 int error; 233 234 error = 0; 235 236 if (dmat->lowaddr < ptoa(Maxmem)) { 237 /* Must bounce */ 238 int maxpages; 239 240 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, 241 M_NOWAIT); 242 if (*mapp == NULL) { 243 error = ENOMEM; 244 } else { 245 /* Initialize the new map */ 246 bzero(*mapp, sizeof(**mapp)); 247 STAILQ_INIT(&((*mapp)->bpages)); 248 } 249 /* 250 * Attempt to add pages to our pool on a per-instance 251 * basis up to a sane limit. 252 */ 253 maxpages = MIN(MAX_BPAGES, Maxmem - atop(dmat->lowaddr)); 254 if (dmat->map_count > 0 255 && total_bpages < maxpages) { 256 int pages; 257 258 pages = atop(dmat->maxsize); 259 pages = MIN(maxpages - total_bpages, pages); 260 alloc_bounce_pages(dmat, pages); 261 } 262 } else { 263 *mapp = NULL; 264 } 265 if (error == 0) 266 dmat->map_count++; 267 return (error); 268} 269 270/* 271 * Destroy a handle for mapping from kva/uva/physical 272 * address space into bus device space. 273 */ 274int 275bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 276{ 277 if (map != NULL) { 278 if (STAILQ_FIRST(&map->bpages) != NULL) 279 return (EBUSY); 280 free(map, M_DEVBUF); 281 } 282 dmat->map_count--; 283 return (0); 284} 285 286#define BUS_DMAMAP_NSEGS ((BUS_SPACE_MAXSIZE / PAGE_SIZE) + 1) 287 288/* 289 * Map the buffer buf into bus space using the dmamap map. 290 */ 291int 292bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 293 bus_size_t buflen, bus_dmamap_callback_t *callback, 294 void *callback_arg, int flags) 295{ 296 vm_offset_t vaddr; 297 vm_offset_t paddr; 298#ifdef __GNUC__ 299 bus_dma_segment_t dm_segments[dmat->nsegments]; 300#else 301 bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; 302#endif 303 bus_dma_segment_t *sg; 304 int seg; 305 int error; 306 307 error = 0; 308 /* 309 * If we are being called during a callback, pagesneeded will 310 * be non-zero, so we can avoid doing the work twice. 311 */ 312 if (dmat->lowaddr < ptoa(Maxmem) && map->pagesneeded == 0) { 313 vm_offset_t vendaddr; 314 315 /* 316 * Count the number of bounce pages 317 * needed in order to complete this transfer 318 */ 319 vaddr = trunc_page(buf); 320 vendaddr = (vm_offset_t)buf + buflen; 321 322 while (vaddr < vendaddr) { 323 paddr = pmap_kextract(vaddr); 324 if (run_filter(dmat, paddr) != 0) { 325 326 map->pagesneeded++; 327 } 328 vaddr += PAGE_SIZE; 329 } 330 } 331 332 if (map == NULL) 333 map = &nobounce_dmamap; 334 335 /* Reserve Necessary Bounce Pages */ 336 if (map->pagesneeded != 0) { 337 int s; 338 339 s = splhigh(); 340 if (reserve_bounce_pages(dmat, map) != 0) { 341 342 /* Queue us for resources */ 343 map->dmat = dmat; 344 map->buf = buf; 345 map->buflen = buflen; 346 map->callback = callback; 347 map->callback_arg = callback_arg; 348 349 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 350 splx(s); 351 352 return (EINPROGRESS); 353 } 354 splx(s); 355 } 356 357 vaddr = (vm_offset_t)buf; 358 sg = &dm_segments[0]; 359 seg = 1; 360 sg->ds_len = 0; 361 362 do { 363 bus_size_t size; 364 vm_offset_t nextpaddr; 365 366 paddr = pmap_kextract(vaddr); 367 size = PAGE_SIZE - (paddr & PAGE_MASK); 368 if (size > buflen) 369 size = buflen; 370 371 if (map->pagesneeded != 0 372 && run_filter(dmat, paddr)) { 373 paddr = add_bounce_page(dmat, map, vaddr, size); 374 } 375 376 if (sg->ds_len == 0) { 377 sg->ds_addr = paddr; 378 sg->ds_len = size; 379 } else if (paddr == nextpaddr) { 380 sg->ds_len += size; 381 } else { 382 /* Go to the next segment */ 383 sg++; 384 seg++; 385 if (seg > dmat->nsegments) 386 break; 387 sg->ds_addr = paddr; 388 sg->ds_len = size; 389 } 390 vaddr += size; 391 nextpaddr = paddr + size; 392 buflen -= size; 393 } while (buflen > 0); 394 395 if (buflen != 0) { 396 printf("bus_dmamap_load: Too many segs!\n"); 397 error = EFBIG; 398 } 399 400 (*callback)(callback_arg, dm_segments, seg, error); 401 402 return (0); 403} 404 405/* 406 * Release the mapping held by map. 407 */ 408void 409_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 410{ 411 struct bounce_page *bpage; 412 413 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 414 STAILQ_REMOVE_HEAD(&map->bpages, links); 415 free_bounce_page(dmat, bpage); 416 } 417} 418 419void 420_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 421{ 422 struct bounce_page *bpage; 423 424 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 425 426 /* 427 * Handle data bouncing. We might also 428 * want to add support for invalidating 429 * the caches on broken hardware 430 */ 431 switch (op) { 432 case BUS_DMASYNC_PREWRITE: 433 while (bpage != NULL) { 434 bcopy((void *)bpage->datavaddr, 435 (void *)bpage->vaddr, 436 bpage->datacount); 437 bpage = STAILQ_NEXT(bpage, links); 438 } 439 break; 440 441 case BUS_DMASYNC_POSTREAD: 442 while (bpage != NULL) { 443 bcopy((void *)bpage->vaddr, 444 (void *)bpage->datavaddr, 445 bpage->datacount); 446 bpage = STAILQ_NEXT(bpage, links); 447 } 448 break; 449 case BUS_DMASYNC_PREREAD: 450 case BUS_DMASYNC_POSTWRITE: 451 /* No-ops */ 452 break; 453 } 454 } 455} 456 457static int 458alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 459{ 460 int count; 461 462 count = 0; 463 if (total_bpages == 0) { 464 STAILQ_INIT(&bounce_page_list); 465 STAILQ_INIT(&bounce_map_waitinglist); 466 STAILQ_INIT(&bounce_map_callbacklist); 467 } 468 469 while (numpages > 0) { 470 struct bounce_page *bpage; 471 int s; 472 473 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 474 M_NOWAIT); 475 476 if (bpage == NULL) 477 break; 478 bzero(bpage, sizeof(*bpage)); 479 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 480 M_NOWAIT, 0ul, 481 dmat->lowaddr, 482 PAGE_SIZE, 0x10000); 483 if (bpage->vaddr == NULL) { 484 free(bpage, M_DEVBUF); 485 break; 486 } 487 bpage->busaddr = pmap_kextract(bpage->vaddr); 488 s = splhigh(); 489 STAILQ_INSERT_TAIL(&bounce_page_list, bpage, links); 490 total_bpages++; 491 free_bpages++; 492 splx(s); 493 count++; 494 numpages--; 495 } 496 return (count); 497} 498 499static int 500reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map) 501{ 502 int pages; 503 504 pages = MIN(free_bpages, map->pagesneeded - map->pagesreserved); 505 free_bpages -= pages; 506 reserved_bpages += pages; 507 map->pagesreserved += pages; 508 pages = map->pagesneeded - map->pagesreserved; 509 510 return (pages); 511} 512 513static vm_offset_t 514add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 515 bus_size_t size) 516{ 517 int s; 518 struct bounce_page *bpage; 519 520 if (map->pagesneeded == 0) 521 panic("add_bounce_page: map doesn't need any pages"); 522 map->pagesneeded--; 523 524 if (map->pagesreserved == 0) 525 panic("add_bounce_page: map doesn't need any pages"); 526 map->pagesreserved--; 527 528 s = splhigh(); 529 bpage = STAILQ_FIRST(&bounce_page_list); 530 if (bpage == NULL) 531 panic("add_bounce_page: free page list is empty"); 532 533 STAILQ_REMOVE_HEAD(&bounce_page_list, links); 534 reserved_bpages--; 535 active_bpages++; 536 splx(s); 537 538 bpage->datavaddr = vaddr; 539 bpage->datacount = size; 540 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 541 return (bpage->busaddr); 542} 543 544static void 545free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 546{ 547 int s; 548 struct bus_dmamap *map; 549 550 bpage->datavaddr = 0; 551 bpage->datacount = 0; 552 553 s = splhigh(); 554 STAILQ_INSERT_HEAD(&bounce_page_list, bpage, links); 555 free_bpages++; 556 active_bpages--; 557 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 558 if (reserve_bounce_pages(map->dmat, map) == 0) { 559 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 560 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 561 map, links); 562 busdma_swi_pending = 1; 563 setsoftvm(); 564 } 565 } 566 splx(s); 567} 568 569void 570busdma_swi() 571{ 572 int s; 573 struct bus_dmamap *map; 574 575 s = splhigh(); 576 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 577 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 578 splx(s); 579 bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 580 map->callback, map->callback_arg, /*flags*/0); 581 s = splhigh(); 582 } 583 splx(s); 584} 585