1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2000,2002 Silicon Graphics, Inc. All rights reserved. 7 * 8 * Routines for PCI DMA mapping. See Documentation/DMA-mapping.txt for 9 * a description of how these routines should be used. 10 */ 11 12#include <linux/types.h> 13#include <linux/mm.h> 14#include <linux/string.h> 15#include <linux/pci.h> 16#include <linux/slab.h> 17#include <linux/devfs_fs_kernel.h> 18#include <linux/module.h> 19 20#include <asm/delay.h> 21#include <asm/io.h> 22#include <asm/sn/sgi.h> 23#include <asm/sn/io.h> 24#include <asm/sn/invent.h> 25#include <asm/sn/hcl.h> 26#include <asm/sn/pci/pcibr.h> 27#include <asm/sn/pci/pcibr_private.h> 28#include <asm/sn/driver.h> 29#include <asm/sn/types.h> 30#include <asm/sn/alenlist.h> 31#include <asm/sn/pci/pci_bus_cvlink.h> 32#include <asm/sn/nag.h> 33 34/* DMA, PIO, and memory allocation flags */ 35#ifdef CONFIG_IA64_SGI_SN1 36#define DMA_DATA_FLAGS ( PCIIO_BYTE_STREAM | PCIIO_DMA_DATA ) 37#define DMA_CONTROL_FLAGS ( PCIIO_BYTE_STREAM | PCIBR_BARRIER | \ 38 PCIIO_DMA_CMD ) 39#elif defined(CONFIG_IA64_SGI_SN2) 40#define DMA_DATA_FLAGS ( PCIIO_DMA_DATA ) 41#define DMA_CONTROL_FLAGS ( PCIBR_BARRIER | PCIIO_DMA_CMD ) 42#else 43#error need to define DMA mapping flags for this platform 44#endif 45 46/* 47 * For ATE allocations 48 */ 49pciio_dmamap_t get_free_pciio_dmamap(devfs_handle_t); 50void free_pciio_dmamap(pcibr_dmamap_t); 51static struct sn_dma_maps_s *find_sn_dma_map(dma_addr_t, unsigned char); 52 53/* 54 * Toplogy stuff 55 */ 56extern devfs_handle_t busnum_to_pcibr_vhdl[]; 57extern nasid_t busnum_to_nid[]; 58extern void * busnum_to_atedmamaps[]; 59 60/** 61 * get_free_pciio_dmamap - find and allocate an ATE 62 * @pci_bus: PCI bus to get an entry for 63 * 64 * Finds and allocates an ATE on the PCI bus specified 65 * by @pci_bus. 66 */ 67pciio_dmamap_t 68get_free_pciio_dmamap(devfs_handle_t pci_bus) 69{ 70 int i; 71 struct sn_dma_maps_s *sn_dma_map = NULL; 72 73 /* 74 * Darn, we need to get the maps allocated for this bus. 75 */ 76 for (i = 0; i < MAX_PCI_XWIDGET; i++) { 77 if (busnum_to_pcibr_vhdl[i] == pci_bus) { 78 sn_dma_map = busnum_to_atedmamaps[i]; 79 } 80 } 81 82 /* 83 * Now get a free dmamap entry from this list. 84 */ 85 for (i = 0; i < MAX_ATE_MAPS; i++, sn_dma_map++) { 86 if (!sn_dma_map->dma_addr) { 87 sn_dma_map->dma_addr = -1; 88 return( (pciio_dmamap_t) sn_dma_map ); 89 } 90 } 91 92 return NULL; 93} 94 95/** 96 * free_pciio_dmamap - free an ATE 97 * @dma_map: ATE to free 98 * 99 * Frees the ATE specified by @dma_map. 100 */ 101void 102free_pciio_dmamap(pcibr_dmamap_t dma_map) 103{ 104 struct sn_dma_maps_s *sn_dma_map; 105 106 sn_dma_map = (struct sn_dma_maps_s *) dma_map; 107 sn_dma_map->dma_addr = 0; 108} 109 110/** 111 * find_sn_dma_map - find an ATE associated with @dma_addr and @busnum 112 * @dma_addr: DMA address to look for 113 * @busnum: PCI bus to look on 114 * 115 * Finds the ATE associated with @dma_addr and @busnum. 116 */ 117static struct sn_dma_maps_s * 118find_sn_dma_map(dma_addr_t dma_addr, unsigned char busnum) 119{ 120 121 struct sn_dma_maps_s *sn_dma_map = NULL; 122 int i; 123 124 sn_dma_map = busnum_to_atedmamaps[busnum]; 125 126 for (i = 0; i < MAX_ATE_MAPS; i++, sn_dma_map++) { 127 if (sn_dma_map->dma_addr == dma_addr) { 128 return sn_dma_map; 129 } 130 } 131 132 printk(KERN_WARNING "find_sn_dma_map: Unable find the corresponding " 133 "dma map\n"); 134 135 return NULL; 136} 137 138/** 139 * sn_dma_sync - try to flush DMA buffers into the coherence domain 140 * @hwdev: device to flush 141 * 142 * This routine flushes all DMA buffers for the device into the II of 143 * the destination hub. 144 * 145 * NOTE!: this does not mean that the data is in the "coherence domain", 146 * but it is very close. In other words, this routine *does not work* 147 * as advertised due to hardware bugs. That said, it should be good enough for 148 * most situations. 149 */ 150void 151sn_dma_sync(struct pci_dev *hwdev) 152{ 153 154 struct sn_device_sysdata *device_sysdata; 155 volatile unsigned long dummy; 156 157 /* 158 * It is expected that on an IA64 platform, a DMA sync ensures that 159 * all the DMA from a particular device is complete and coherent. We 160 * try to do this by 161 * 1. flushing the write wuffers from Bridge 162 * 2. flushing the Xbow port. 163 * Unfortunately, this only gets the DMA transactions 'very close' to 164 * the coherence domain, but not quite in it. 165 */ 166 device_sysdata = (struct sn_device_sysdata *)hwdev->sysdata; 167 dummy = (volatile unsigned long ) *device_sysdata->dma_buf_sync; 168 169 /* 170 * For the Xbow port flush, we maybe denied the request because 171 * someone else may be flushing the port .. try again. 172 */ 173 while((volatile unsigned long ) *device_sysdata->xbow_buf_sync) { 174 udelay(2); 175 } 176} 177 178/** 179 * sn_pci_alloc_consistent - allocate memory for coherent DMA 180 * @hwdev: device to allocate for 181 * @size: size of the region 182 * @dma_handle: DMA (bus) address 183 * 184 * pci_alloc_consistent() returns a pointer to a memory region suitable for 185 * coherent DMA traffic to/from a PCI device. On SN platforms, this means 186 * that @dma_handle will have the PCIBR_BARRIER and PCIIO_DMA_CMD flags 187 * set. 188 * 189 * This interface is usually used for "command" streams (e.g. the command 190 * queue for a SCSI controller). See Documentation/DMA-mapping.txt for 191 * more information. Note that this routine should always put a 32 bit 192 * DMA address into @dma_handle. This is because most other platforms 193 * that are capable of 64 bit PCI DMA transactions can't do 64 bit _coherent_ 194 * DMAs, and unfortunately this interface has to cater to the LCD. Oh well. 195 * 196 * Also known as platform_pci_alloc_consistent() by the IA64 machvec code. 197 */ 198void * 199sn_pci_alloc_consistent(struct pci_dev *hwdev, size_t size, dma_addr_t *dma_handle) 200{ 201 void *cpuaddr; 202 devfs_handle_t vhdl; 203 struct sn_device_sysdata *device_sysdata; 204 unsigned long phys_addr; 205 pciio_dmamap_t dma_map = 0; 206 struct sn_dma_maps_s *sn_dma_map; 207 208 *dma_handle = 0; 209 210 /* We can't easily support < 32 bit devices */ 211 if (IS_PCI32L(hwdev)) 212 return NULL; 213 214 /* 215 * Get hwgraph vertex for the device 216 */ 217 device_sysdata = (struct sn_device_sysdata *) hwdev->sysdata; 218 vhdl = device_sysdata->vhdl; 219 220 if(!(cpuaddr = (void *)__get_free_pages(GFP_ATOMIC, get_order(size)))) 221 return NULL; 222 223 memset(cpuaddr, 0, size); /* have to zero it out */ 224 225 /* physical addr. of the memory we just got */ 226 phys_addr = __pa(cpuaddr); 227 228 *dma_handle = pciio_dmatrans_addr(vhdl, NULL, phys_addr, size, 229 DMA_CONTROL_FLAGS); 230 /* 231 * It is a 32 bit card and we cannot do direct mapping, 232 * so we use an ATE. 233 */ 234 if (!(*dma_handle)) { 235 dma_map = pciio_dmamap_alloc(vhdl, NULL, size, 236 DMA_CONTROL_FLAGS | PCIIO_FIXED); 237 if (!dma_map) { 238 printk(KERN_ERR "sn_pci_alloc_consistent: Unable to " 239 "allocate anymore 32 bit page map entries.\n"); 240 BUG(); 241 } 242 *dma_handle = (dma_addr_t) pciio_dmamap_addr(dma_map,phys_addr, 243 size); 244 sn_dma_map = (struct sn_dma_maps_s *)dma_map; 245 sn_dma_map->dma_addr = *dma_handle; 246 printk(KERN_INFO "%s: PMU mapping: %p\n", __FUNCTION__, 247 (void *)*dma_handle); 248 } 249 else 250 printk(KERN_INFO "%s: direct mapping: %p\n", __FUNCTION__, 251 (void *)*dma_handle); 252 253 254 return cpuaddr; 255} 256 257/** 258 * sn_pci_free_consistent - free memory associated with coherent DMAable region 259 * @hwdev: device to free for 260 * @size: size to free 261 * @vaddr: kernel virtual address to free 262 * @dma_handle: DMA address associated with this region 263 * 264 * Frees the memory allocated by pci_alloc_consistent(). Also known 265 * as platform_pci_free_consistent() by the IA64 machvec code. 266 */ 267void 268sn_pci_free_consistent(struct pci_dev *hwdev, size_t size, void *vaddr, dma_addr_t dma_handle) 269{ 270 struct sn_dma_maps_s *sn_dma_map = NULL; 271 272 /* 273 * Get the sn_dma_map entry. 274 */ 275 if (IS_PCI32_MAPPED(dma_handle)) 276 sn_dma_map = find_sn_dma_map(dma_handle, hwdev->bus->number); 277 278 /* 279 * and free it if necessary... 280 */ 281 if (sn_dma_map) { 282 pciio_dmamap_done((pciio_dmamap_t)sn_dma_map); 283 pciio_dmamap_free((pciio_dmamap_t)sn_dma_map); 284 sn_dma_map->dma_addr = (dma_addr_t)NULL; 285 } 286 free_pages((unsigned long) vaddr, get_order(size)); 287} 288 289/** 290 * sn_pci_map_sg - map a scatter-gather list for DMA 291 * @hwdev: device to map for 292 * @sg: scatterlist to map 293 * @nents: number of entries 294 * @direction: direction of the DMA transaction 295 * 296 * Maps each entry of @sg for DMA. Also known as platform_pci_map_sg by the 297 * IA64 machvec code. 298 */ 299int 300sn_pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int direction) 301{ 302 303 int i; 304 devfs_handle_t vhdl; 305 dma_addr_t dma_addr; 306 unsigned long phys_addr; 307 struct sn_device_sysdata *device_sysdata; 308 pciio_dmamap_t dma_map; 309 310 /* can't go anywhere w/o a direction in life */ 311 if (direction == PCI_DMA_NONE) 312 BUG(); 313 314 /* 315 * Get the hwgraph vertex for the device 316 */ 317 device_sysdata = (struct sn_device_sysdata *) hwdev->sysdata; 318 vhdl = device_sysdata->vhdl; 319 320 /* 321 * Setup a DMA address for each entry in the 322 * scatterlist. 323 */ 324 for (i = 0; i < nents; i++, sg++) { 325 /* this catches incorrectly written drivers that 326 attempt to map scatterlists that they have 327 previously mapped. we print a warning and 328 continue, but the driver should be fixed */ 329 switch (((u64)sg->address) >> 60) { 330 case 0xa: 331 case 0xb: 332#ifdef DEBUG 333/* This needs to be cleaned up at some point. */ 334 NAG("A PCI driver (for device at%8s) has attempted to " 335 "map a scatterlist that was previously mapped at " 336 "%p - this is currently being worked around.\n", 337 hwdev->slot_name, (void *)sg->address); 338#endif 339 phys_addr = (u64)sg->address & TO_PHYS_MASK; 340 break; 341 default: /* not previously mapped, get the phys. addr */ 342 phys_addr = __pa(sg->address); 343 break; 344 } 345 sg->page = NULL; 346 dma_addr = 0; 347 348 /* 349 * Handle the most common case: 64 bit cards. This 350 * call should always succeed. 351 */ 352 if (IS_PCIA64(hwdev)) { 353 dma_addr = pciio_dmatrans_addr(vhdl, NULL, phys_addr, 354 sg->length, 355 DMA_DATA_FLAGS | PCIIO_DMA_A64 ); 356 sg->address = (char *)dma_addr; 357 continue; 358 } 359 360 /* 361 * Handle 32-63 bit cards via direct mapping 362 */ 363 if (IS_PCI32G(hwdev)) { 364 dma_addr = pciio_dmatrans_addr(vhdl, NULL, phys_addr, 365 sg->length, 366 DMA_DATA_FLAGS); 367 /* 368 * See if we got a direct map entry 369 */ 370 if (dma_addr) { 371 sg->address = (char *)dma_addr; 372 continue; 373 } 374 375 } 376 377 /* 378 * It is a 32 bit card and we cannot do direct mapping, 379 * so we use an ATE. 380 */ 381 dma_map = 0; 382 dma_map = pciio_dmamap_alloc(vhdl, NULL, sg->length, 383 DMA_DATA_FLAGS); 384 if (!dma_map) { 385 printk(KERN_ERR "sn_pci_map_sg: Unable to allocate " 386 "anymore 32 bit page map entries.\n"); 387 BUG(); 388 } 389 dma_addr = pciio_dmamap_addr(dma_map, phys_addr, sg->length); 390 sg->address = (char *)dma_addr; 391 sg->page = (char *)dma_map; 392 393 } 394 395 return nents; 396 397} 398 399/** 400 * sn_pci_unmap_sg - unmap a scatter-gather list 401 * @hwdev: device to unmap 402 * @sg: scatterlist to unmap 403 * @nents: number of scatterlist entries 404 * @direction: DMA direction 405 * 406 * Unmap a set of streaming mode DMA translations. Again, cpu read rules 407 * concerning calls here are the same as for pci_unmap_single() below. Also 408 * known as sn_pci_unmap_sg() by the IA64 machvec code. 409 */ 410void 411sn_pci_unmap_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int direction) 412{ 413 int i; 414 struct sn_dma_maps_s *sn_dma_map; 415 416 /* can't go anywhere w/o a direction in life */ 417 if (direction == PCI_DMA_NONE) 418 BUG(); 419 420 for (i = 0; i < nents; i++, sg++) 421 if (sg->page) { 422 /* 423 * We maintain the DMA Map pointer in sg->page if 424 * it is ever allocated. 425 */ 426 sg->address = 0; 427 sn_dma_map = (struct sn_dma_maps_s *)sg->page; 428 pciio_dmamap_done((pciio_dmamap_t)sn_dma_map); 429 pciio_dmamap_free((pciio_dmamap_t)sn_dma_map); 430 sn_dma_map->dma_addr = 0; 431 sg->page = 0; 432 } 433 434} 435 436/** 437 * sn_pci_map_single - map a single region for DMA 438 * @hwdev: device to map for 439 * @ptr: kernel virtual address of the region to map 440 * @size: size of the region 441 * @direction: DMA direction 442 * 443 * Map the region pointed to by @ptr for DMA and return the 444 * DMA address. Also known as platform_pci_map_single() by 445 * the IA64 machvec code. 446 * 447 * We map this to the one step pciio_dmamap_trans interface rather than 448 * the two step pciio_dmamap_alloc/pciio_dmamap_addr because we have 449 * no way of saving the dmamap handle from the alloc to later free 450 * (which is pretty much unacceptable). 451 * 452 * TODO: simplify our interface; 453 * get rid of dev_desc and vhdl (seems redundant given a pci_dev); 454 * figure out how to save dmamap handle so can use two step. 455 */ 456dma_addr_t 457sn_pci_map_single(struct pci_dev *hwdev, void *ptr, size_t size, int direction) 458{ 459 devfs_handle_t vhdl; 460 dma_addr_t dma_addr; 461 unsigned long phys_addr; 462 struct sn_device_sysdata *device_sysdata; 463 pciio_dmamap_t dma_map = NULL; 464 struct sn_dma_maps_s *sn_dma_map; 465 466 if (direction == PCI_DMA_NONE) 467 BUG(); 468 469 /* SN cannot support DMA addresses smaller than 32 bits. */ 470 if (IS_PCI32L(hwdev)) return 0; 471 472 /* 473 * find vertex for the device 474 */ 475 device_sysdata = (struct sn_device_sysdata *)hwdev->sysdata; 476 vhdl = device_sysdata->vhdl; 477 478 /* 479 * Call our dmamap interface 480 */ 481 dma_addr = 0; 482 phys_addr = __pa(ptr); 483 484 if (IS_PCIA64(hwdev)) { 485 /* 486 * This device supports 64 bit DMA addresses. 487 */ 488 dma_addr = pciio_dmatrans_addr(vhdl, NULL, phys_addr, size, 489 DMA_DATA_FLAGS | PCIIO_DMA_A64); 490 return dma_addr; 491 } 492 493 /* 494 * Devices that supports 32 bit to 63 bit DMA addresses get 495 * 32 bit DMA addresses. 496 * 497 * First try to get a 32 bit direct map register. 498 */ 499 if (IS_PCI32G(hwdev)) { 500 dma_addr = pciio_dmatrans_addr(vhdl, NULL, phys_addr, size, 501 DMA_DATA_FLAGS); 502 if (dma_addr) 503 return dma_addr; 504 } 505 506 /* 507 * It's a 32 bit card and we cannot do direct mapping so 508 * let's use the PMU instead. 509 */ 510 dma_map = NULL; 511 dma_map = pciio_dmamap_alloc(vhdl, NULL, size, DMA_DATA_FLAGS); 512 513 if (!dma_map) { 514 printk(KERN_ERR "pci_map_single: Unable to allocate anymore " 515 "32 bits page map entries.\n"); 516 BUG(); 517 } 518 519 dma_addr = (dma_addr_t) pciio_dmamap_addr(dma_map, phys_addr, size); 520 sn_dma_map = (struct sn_dma_maps_s *)dma_map; 521 sn_dma_map->dma_addr = dma_addr; 522 523 return ((dma_addr_t)dma_addr); 524} 525 526/** 527 * sn_pci_unmap_single - unmap a region used for DMA 528 * @hwdev: device to unmap 529 * @dma_addr: DMA address to unmap 530 * @size: size of region 531 * @direction: DMA direction 532 * 533 * Unmaps the region pointed to by @dma_addr. Also known as 534 * platform_pci_unmap_single() by the IA64 machvec code. 535 */ 536void 537sn_pci_unmap_single(struct pci_dev *hwdev, dma_addr_t dma_addr, size_t size, int direction) 538{ 539 struct sn_dma_maps_s *sn_dma_map = NULL; 540 541 if (direction == PCI_DMA_NONE) 542 BUG(); 543 544 /* 545 * Get the sn_dma_map entry. 546 */ 547 if (IS_PCI32_MAPPED(dma_addr)) 548 sn_dma_map = find_sn_dma_map(dma_addr, hwdev->bus->number); 549 550 /* 551 * and free it if necessary... 552 */ 553 if (sn_dma_map) { 554 pciio_dmamap_done((pciio_dmamap_t)sn_dma_map); 555 pciio_dmamap_free((pciio_dmamap_t)sn_dma_map); 556 sn_dma_map->dma_addr = (dma_addr_t)NULL; 557 } 558} 559 560/** 561 * sn_pci_dma_sync_single - make sure all DMAs have completed 562 * @hwdev: device to sync 563 * @dma_handle: DMA address to sync 564 * @size: size of region 565 * @direction: DMA direction 566 * 567 * This routine is supposed to sync the DMA region specified 568 * by @dma_handle into the 'coherence domain'. See sn_dma_sync() 569 * above for more information. Also known as 570 * platform_pci_dma_sync_single() by the IA64 machvec code. 571 */ 572void 573sn_pci_dma_sync_single(struct pci_dev *hwdev, dma_addr_t dma_handle, size_t size, int direction) 574{ 575 if (direction == PCI_DMA_NONE) 576 BUG(); 577 578 sn_dma_sync(hwdev); 579} 580 581/** 582 * sn_pci_dma_sync_sg - make sure all DMAs have completed 583 * @hwdev: device to sync 584 * @sg: scatterlist to sync 585 * @nents: number of entries in the scatterlist 586 * @direction: DMA direction 587 * 588 * This routine is supposed to sync the DMA regions specified 589 * by @sg into the 'coherence domain'. See sn_dma_sync() 590 * above for more information. Also known as 591 * platform_pci_dma_sync_sg() by the IA64 machvec code. 592 */ 593void 594sn_pci_dma_sync_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int direction) 595{ 596 if (direction == PCI_DMA_NONE) 597 BUG(); 598 599 sn_dma_sync(hwdev); 600} 601 602/** 603 * sn_dma_address - get the DMA address for the first entry of a scatterlist 604 * @sg: sg to look at 605 * 606 * Gets the DMA address for the scatterlist @sg. Also known as 607 * platform_dma_address() by the IA64 machvec code. 608 */ 609unsigned long 610sn_dma_address(struct scatterlist *sg) 611{ 612 return ((unsigned long)sg->address); 613} 614 615/** 616 * sn_dma_supported - test a DMA mask 617 * @hwdev: device to test 618 * @mask: DMA mask to test 619 * 620 * Return whether the given PCI device DMA address mask can be supported 621 * properly. For example, if your device can only drive the low 24-bits 622 * during PCI bus mastering, then you would pass 0x00ffffff as the mask to 623 * this function. Of course, SN only supports devices that have 32 or more 624 * address bits. 625 */ 626int 627sn_pci_dma_supported(struct pci_dev *hwdev, u64 mask) 628{ 629 if (mask < 0xffffffff) 630 return 0; 631 return 1; 632} 633 634EXPORT_SYMBOL(sn_pci_unmap_single); 635EXPORT_SYMBOL(sn_pci_map_single); 636EXPORT_SYMBOL(sn_pci_dma_sync_single); 637EXPORT_SYMBOL(sn_pci_map_sg); 638EXPORT_SYMBOL(sn_pci_unmap_sg); 639EXPORT_SYMBOL(sn_pci_alloc_consistent); 640EXPORT_SYMBOL(sn_pci_free_consistent); 641EXPORT_SYMBOL(sn_dma_address); 642EXPORT_SYMBOL(sn_pci_dma_supported); 643 644