1/* 2 * IBM PPC4xx DMA engine scatter/gather library 3 * 4 * Copyright 2002-2003 MontaVista Software Inc. 5 * 6 * Cleaned up and converted to new DCR access 7 * Matt Porter <mporter@kernel.crashing.org> 8 * 9 * Original code by Armin Kuster <akuster@mvista.com> 10 * and Pete Popov <ppopov@mvista.com> 11 * 12 * This program is free software; you can redistribute it and/or modify it 13 * under the terms of the GNU General Public License as published by the 14 * Free Software Foundation; either version 2 of the License, or (at your 15 * option) any later version. 16 * 17 * You should have received a copy of the GNU General Public License along 18 * with this program; if not, write to the Free Software Foundation, Inc., 19 * 675 Mass Ave, Cambridge, MA 02139, USA. 20 */ 21 22#include <linux/kernel.h> 23#include <linux/mm.h> 24#include <linux/init.h> 25#include <linux/module.h> 26#include <linux/dma-mapping.h> 27 28#include <asm/system.h> 29#include <asm/io.h> 30#include <asm/ppc4xx_dma.h> 31 32void 33ppc4xx_set_sg_addr(int dmanr, phys_addr_t sg_addr) 34{ 35 if (dmanr >= MAX_PPC4xx_DMA_CHANNELS) { 36 printk("ppc4xx_set_sg_addr: bad channel: %d\n", dmanr); 37 return; 38 } 39 40#ifdef PPC4xx_DMA_64BIT 41 mtdcr(DCRN_ASGH0 + (dmanr * 0x8), (u32)(sg_addr >> 32)); 42#endif 43 mtdcr(DCRN_ASG0 + (dmanr * 0x8), (u32)sg_addr); 44} 45 46/* 47 * Add a new sgl descriptor to the end of a scatter/gather list 48 * which was created by alloc_dma_handle(). 49 * 50 * For a memory to memory transfer, both dma addresses must be 51 * valid. For a peripheral to memory transfer, one of the addresses 52 * must be set to NULL, depending on the direction of the transfer: 53 * memory to peripheral: set dst_addr to NULL, 54 * peripheral to memory: set src_addr to NULL. 55 */ 56int 57ppc4xx_add_dma_sgl(sgl_handle_t handle, phys_addr_t src_addr, phys_addr_t dst_addr, 58 unsigned int count) 59{ 60 sgl_list_info_t *psgl = (sgl_list_info_t *) handle; 61 ppc_dma_ch_t *p_dma_ch; 62 63 if (!handle) { 64 printk("ppc4xx_add_dma_sgl: null handle\n"); 65 return DMA_STATUS_BAD_HANDLE; 66 } 67 68 if (psgl->dmanr >= MAX_PPC4xx_DMA_CHANNELS) { 69 printk("ppc4xx_add_dma_sgl: bad channel: %d\n", psgl->dmanr); 70 return DMA_STATUS_BAD_CHANNEL; 71 } 72 73 p_dma_ch = &dma_channels[psgl->dmanr]; 74 75#ifdef DEBUG_4xxDMA 76 { 77 int error = 0; 78 unsigned int aligned = 79 (unsigned) src_addr | (unsigned) dst_addr | count; 80 switch (p_dma_ch->pwidth) { 81 case PW_8: 82 break; 83 case PW_16: 84 if (aligned & 0x1) 85 error = 1; 86 break; 87 case PW_32: 88 if (aligned & 0x3) 89 error = 1; 90 break; 91 case PW_64: 92 if (aligned & 0x7) 93 error = 1; 94 break; 95 default: 96 printk("ppc4xx_add_dma_sgl: invalid bus width: 0x%x\n", 97 p_dma_ch->pwidth); 98 return DMA_STATUS_GENERAL_ERROR; 99 } 100 if (error) 101 printk 102 ("Alignment warning: ppc4xx_add_dma_sgl src 0x%x dst 0x%x count 0x%x bus width var %d\n", 103 src_addr, dst_addr, count, p_dma_ch->pwidth); 104 105 } 106#endif 107 108 if ((unsigned) (psgl->ptail + 1) >= ((unsigned) psgl + SGL_LIST_SIZE)) { 109 printk("sgl handle out of memory \n"); 110 return DMA_STATUS_OUT_OF_MEMORY; 111 } 112 113 if (!psgl->ptail) { 114 psgl->phead = (ppc_sgl_t *) 115 ((unsigned) psgl + sizeof (sgl_list_info_t)); 116 psgl->phead_dma = psgl->dma_addr + sizeof(sgl_list_info_t); 117 psgl->ptail = psgl->phead; 118 psgl->ptail_dma = psgl->phead_dma; 119 } else { 120 if(p_dma_ch->int_on_final_sg) { 121 /* mask out all dma interrupts, except error, on tail 122 before adding new tail. */ 123 psgl->ptail->control_count &= 124 ~(SG_TCI_ENABLE | SG_ETI_ENABLE); 125 } 126 psgl->ptail->next = psgl->ptail_dma + sizeof(ppc_sgl_t); 127 psgl->ptail++; 128 psgl->ptail_dma += sizeof(ppc_sgl_t); 129 } 130 131 psgl->ptail->control = psgl->control; 132 psgl->ptail->src_addr = src_addr; 133 psgl->ptail->dst_addr = dst_addr; 134 psgl->ptail->control_count = (count >> p_dma_ch->shift) | 135 psgl->sgl_control; 136 psgl->ptail->next = (uint32_t) NULL; 137 138 return DMA_STATUS_GOOD; 139} 140 141/* 142 * Enable (start) the DMA described by the sgl handle. 143 */ 144void 145ppc4xx_enable_dma_sgl(sgl_handle_t handle) 146{ 147 sgl_list_info_t *psgl = (sgl_list_info_t *) handle; 148 ppc_dma_ch_t *p_dma_ch; 149 uint32_t sg_command; 150 151 if (!handle) { 152 printk("ppc4xx_enable_dma_sgl: null handle\n"); 153 return; 154 } else if (psgl->dmanr > (MAX_PPC4xx_DMA_CHANNELS - 1)) { 155 printk("ppc4xx_enable_dma_sgl: bad channel in handle %d\n", 156 psgl->dmanr); 157 return; 158 } else if (!psgl->phead) { 159 printk("ppc4xx_enable_dma_sgl: sg list empty\n"); 160 return; 161 } 162 163 p_dma_ch = &dma_channels[psgl->dmanr]; 164 psgl->ptail->control_count &= ~SG_LINK; /* make this the last dscrptr */ 165 sg_command = mfdcr(DCRN_ASGC); 166 167 ppc4xx_set_sg_addr(psgl->dmanr, psgl->phead_dma); 168 169 sg_command |= SSG_ENABLE(psgl->dmanr); 170 171 mtdcr(DCRN_ASGC, sg_command); /* start transfer */ 172} 173 174/* 175 * Halt an active scatter/gather DMA operation. 176 */ 177void 178ppc4xx_disable_dma_sgl(sgl_handle_t handle) 179{ 180 sgl_list_info_t *psgl = (sgl_list_info_t *) handle; 181 uint32_t sg_command; 182 183 if (!handle) { 184 printk("ppc4xx_enable_dma_sgl: null handle\n"); 185 return; 186 } else if (psgl->dmanr > (MAX_PPC4xx_DMA_CHANNELS - 1)) { 187 printk("ppc4xx_enable_dma_sgl: bad channel in handle %d\n", 188 psgl->dmanr); 189 return; 190 } 191 192 sg_command = mfdcr(DCRN_ASGC); 193 sg_command &= ~SSG_ENABLE(psgl->dmanr); 194 mtdcr(DCRN_ASGC, sg_command); /* stop transfer */ 195} 196 197/* 198 * Returns number of bytes left to be transferred from the entire sgl list. 199 * *src_addr and *dst_addr get set to the source/destination address of 200 * the sgl descriptor where the DMA stopped. 201 * 202 * An sgl transfer must NOT be active when this function is called. 203 */ 204int 205ppc4xx_get_dma_sgl_residue(sgl_handle_t handle, phys_addr_t * src_addr, 206 phys_addr_t * dst_addr) 207{ 208 sgl_list_info_t *psgl = (sgl_list_info_t *) handle; 209 ppc_dma_ch_t *p_dma_ch; 210 ppc_sgl_t *pnext, *sgl_addr; 211 uint32_t count_left; 212 213 if (!handle) { 214 printk("ppc4xx_get_dma_sgl_residue: null handle\n"); 215 return DMA_STATUS_BAD_HANDLE; 216 } else if (psgl->dmanr > (MAX_PPC4xx_DMA_CHANNELS - 1)) { 217 printk("ppc4xx_get_dma_sgl_residue: bad channel in handle %d\n", 218 psgl->dmanr); 219 return DMA_STATUS_BAD_CHANNEL; 220 } 221 222 sgl_addr = (ppc_sgl_t *) __va(mfdcr(DCRN_ASG0 + (psgl->dmanr * 0x8))); 223 count_left = mfdcr(DCRN_DMACT0 + (psgl->dmanr * 0x8)) & SG_COUNT_MASK; 224 225 if (!sgl_addr) { 226 printk("ppc4xx_get_dma_sgl_residue: sgl addr register is null\n"); 227 goto error; 228 } 229 230 pnext = psgl->phead; 231 while (pnext && 232 ((unsigned) pnext < ((unsigned) psgl + SGL_LIST_SIZE) && 233 (pnext != sgl_addr)) 234 ) { 235 pnext++; 236 } 237 238 if (pnext == sgl_addr) { /* found the sgl descriptor */ 239 240 *src_addr = pnext->src_addr; 241 *dst_addr = pnext->dst_addr; 242 243 /* 244 * Now search the remaining descriptors and add their count. 245 * We already have the remaining count from this descriptor in 246 * count_left. 247 */ 248 pnext++; 249 250 while ((pnext != psgl->ptail) && 251 ((unsigned) pnext < ((unsigned) psgl + SGL_LIST_SIZE)) 252 ) { 253 count_left += pnext->control_count & SG_COUNT_MASK; 254 } 255 256 if (pnext != psgl->ptail) { /* should never happen */ 257 printk 258 ("ppc4xx_get_dma_sgl_residue error (1) psgl->ptail 0x%x handle 0x%x\n", 259 (unsigned int) psgl->ptail, (unsigned int) handle); 260 goto error; 261 } 262 263 /* success */ 264 p_dma_ch = &dma_channels[psgl->dmanr]; 265 return (count_left << p_dma_ch->shift); /* count in bytes */ 266 267 } else { 268 /* this shouldn't happen */ 269 printk 270 ("get_dma_sgl_residue, unable to match current address 0x%x, handle 0x%x\n", 271 (unsigned int) sgl_addr, (unsigned int) handle); 272 273 } 274 275 error: 276 *src_addr = (phys_addr_t) NULL; 277 *dst_addr = (phys_addr_t) NULL; 278 return 0; 279} 280 281/* 282 * Returns the address(es) of the buffer(s) contained in the head element of 283 * the scatter/gather list. The element is removed from the scatter/gather 284 * list and the next element becomes the head. 285 * 286 * This function should only be called when the DMA is not active. 287 */ 288int 289ppc4xx_delete_dma_sgl_element(sgl_handle_t handle, phys_addr_t * src_dma_addr, 290 phys_addr_t * dst_dma_addr) 291{ 292 sgl_list_info_t *psgl = (sgl_list_info_t *) handle; 293 294 if (!handle) { 295 printk("ppc4xx_delete_sgl_element: null handle\n"); 296 return DMA_STATUS_BAD_HANDLE; 297 } else if (psgl->dmanr > (MAX_PPC4xx_DMA_CHANNELS - 1)) { 298 printk("ppc4xx_delete_sgl_element: bad channel in handle %d\n", 299 psgl->dmanr); 300 return DMA_STATUS_BAD_CHANNEL; 301 } 302 303 if (!psgl->phead) { 304 printk("ppc4xx_delete_sgl_element: sgl list empty\n"); 305 *src_dma_addr = (phys_addr_t) NULL; 306 *dst_dma_addr = (phys_addr_t) NULL; 307 return DMA_STATUS_SGL_LIST_EMPTY; 308 } 309 310 *src_dma_addr = (phys_addr_t) psgl->phead->src_addr; 311 *dst_dma_addr = (phys_addr_t) psgl->phead->dst_addr; 312 313 if (psgl->phead == psgl->ptail) { 314 /* last descriptor on the list */ 315 psgl->phead = NULL; 316 psgl->ptail = NULL; 317 } else { 318 psgl->phead++; 319 psgl->phead_dma += sizeof(ppc_sgl_t); 320 } 321 322 return DMA_STATUS_GOOD; 323} 324 325 326/* 327 * Create a scatter/gather list handle. This is simply a structure which 328 * describes a scatter/gather list. 329 * 330 * A handle is returned in "handle" which the driver should save in order to 331 * be able to access this list later. A chunk of memory will be allocated 332 * to be used by the API for internal management purposes, including managing 333 * the sg list and allocating memory for the sgl descriptors. One page should 334 * be more than enough for that purpose. Perhaps it's a bit wasteful to use 335 * a whole page for a single sg list, but most likely there will be only one 336 * sg list per channel. 337 * 338 * Interrupt notes: 339 * Each sgl descriptor has a copy of the DMA control word which the DMA engine 340 * loads in the control register. The control word has a "global" interrupt 341 * enable bit for that channel. Interrupts are further qualified by a few bits 342 * in the sgl descriptor count register. In order to setup an sgl, we have to 343 * know ahead of time whether or not interrupts will be enabled at the completion 344 * of the transfers. Thus, enable_dma_interrupt()/disable_dma_interrupt() MUST 345 * be called before calling alloc_dma_handle(). If the interrupt mode will never 346 * change after powerup, then enable_dma_interrupt()/disable_dma_interrupt() 347 * do not have to be called -- interrupts will be enabled or disabled based 348 * on how the channel was configured after powerup by the hw_init_dma_channel() 349 * function. Each sgl descriptor will be setup to interrupt if an error occurs; 350 * however, only the last descriptor will be setup to interrupt. Thus, an 351 * interrupt will occur (if interrupts are enabled) only after the complete 352 * sgl transfer is done. 353 */ 354int 355ppc4xx_alloc_dma_handle(sgl_handle_t * phandle, unsigned int mode, unsigned int dmanr) 356{ 357 sgl_list_info_t *psgl=NULL; 358 dma_addr_t dma_addr; 359 ppc_dma_ch_t *p_dma_ch = &dma_channels[dmanr]; 360 uint32_t sg_command; 361 uint32_t ctc_settings; 362 void *ret; 363 364 if (dmanr >= MAX_PPC4xx_DMA_CHANNELS) { 365 printk("ppc4xx_alloc_dma_handle: invalid channel 0x%x\n", dmanr); 366 return DMA_STATUS_BAD_CHANNEL; 367 } 368 369 if (!phandle) { 370 printk("ppc4xx_alloc_dma_handle: null handle pointer\n"); 371 return DMA_STATUS_NULL_POINTER; 372 } 373 374 /* Get a page of memory, which is zeroed out by consistent_alloc() */ 375 ret = dma_alloc_coherent(NULL, DMA_PPC4xx_SIZE, &dma_addr, GFP_KERNEL); 376 if (ret != NULL) { 377 memset(ret, 0, DMA_PPC4xx_SIZE); 378 psgl = (sgl_list_info_t *) ret; 379 } 380 381 if (psgl == NULL) { 382 *phandle = (sgl_handle_t) NULL; 383 return DMA_STATUS_OUT_OF_MEMORY; 384 } 385 386 psgl->dma_addr = dma_addr; 387 psgl->dmanr = dmanr; 388 389 /* 390 * Modify and save the control word. These words will be 391 * written to each sgl descriptor. The DMA engine then 392 * loads this control word into the control register 393 * every time it reads a new descriptor. 394 */ 395 psgl->control = p_dma_ch->control; 396 /* Clear all mode bits */ 397 psgl->control &= ~(DMA_TM_MASK | DMA_TD); 398 /* Save control word and mode */ 399 psgl->control |= (mode | DMA_CE_ENABLE); 400 401 /* In MM mode, we must set ETD/TCE */ 402 if (mode == DMA_MODE_MM) 403 psgl->control |= DMA_ETD_OUTPUT | DMA_TCE_ENABLE; 404 405 if (p_dma_ch->int_enable) { 406 /* Enable channel interrupt */ 407 psgl->control |= DMA_CIE_ENABLE; 408 } else { 409 psgl->control &= ~DMA_CIE_ENABLE; 410 } 411 412 sg_command = mfdcr(DCRN_ASGC); 413 sg_command |= SSG_MASK_ENABLE(dmanr); 414 415 /* Enable SGL control access */ 416 mtdcr(DCRN_ASGC, sg_command); 417 psgl->sgl_control = SG_ERI_ENABLE | SG_LINK; 418 419 /* keep control count register settings */ 420 ctc_settings = mfdcr(DCRN_DMACT0 + (dmanr * 0x8)) 421 & (DMA_CTC_BSIZ_MSK | DMA_CTC_BTEN); /*burst mode settings*/ 422 psgl->sgl_control |= ctc_settings; 423 424 if (p_dma_ch->int_enable) { 425 if (p_dma_ch->tce_enable) 426 psgl->sgl_control |= SG_TCI_ENABLE; 427 else 428 psgl->sgl_control |= SG_ETI_ENABLE; 429 } 430 431 *phandle = (sgl_handle_t) psgl; 432 return DMA_STATUS_GOOD; 433} 434 435/* 436 * Destroy a scatter/gather list handle that was created by alloc_dma_handle(). 437 * The list must be empty (contain no elements). 438 */ 439void 440ppc4xx_free_dma_handle(sgl_handle_t handle) 441{ 442 sgl_list_info_t *psgl = (sgl_list_info_t *) handle; 443 444 if (!handle) { 445 printk("ppc4xx_free_dma_handle: got NULL\n"); 446 return; 447 } else if (psgl->phead) { 448 printk("ppc4xx_free_dma_handle: list not empty\n"); 449 return; 450 } else if (!psgl->dma_addr) { /* should never happen */ 451 printk("ppc4xx_free_dma_handle: no dma address\n"); 452 return; 453 } 454 455 dma_free_coherent(NULL, DMA_PPC4xx_SIZE, (void *) psgl, 0); 456} 457 458EXPORT_SYMBOL(ppc4xx_alloc_dma_handle); 459EXPORT_SYMBOL(ppc4xx_free_dma_handle); 460EXPORT_SYMBOL(ppc4xx_add_dma_sgl); 461EXPORT_SYMBOL(ppc4xx_delete_dma_sgl_element); 462EXPORT_SYMBOL(ppc4xx_enable_dma_sgl); 463EXPORT_SYMBOL(ppc4xx_disable_dma_sgl); 464EXPORT_SYMBOL(ppc4xx_get_dma_sgl_residue); 465