1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) ST-Ericsson SA 2007-2010 4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson 5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson 6 */ 7 8#include <linux/kernel.h> 9#include <linux/dmaengine.h> 10 11#include "ste_dma40.h" 12#include "ste_dma40_ll.h" 13 14static u8 d40_width_to_bits(enum dma_slave_buswidth width) 15{ 16 if (width == DMA_SLAVE_BUSWIDTH_1_BYTE) 17 return STEDMA40_ESIZE_8_BIT; 18 else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES) 19 return STEDMA40_ESIZE_16_BIT; 20 else if (width == DMA_SLAVE_BUSWIDTH_8_BYTES) 21 return STEDMA40_ESIZE_64_BIT; 22 else 23 return STEDMA40_ESIZE_32_BIT; 24} 25 26/* Sets up proper LCSP1 and LCSP3 register for a logical channel */ 27void d40_log_cfg(struct stedma40_chan_cfg *cfg, 28 u32 *lcsp1, u32 *lcsp3) 29{ 30 u32 l3 = 0; /* dst */ 31 u32 l1 = 0; /* src */ 32 33 /* src is mem? -> increase address pos */ 34 if (cfg->dir == DMA_MEM_TO_DEV || 35 cfg->dir == DMA_MEM_TO_MEM) 36 l1 |= BIT(D40_MEM_LCSP1_SCFG_INCR_POS); 37 38 /* dst is mem? -> increase address pos */ 39 if (cfg->dir == DMA_DEV_TO_MEM || 40 cfg->dir == DMA_MEM_TO_MEM) 41 l3 |= BIT(D40_MEM_LCSP3_DCFG_INCR_POS); 42 43 /* src is hw? -> master port 1 */ 44 if (cfg->dir == DMA_DEV_TO_MEM || 45 cfg->dir == DMA_DEV_TO_DEV) 46 l1 |= BIT(D40_MEM_LCSP1_SCFG_MST_POS); 47 48 /* dst is hw? -> master port 1 */ 49 if (cfg->dir == DMA_MEM_TO_DEV || 50 cfg->dir == DMA_DEV_TO_DEV) 51 l3 |= BIT(D40_MEM_LCSP3_DCFG_MST_POS); 52 53 l3 |= BIT(D40_MEM_LCSP3_DCFG_EIM_POS); 54 l3 |= cfg->dst_info.psize << D40_MEM_LCSP3_DCFG_PSIZE_POS; 55 l3 |= d40_width_to_bits(cfg->dst_info.data_width) 56 << D40_MEM_LCSP3_DCFG_ESIZE_POS; 57 58 l1 |= BIT(D40_MEM_LCSP1_SCFG_EIM_POS); 59 l1 |= cfg->src_info.psize << D40_MEM_LCSP1_SCFG_PSIZE_POS; 60 l1 |= d40_width_to_bits(cfg->src_info.data_width) 61 << D40_MEM_LCSP1_SCFG_ESIZE_POS; 62 63 *lcsp1 = l1; 64 *lcsp3 = l3; 65 66} 67 68void d40_phy_cfg(struct stedma40_chan_cfg *cfg, u32 *src_cfg, u32 *dst_cfg) 69{ 70 u32 src = 0; 71 u32 dst = 0; 72 73 if ((cfg->dir == DMA_DEV_TO_MEM) || 74 (cfg->dir == DMA_DEV_TO_DEV)) { 75 /* Set master port to 1 */ 76 src |= BIT(D40_SREG_CFG_MST_POS); 77 src |= D40_TYPE_TO_EVENT(cfg->dev_type); 78 79 if (cfg->src_info.flow_ctrl == STEDMA40_NO_FLOW_CTRL) 80 src |= BIT(D40_SREG_CFG_PHY_TM_POS); 81 else 82 src |= 3 << D40_SREG_CFG_PHY_TM_POS; 83 } 84 if ((cfg->dir == DMA_MEM_TO_DEV) || 85 (cfg->dir == DMA_DEV_TO_DEV)) { 86 /* Set master port to 1 */ 87 dst |= BIT(D40_SREG_CFG_MST_POS); 88 dst |= D40_TYPE_TO_EVENT(cfg->dev_type); 89 90 if (cfg->dst_info.flow_ctrl == STEDMA40_NO_FLOW_CTRL) 91 dst |= BIT(D40_SREG_CFG_PHY_TM_POS); 92 else 93 dst |= 3 << D40_SREG_CFG_PHY_TM_POS; 94 } 95 /* Interrupt on end of transfer for destination */ 96 dst |= BIT(D40_SREG_CFG_TIM_POS); 97 98 /* Generate interrupt on error */ 99 src |= BIT(D40_SREG_CFG_EIM_POS); 100 dst |= BIT(D40_SREG_CFG_EIM_POS); 101 102 /* PSIZE */ 103 if (cfg->src_info.psize != STEDMA40_PSIZE_PHY_1) { 104 src |= BIT(D40_SREG_CFG_PHY_PEN_POS); 105 src |= cfg->src_info.psize << D40_SREG_CFG_PSIZE_POS; 106 } 107 if (cfg->dst_info.psize != STEDMA40_PSIZE_PHY_1) { 108 dst |= BIT(D40_SREG_CFG_PHY_PEN_POS); 109 dst |= cfg->dst_info.psize << D40_SREG_CFG_PSIZE_POS; 110 } 111 112 /* Element size */ 113 src |= d40_width_to_bits(cfg->src_info.data_width) 114 << D40_SREG_CFG_ESIZE_POS; 115 dst |= d40_width_to_bits(cfg->dst_info.data_width) 116 << D40_SREG_CFG_ESIZE_POS; 117 118 /* Set the priority bit to high for the physical channel */ 119 if (cfg->high_priority) { 120 src |= BIT(D40_SREG_CFG_PRI_POS); 121 dst |= BIT(D40_SREG_CFG_PRI_POS); 122 } 123 124 if (cfg->src_info.big_endian) 125 src |= BIT(D40_SREG_CFG_LBE_POS); 126 if (cfg->dst_info.big_endian) 127 dst |= BIT(D40_SREG_CFG_LBE_POS); 128 129 *src_cfg = src; 130 *dst_cfg = dst; 131} 132 133static int d40_phy_fill_lli(struct d40_phy_lli *lli, 134 dma_addr_t data, 135 u32 data_size, 136 dma_addr_t next_lli, 137 u32 reg_cfg, 138 struct stedma40_half_channel_info *info, 139 unsigned int flags) 140{ 141 bool addr_inc = flags & LLI_ADDR_INC; 142 bool term_int = flags & LLI_TERM_INT; 143 unsigned int data_width = info->data_width; 144 int psize = info->psize; 145 int num_elems; 146 147 if (psize == STEDMA40_PSIZE_PHY_1) 148 num_elems = 1; 149 else 150 num_elems = 2 << psize; 151 152 /* Must be aligned */ 153 if (!IS_ALIGNED(data, data_width)) 154 return -EINVAL; 155 156 /* Transfer size can't be smaller than (num_elms * elem_size) */ 157 if (data_size < num_elems * data_width) 158 return -EINVAL; 159 160 /* The number of elements. IE now many chunks */ 161 lli->reg_elt = (data_size / data_width) << D40_SREG_ELEM_PHY_ECNT_POS; 162 163 /* 164 * Distance to next element sized entry. 165 * Usually the size of the element unless you want gaps. 166 */ 167 if (addr_inc) 168 lli->reg_elt |= data_width << D40_SREG_ELEM_PHY_EIDX_POS; 169 170 /* Where the data is */ 171 lli->reg_ptr = data; 172 lli->reg_cfg = reg_cfg; 173 174 /* If this scatter list entry is the last one, no next link */ 175 if (next_lli == 0) 176 lli->reg_lnk = BIT(D40_SREG_LNK_PHY_TCP_POS); 177 else 178 lli->reg_lnk = next_lli; 179 180 /* Set/clear interrupt generation on this link item.*/ 181 if (term_int) 182 lli->reg_cfg |= BIT(D40_SREG_CFG_TIM_POS); 183 else 184 lli->reg_cfg &= ~BIT(D40_SREG_CFG_TIM_POS); 185 186 /* 187 * Post link - D40_SREG_LNK_PHY_PRE_POS = 0 188 * Relink happens after transfer completion. 189 */ 190 191 return 0; 192} 193 194static int d40_seg_size(int size, int data_width1, int data_width2) 195{ 196 u32 max_w = max(data_width1, data_width2); 197 u32 min_w = min(data_width1, data_width2); 198 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE * min_w, max_w); 199 200 if (seg_max > STEDMA40_MAX_SEG_SIZE) 201 seg_max -= max_w; 202 203 if (size <= seg_max) 204 return size; 205 206 if (size <= 2 * seg_max) 207 return ALIGN(size / 2, max_w); 208 209 return seg_max; 210} 211 212static struct d40_phy_lli * 213d40_phy_buf_to_lli(struct d40_phy_lli *lli, dma_addr_t addr, u32 size, 214 dma_addr_t lli_phys, dma_addr_t first_phys, u32 reg_cfg, 215 struct stedma40_half_channel_info *info, 216 struct stedma40_half_channel_info *otherinfo, 217 unsigned long flags) 218{ 219 bool lastlink = flags & LLI_LAST_LINK; 220 bool addr_inc = flags & LLI_ADDR_INC; 221 bool term_int = flags & LLI_TERM_INT; 222 bool cyclic = flags & LLI_CYCLIC; 223 int err; 224 dma_addr_t next = lli_phys; 225 int size_rest = size; 226 int size_seg = 0; 227 228 /* 229 * This piece may be split up based on d40_seg_size(); we only want the 230 * term int on the last part. 231 */ 232 if (term_int) 233 flags &= ~LLI_TERM_INT; 234 235 do { 236 size_seg = d40_seg_size(size_rest, info->data_width, 237 otherinfo->data_width); 238 size_rest -= size_seg; 239 240 if (size_rest == 0 && term_int) 241 flags |= LLI_TERM_INT; 242 243 if (size_rest == 0 && lastlink) 244 next = cyclic ? first_phys : 0; 245 else 246 next = ALIGN(next + sizeof(struct d40_phy_lli), 247 D40_LLI_ALIGN); 248 249 err = d40_phy_fill_lli(lli, addr, size_seg, next, 250 reg_cfg, info, flags); 251 252 if (err) 253 goto err; 254 255 lli++; 256 if (addr_inc) 257 addr += size_seg; 258 } while (size_rest); 259 260 return lli; 261 262err: 263 return NULL; 264} 265 266int d40_phy_sg_to_lli(struct scatterlist *sg, 267 int sg_len, 268 dma_addr_t target, 269 struct d40_phy_lli *lli_sg, 270 dma_addr_t lli_phys, 271 u32 reg_cfg, 272 struct stedma40_half_channel_info *info, 273 struct stedma40_half_channel_info *otherinfo, 274 unsigned long flags) 275{ 276 int total_size = 0; 277 int i; 278 struct scatterlist *current_sg = sg; 279 struct d40_phy_lli *lli = lli_sg; 280 dma_addr_t l_phys = lli_phys; 281 282 if (!target) 283 flags |= LLI_ADDR_INC; 284 285 for_each_sg(sg, current_sg, sg_len, i) { 286 dma_addr_t sg_addr = sg_dma_address(current_sg); 287 unsigned int len = sg_dma_len(current_sg); 288 dma_addr_t dst = target ?: sg_addr; 289 290 total_size += sg_dma_len(current_sg); 291 292 if (i == sg_len - 1) 293 flags |= LLI_TERM_INT | LLI_LAST_LINK; 294 295 l_phys = ALIGN(lli_phys + (lli - lli_sg) * 296 sizeof(struct d40_phy_lli), D40_LLI_ALIGN); 297 298 lli = d40_phy_buf_to_lli(lli, dst, len, l_phys, lli_phys, 299 reg_cfg, info, otherinfo, flags); 300 301 if (lli == NULL) 302 return -EINVAL; 303 } 304 305 return total_size; 306} 307 308 309/* DMA logical lli operations */ 310 311static void d40_log_lli_link(struct d40_log_lli *lli_dst, 312 struct d40_log_lli *lli_src, 313 int next, unsigned int flags) 314{ 315 bool interrupt = flags & LLI_TERM_INT; 316 u32 slos = 0; 317 u32 dlos = 0; 318 319 if (next != -EINVAL) { 320 slos = next * 2; 321 dlos = next * 2 + 1; 322 } 323 324 if (interrupt) { 325 lli_dst->lcsp13 |= D40_MEM_LCSP1_SCFG_TIM_MASK; 326 lli_dst->lcsp13 |= D40_MEM_LCSP3_DTCP_MASK; 327 } 328 329 lli_src->lcsp13 = (lli_src->lcsp13 & ~D40_MEM_LCSP1_SLOS_MASK) | 330 (slos << D40_MEM_LCSP1_SLOS_POS); 331 332 lli_dst->lcsp13 = (lli_dst->lcsp13 & ~D40_MEM_LCSP1_SLOS_MASK) | 333 (dlos << D40_MEM_LCSP1_SLOS_POS); 334} 335 336void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa, 337 struct d40_log_lli *lli_dst, 338 struct d40_log_lli *lli_src, 339 int next, unsigned int flags) 340{ 341 d40_log_lli_link(lli_dst, lli_src, next, flags); 342 343 writel_relaxed(lli_src->lcsp02, &lcpa[0].lcsp0); 344 writel_relaxed(lli_src->lcsp13, &lcpa[0].lcsp1); 345 writel_relaxed(lli_dst->lcsp02, &lcpa[0].lcsp2); 346 writel_relaxed(lli_dst->lcsp13, &lcpa[0].lcsp3); 347} 348 349void d40_log_lli_lcla_write(struct d40_log_lli *lcla, 350 struct d40_log_lli *lli_dst, 351 struct d40_log_lli *lli_src, 352 int next, unsigned int flags) 353{ 354 d40_log_lli_link(lli_dst, lli_src, next, flags); 355 356 writel_relaxed(lli_src->lcsp02, &lcla[0].lcsp02); 357 writel_relaxed(lli_src->lcsp13, &lcla[0].lcsp13); 358 writel_relaxed(lli_dst->lcsp02, &lcla[1].lcsp02); 359 writel_relaxed(lli_dst->lcsp13, &lcla[1].lcsp13); 360} 361 362static void d40_log_fill_lli(struct d40_log_lli *lli, 363 dma_addr_t data, u32 data_size, 364 u32 reg_cfg, 365 u32 data_width, 366 unsigned int flags) 367{ 368 bool addr_inc = flags & LLI_ADDR_INC; 369 370 lli->lcsp13 = reg_cfg; 371 372 /* The number of elements to transfer */ 373 lli->lcsp02 = ((data_size / data_width) << 374 D40_MEM_LCSP0_ECNT_POS) & D40_MEM_LCSP0_ECNT_MASK; 375 376 BUG_ON((data_size / data_width) > STEDMA40_MAX_SEG_SIZE); 377 378 /* 16 LSBs address of the current element */ 379 lli->lcsp02 |= data & D40_MEM_LCSP0_SPTR_MASK; 380 /* 16 MSBs address of the current element */ 381 lli->lcsp13 |= data & D40_MEM_LCSP1_SPTR_MASK; 382 383 if (addr_inc) 384 lli->lcsp13 |= D40_MEM_LCSP1_SCFG_INCR_MASK; 385 386} 387 388static struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg, 389 dma_addr_t addr, 390 int size, 391 u32 lcsp13, /* src or dst*/ 392 u32 data_width1, 393 u32 data_width2, 394 unsigned int flags) 395{ 396 bool addr_inc = flags & LLI_ADDR_INC; 397 struct d40_log_lli *lli = lli_sg; 398 int size_rest = size; 399 int size_seg = 0; 400 401 do { 402 size_seg = d40_seg_size(size_rest, data_width1, data_width2); 403 size_rest -= size_seg; 404 405 d40_log_fill_lli(lli, 406 addr, 407 size_seg, 408 lcsp13, data_width1, 409 flags); 410 if (addr_inc) 411 addr += size_seg; 412 lli++; 413 } while (size_rest); 414 415 return lli; 416} 417 418int d40_log_sg_to_lli(struct scatterlist *sg, 419 int sg_len, 420 dma_addr_t dev_addr, 421 struct d40_log_lli *lli_sg, 422 u32 lcsp13, /* src or dst*/ 423 u32 data_width1, u32 data_width2) 424{ 425 int total_size = 0; 426 struct scatterlist *current_sg = sg; 427 int i; 428 struct d40_log_lli *lli = lli_sg; 429 unsigned long flags = 0; 430 431 if (!dev_addr) 432 flags |= LLI_ADDR_INC; 433 434 for_each_sg(sg, current_sg, sg_len, i) { 435 dma_addr_t sg_addr = sg_dma_address(current_sg); 436 unsigned int len = sg_dma_len(current_sg); 437 dma_addr_t addr = dev_addr ?: sg_addr; 438 439 total_size += sg_dma_len(current_sg); 440 441 lli = d40_log_buf_to_lli(lli, addr, len, 442 lcsp13, 443 data_width1, 444 data_width2, 445 flags); 446 } 447 448 return total_size; 449} 450