1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (c) 2021, MediaTek Inc. 4 * Copyright (c) 2021-2022, Intel Corporation. 5 * 6 * Authors: 7 * Amir Hanania <amir.hanania@intel.com> 8 * Haijun Liu <haijun.liu@mediatek.com> 9 * Moises Veleta <moises.veleta@intel.com> 10 * Ricardo Martinez <ricardo.martinez@linux.intel.com> 11 * 12 * Contributors: 13 * Andy Shevchenko <andriy.shevchenko@linux.intel.com> 14 * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> 15 * Eliot Lee <eliot.lee@intel.com> 16 * Sreehari Kancharla <sreehari.kancharla@intel.com> 17 */ 18 19#include <linux/bits.h> 20#include <linux/bitfield.h> 21#include <linux/bitops.h> 22#include <linux/delay.h> 23#include <linux/dev_printk.h> 24#include <linux/io.h> 25#include <linux/iopoll.h> 26#include <linux/types.h> 27 28#include "t7xx_dpmaif.h" 29#include "t7xx_reg.h" 30 31#define ioread32_poll_timeout_atomic(addr, val, cond, delay_us, timeout_us) \ 32 readx_poll_timeout_atomic(ioread32, addr, val, cond, delay_us, timeout_us) 33 34static int t7xx_dpmaif_init_intr(struct dpmaif_hw_info *hw_info) 35{ 36 struct dpmaif_isr_en_mask *isr_en_msk = &hw_info->isr_en_mask; 37 u32 value, ul_intr_enable, dl_intr_enable; 38 int ret; 39 40 ul_intr_enable = DP_UL_INT_ERR_MSK | DP_UL_INT_QDONE_MSK; 41 isr_en_msk->ap_ul_l2intr_en_msk = ul_intr_enable; 42 iowrite32(DPMAIF_AP_ALL_L2TISAR0_MASK, hw_info->pcie_base + DPMAIF_AP_L2TISAR0); 43 44 /* Set interrupt enable mask */ 45 iowrite32(ul_intr_enable, hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMCR0); 46 iowrite32(~ul_intr_enable, hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMSR0); 47 48 /* Check mask status */ 49 ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMR0, 50 value, (value & ul_intr_enable) != ul_intr_enable, 0, 51 DPMAIF_CHECK_INIT_TIMEOUT_US); 52 if (ret) 53 return ret; 54 55 dl_intr_enable = DP_DL_INT_PITCNT_LEN_ERR | DP_DL_INT_BATCNT_LEN_ERR; 56 isr_en_msk->ap_dl_l2intr_err_en_msk = dl_intr_enable; 57 ul_intr_enable = DPMAIF_DL_INT_DLQ0_QDONE | DPMAIF_DL_INT_DLQ0_PITCNT_LEN | 58 DPMAIF_DL_INT_DLQ1_QDONE | DPMAIF_DL_INT_DLQ1_PITCNT_LEN; 59 isr_en_msk->ap_ul_l2intr_en_msk = ul_intr_enable; 60 iowrite32(DPMAIF_AP_APDL_ALL_L2TISAR0_MASK, hw_info->pcie_base + DPMAIF_AP_APDL_L2TISAR0); 61 62 /* Set DL ISR PD enable mask */ 63 iowrite32(~ul_intr_enable, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0); 64 ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMR0, 65 value, (value & ul_intr_enable) != ul_intr_enable, 0, 66 DPMAIF_CHECK_INIT_TIMEOUT_US); 67 if (ret) 68 return ret; 69 70 isr_en_msk->ap_udl_ip_busy_en_msk = DPMAIF_UDL_IP_BUSY; 71 iowrite32(DPMAIF_AP_IP_BUSY_MASK, hw_info->pcie_base + DPMAIF_AP_IP_BUSY); 72 iowrite32(isr_en_msk->ap_udl_ip_busy_en_msk, 73 hw_info->pcie_base + DPMAIF_AO_AP_DLUL_IP_BUSY_MASK); 74 value = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_AP_L1TIMR0); 75 value |= DPMAIF_DL_INT_Q2APTOP | DPMAIF_DL_INT_Q2TOQ1; 76 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_UL_AP_L1TIMR0); 77 iowrite32(DPMA_HPC_ALL_INT_MASK, hw_info->pcie_base + DPMAIF_HPC_INTR_MASK); 78 79 return 0; 80} 81 82static void t7xx_dpmaif_mask_ulq_intr(struct dpmaif_hw_info *hw_info, unsigned int q_num) 83{ 84 struct dpmaif_isr_en_mask *isr_en_msk; 85 u32 value, ul_int_que_done; 86 int ret; 87 88 isr_en_msk = &hw_info->isr_en_mask; 89 ul_int_que_done = BIT(q_num + DP_UL_INT_DONE_OFFSET) & DP_UL_INT_QDONE_MSK; 90 isr_en_msk->ap_ul_l2intr_en_msk &= ~ul_int_que_done; 91 iowrite32(ul_int_que_done, hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMSR0); 92 93 ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMR0, 94 value, (value & ul_int_que_done) == ul_int_que_done, 0, 95 DPMAIF_CHECK_TIMEOUT_US); 96 if (ret) 97 dev_err(hw_info->dev, 98 "Could not mask the UL interrupt. DPMAIF_AO_UL_AP_L2TIMR0 is 0x%x\n", 99 value); 100} 101 102void t7xx_dpmaif_unmask_ulq_intr(struct dpmaif_hw_info *hw_info, unsigned int q_num) 103{ 104 struct dpmaif_isr_en_mask *isr_en_msk; 105 u32 value, ul_int_que_done; 106 int ret; 107 108 isr_en_msk = &hw_info->isr_en_mask; 109 ul_int_que_done = BIT(q_num + DP_UL_INT_DONE_OFFSET) & DP_UL_INT_QDONE_MSK; 110 isr_en_msk->ap_ul_l2intr_en_msk |= ul_int_que_done; 111 iowrite32(ul_int_que_done, hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMCR0); 112 113 ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMR0, 114 value, (value & ul_int_que_done) != ul_int_que_done, 0, 115 DPMAIF_CHECK_TIMEOUT_US); 116 if (ret) 117 dev_err(hw_info->dev, 118 "Could not unmask the UL interrupt. DPMAIF_AO_UL_AP_L2TIMR0 is 0x%x\n", 119 value); 120} 121 122void t7xx_dpmaif_dl_unmask_batcnt_len_err_intr(struct dpmaif_hw_info *hw_info) 123{ 124 hw_info->isr_en_mask.ap_dl_l2intr_en_msk |= DP_DL_INT_BATCNT_LEN_ERR; 125 iowrite32(DP_DL_INT_BATCNT_LEN_ERR, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMCR0); 126} 127 128void t7xx_dpmaif_dl_unmask_pitcnt_len_err_intr(struct dpmaif_hw_info *hw_info) 129{ 130 hw_info->isr_en_mask.ap_dl_l2intr_en_msk |= DP_DL_INT_PITCNT_LEN_ERR; 131 iowrite32(DP_DL_INT_PITCNT_LEN_ERR, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMCR0); 132} 133 134static u32 t7xx_update_dlq_intr(struct dpmaif_hw_info *hw_info, u32 q_done) 135{ 136 u32 value; 137 138 value = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMR0); 139 iowrite32(q_done, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0); 140 return value; 141} 142 143static int t7xx_mask_dlq_intr(struct dpmaif_hw_info *hw_info, unsigned int qno) 144{ 145 u32 value, q_done; 146 int ret; 147 148 q_done = qno == DPF_RX_QNO0 ? DPMAIF_DL_INT_DLQ0_QDONE : DPMAIF_DL_INT_DLQ1_QDONE; 149 iowrite32(q_done, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0); 150 151 ret = read_poll_timeout_atomic(t7xx_update_dlq_intr, value, value & q_done, 152 0, DPMAIF_CHECK_TIMEOUT_US, false, hw_info, q_done); 153 if (ret) { 154 dev_err(hw_info->dev, 155 "Could not mask the DL interrupt. DPMAIF_AO_UL_AP_L2TIMR0 is 0x%x\n", 156 value); 157 return -ETIMEDOUT; 158 } 159 160 hw_info->isr_en_mask.ap_dl_l2intr_en_msk &= ~q_done; 161 return 0; 162} 163 164void t7xx_dpmaif_dlq_unmask_rx_done(struct dpmaif_hw_info *hw_info, unsigned int qno) 165{ 166 u32 mask; 167 168 mask = qno == DPF_RX_QNO0 ? DPMAIF_DL_INT_DLQ0_QDONE : DPMAIF_DL_INT_DLQ1_QDONE; 169 iowrite32(mask, hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMCR0); 170 hw_info->isr_en_mask.ap_dl_l2intr_en_msk |= mask; 171} 172 173void t7xx_dpmaif_clr_ip_busy_sts(struct dpmaif_hw_info *hw_info) 174{ 175 u32 ip_busy_sts; 176 177 ip_busy_sts = ioread32(hw_info->pcie_base + DPMAIF_AP_IP_BUSY); 178 iowrite32(ip_busy_sts, hw_info->pcie_base + DPMAIF_AP_IP_BUSY); 179} 180 181static void t7xx_dpmaif_dlq_mask_rx_pitcnt_len_err_intr(struct dpmaif_hw_info *hw_info, 182 unsigned int qno) 183{ 184 if (qno == DPF_RX_QNO0) 185 iowrite32(DPMAIF_DL_INT_DLQ0_PITCNT_LEN, 186 hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0); 187 else 188 iowrite32(DPMAIF_DL_INT_DLQ1_PITCNT_LEN, 189 hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0); 190} 191 192void t7xx_dpmaif_dlq_unmask_pitcnt_len_err_intr(struct dpmaif_hw_info *hw_info, 193 unsigned int qno) 194{ 195 if (qno == DPF_RX_QNO0) 196 iowrite32(DPMAIF_DL_INT_DLQ0_PITCNT_LEN, 197 hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMCR0); 198 else 199 iowrite32(DPMAIF_DL_INT_DLQ1_PITCNT_LEN, 200 hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMCR0); 201} 202 203void t7xx_dpmaif_ul_clr_all_intr(struct dpmaif_hw_info *hw_info) 204{ 205 iowrite32(DPMAIF_AP_ALL_L2TISAR0_MASK, hw_info->pcie_base + DPMAIF_AP_L2TISAR0); 206} 207 208void t7xx_dpmaif_dl_clr_all_intr(struct dpmaif_hw_info *hw_info) 209{ 210 iowrite32(DPMAIF_AP_APDL_ALL_L2TISAR0_MASK, hw_info->pcie_base + DPMAIF_AP_APDL_L2TISAR0); 211} 212 213static void t7xx_dpmaif_set_intr_para(struct dpmaif_hw_intr_st_para *para, 214 enum dpmaif_hw_intr_type intr_type, unsigned int intr_queue) 215{ 216 para->intr_types[para->intr_cnt] = intr_type; 217 para->intr_queues[para->intr_cnt] = intr_queue; 218 para->intr_cnt++; 219} 220 221/* The para->intr_cnt counter is set to zero before this function is called. 222 * It does not check for overflow as there is no risk of overflowing intr_types or intr_queues. 223 */ 224static void t7xx_dpmaif_hw_check_tx_intr(struct dpmaif_hw_info *hw_info, 225 unsigned int intr_status, 226 struct dpmaif_hw_intr_st_para *para) 227{ 228 unsigned long value; 229 230 value = FIELD_GET(DP_UL_INT_QDONE_MSK, intr_status); 231 if (value) { 232 unsigned int index; 233 234 t7xx_dpmaif_set_intr_para(para, DPF_INTR_UL_DONE, value); 235 236 for_each_set_bit(index, &value, DPMAIF_TXQ_NUM) 237 t7xx_dpmaif_mask_ulq_intr(hw_info, index); 238 } 239 240 value = FIELD_GET(DP_UL_INT_EMPTY_MSK, intr_status); 241 if (value) 242 t7xx_dpmaif_set_intr_para(para, DPF_INTR_UL_DRB_EMPTY, value); 243 244 value = FIELD_GET(DP_UL_INT_MD_NOTREADY_MSK, intr_status); 245 if (value) 246 t7xx_dpmaif_set_intr_para(para, DPF_INTR_UL_MD_NOTREADY, value); 247 248 value = FIELD_GET(DP_UL_INT_MD_PWR_NOTREADY_MSK, intr_status); 249 if (value) 250 t7xx_dpmaif_set_intr_para(para, DPF_INTR_UL_MD_PWR_NOTREADY, value); 251 252 value = FIELD_GET(DP_UL_INT_ERR_MSK, intr_status); 253 if (value) 254 t7xx_dpmaif_set_intr_para(para, DPF_INTR_UL_LEN_ERR, value); 255 256 /* Clear interrupt status */ 257 iowrite32(intr_status, hw_info->pcie_base + DPMAIF_AP_L2TISAR0); 258} 259 260/* The para->intr_cnt counter is set to zero before this function is called. 261 * It does not check for overflow as there is no risk of overflowing intr_types or intr_queues. 262 */ 263static void t7xx_dpmaif_hw_check_rx_intr(struct dpmaif_hw_info *hw_info, 264 unsigned int intr_status, 265 struct dpmaif_hw_intr_st_para *para, int qno) 266{ 267 if (qno == DPF_RX_QNO_DFT) { 268 if (intr_status & DP_DL_INT_SKB_LEN_ERR) 269 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_SKB_LEN_ERR, DPF_RX_QNO_DFT); 270 271 if (intr_status & DP_DL_INT_BATCNT_LEN_ERR) { 272 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_BATCNT_LEN_ERR, DPF_RX_QNO_DFT); 273 hw_info->isr_en_mask.ap_dl_l2intr_en_msk &= ~DP_DL_INT_BATCNT_LEN_ERR; 274 iowrite32(DP_DL_INT_BATCNT_LEN_ERR, 275 hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0); 276 } 277 278 if (intr_status & DP_DL_INT_PITCNT_LEN_ERR) { 279 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_PITCNT_LEN_ERR, DPF_RX_QNO_DFT); 280 hw_info->isr_en_mask.ap_dl_l2intr_en_msk &= ~DP_DL_INT_PITCNT_LEN_ERR; 281 iowrite32(DP_DL_INT_PITCNT_LEN_ERR, 282 hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMSR0); 283 } 284 285 if (intr_status & DP_DL_INT_PKT_EMPTY_MSK) 286 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_PKT_EMPTY_SET, DPF_RX_QNO_DFT); 287 288 if (intr_status & DP_DL_INT_FRG_EMPTY_MSK) 289 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_FRG_EMPTY_SET, DPF_RX_QNO_DFT); 290 291 if (intr_status & DP_DL_INT_MTU_ERR_MSK) 292 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_MTU_ERR, DPF_RX_QNO_DFT); 293 294 if (intr_status & DP_DL_INT_FRG_LEN_ERR_MSK) 295 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_FRGCNT_LEN_ERR, DPF_RX_QNO_DFT); 296 297 if (intr_status & DP_DL_INT_Q0_PITCNT_LEN_ERR) { 298 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_Q0_PITCNT_LEN_ERR, BIT(qno)); 299 t7xx_dpmaif_dlq_mask_rx_pitcnt_len_err_intr(hw_info, qno); 300 } 301 302 if (intr_status & DP_DL_INT_HPC_ENT_TYPE_ERR) 303 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_HPC_ENT_TYPE_ERR, 304 DPF_RX_QNO_DFT); 305 306 if (intr_status & DP_DL_INT_Q0_DONE) { 307 /* Mask RX done interrupt immediately after it occurs, do not clear 308 * the interrupt if the mask operation fails. 309 */ 310 if (!t7xx_mask_dlq_intr(hw_info, qno)) 311 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_Q0_DONE, BIT(qno)); 312 else 313 intr_status &= ~DP_DL_INT_Q0_DONE; 314 } 315 } else { 316 if (intr_status & DP_DL_INT_Q1_PITCNT_LEN_ERR) { 317 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_Q1_PITCNT_LEN_ERR, BIT(qno)); 318 t7xx_dpmaif_dlq_mask_rx_pitcnt_len_err_intr(hw_info, qno); 319 } 320 321 if (intr_status & DP_DL_INT_Q1_DONE) { 322 if (!t7xx_mask_dlq_intr(hw_info, qno)) 323 t7xx_dpmaif_set_intr_para(para, DPF_INTR_DL_Q1_DONE, BIT(qno)); 324 else 325 intr_status &= ~DP_DL_INT_Q1_DONE; 326 } 327 } 328 329 intr_status |= DP_DL_INT_BATCNT_LEN_ERR; 330 /* Clear interrupt status */ 331 iowrite32(intr_status, hw_info->pcie_base + DPMAIF_AP_APDL_L2TISAR0); 332} 333 334/** 335 * t7xx_dpmaif_hw_get_intr_cnt() - Reads interrupt status and count from HW. 336 * @hw_info: Pointer to struct hw_info. 337 * @para: Pointer to struct dpmaif_hw_intr_st_para. 338 * @qno: Queue number. 339 * 340 * Reads RX/TX interrupt status from HW and clears UL/DL status as needed. 341 * 342 * Return: Interrupt count. 343 */ 344int t7xx_dpmaif_hw_get_intr_cnt(struct dpmaif_hw_info *hw_info, 345 struct dpmaif_hw_intr_st_para *para, int qno) 346{ 347 u32 rx_intr_status, tx_intr_status = 0; 348 u32 rx_intr_qdone, tx_intr_qdone = 0; 349 350 rx_intr_status = ioread32(hw_info->pcie_base + DPMAIF_AP_APDL_L2TISAR0); 351 rx_intr_qdone = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_APDL_L2TIMR0); 352 353 /* TX interrupt status */ 354 if (qno == DPF_RX_QNO_DFT) { 355 /* All ULQ and DLQ0 interrupts use the same source no need to check ULQ interrupts 356 * when a DLQ1 interrupt has occurred. 357 */ 358 tx_intr_status = ioread32(hw_info->pcie_base + DPMAIF_AP_L2TISAR0); 359 tx_intr_qdone = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_AP_L2TIMR0); 360 } 361 362 t7xx_dpmaif_clr_ip_busy_sts(hw_info); 363 364 if (qno == DPF_RX_QNO_DFT) { 365 /* Do not schedule bottom half again or clear UL interrupt status when we 366 * have already masked it. 367 */ 368 tx_intr_status &= ~tx_intr_qdone; 369 if (tx_intr_status) 370 t7xx_dpmaif_hw_check_tx_intr(hw_info, tx_intr_status, para); 371 } 372 373 if (rx_intr_status) { 374 if (qno == DPF_RX_QNO0) { 375 rx_intr_status &= DP_DL_Q0_STATUS_MASK; 376 if (rx_intr_qdone & DPMAIF_DL_INT_DLQ0_QDONE) 377 /* Do not schedule bottom half again or clear DL 378 * queue done interrupt status when we have already masked it. 379 */ 380 rx_intr_status &= ~DP_DL_INT_Q0_DONE; 381 } else { 382 rx_intr_status &= DP_DL_Q1_STATUS_MASK; 383 if (rx_intr_qdone & DPMAIF_DL_INT_DLQ1_QDONE) 384 rx_intr_status &= ~DP_DL_INT_Q1_DONE; 385 } 386 387 if (rx_intr_status) 388 t7xx_dpmaif_hw_check_rx_intr(hw_info, rx_intr_status, para, qno); 389 } 390 391 return para->intr_cnt; 392} 393 394static int t7xx_dpmaif_sram_init(struct dpmaif_hw_info *hw_info) 395{ 396 u32 value; 397 398 value = ioread32(hw_info->pcie_base + DPMAIF_AP_MEM_CLR); 399 value |= DPMAIF_MEM_CLR; 400 iowrite32(value, hw_info->pcie_base + DPMAIF_AP_MEM_CLR); 401 402 return ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AP_MEM_CLR, 403 value, !(value & DPMAIF_MEM_CLR), 0, 404 DPMAIF_CHECK_INIT_TIMEOUT_US); 405} 406 407static void t7xx_dpmaif_hw_reset(struct dpmaif_hw_info *hw_info) 408{ 409 iowrite32(DPMAIF_AP_AO_RST_BIT, hw_info->pcie_base + DPMAIF_AP_AO_RGU_ASSERT); 410 udelay(2); 411 iowrite32(DPMAIF_AP_RST_BIT, hw_info->pcie_base + DPMAIF_AP_RGU_ASSERT); 412 udelay(2); 413 iowrite32(DPMAIF_AP_AO_RST_BIT, hw_info->pcie_base + DPMAIF_AP_AO_RGU_DEASSERT); 414 udelay(2); 415 iowrite32(DPMAIF_AP_RST_BIT, hw_info->pcie_base + DPMAIF_AP_RGU_DEASSERT); 416 udelay(2); 417} 418 419static int t7xx_dpmaif_hw_config(struct dpmaif_hw_info *hw_info) 420{ 421 u32 ap_port_mode; 422 int ret; 423 424 t7xx_dpmaif_hw_reset(hw_info); 425 426 ret = t7xx_dpmaif_sram_init(hw_info); 427 if (ret) 428 return ret; 429 430 ap_port_mode = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES); 431 ap_port_mode |= DPMAIF_PORT_MODE_PCIE; 432 iowrite32(ap_port_mode, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES); 433 iowrite32(DPMAIF_CG_EN, hw_info->pcie_base + DPMAIF_AP_CG_EN); 434 return 0; 435} 436 437static void t7xx_dpmaif_pcie_dpmaif_sign(struct dpmaif_hw_info *hw_info) 438{ 439 iowrite32(DPMAIF_PCIE_MODE_SET_VALUE, hw_info->pcie_base + DPMAIF_UL_RESERVE_AO_RW); 440} 441 442static void t7xx_dpmaif_dl_performance(struct dpmaif_hw_info *hw_info) 443{ 444 u32 enable_bat_cache, enable_pit_burst; 445 446 enable_bat_cache = ioread32(hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1); 447 enable_bat_cache |= DPMAIF_DL_BAT_CACHE_PRI; 448 iowrite32(enable_bat_cache, hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1); 449 450 enable_pit_burst = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES); 451 enable_pit_burst |= DPMAIF_DL_BURST_PIT_EN; 452 iowrite32(enable_pit_burst, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES); 453} 454 455 /* DPMAIF DL DLQ part HW setting */ 456 457static void t7xx_dpmaif_hw_hpc_cntl_set(struct dpmaif_hw_info *hw_info) 458{ 459 unsigned int value; 460 461 value = DPMAIF_HPC_DLQ_PATH_MODE | DPMAIF_HPC_ADD_MODE_DF << 2; 462 value |= DPMAIF_HASH_PRIME_DF << 4; 463 value |= DPMAIF_HPC_TOTAL_NUM << 8; 464 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_HPC_CNTL); 465} 466 467static void t7xx_dpmaif_hw_agg_cfg_set(struct dpmaif_hw_info *hw_info) 468{ 469 unsigned int value; 470 471 value = DPMAIF_AGG_MAX_LEN_DF | DPMAIF_AGG_TBL_ENT_NUM_DF << 16; 472 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_DLQ_AGG_CFG); 473} 474 475static void t7xx_dpmaif_hw_hash_bit_choose_set(struct dpmaif_hw_info *hw_info) 476{ 477 iowrite32(DPMAIF_DLQ_HASH_BIT_CHOOSE_DF, 478 hw_info->pcie_base + DPMAIF_AO_DL_DLQPIT_INIT_CON5); 479} 480 481static void t7xx_dpmaif_hw_mid_pit_timeout_thres_set(struct dpmaif_hw_info *hw_info) 482{ 483 iowrite32(DPMAIF_MID_TIMEOUT_THRES_DF, hw_info->pcie_base + DPMAIF_AO_DL_DLQPIT_TIMEOUT0); 484} 485 486static void t7xx_dpmaif_hw_dlq_timeout_thres_set(struct dpmaif_hw_info *hw_info) 487{ 488 unsigned int value, i; 489 490 /* Each register holds two DLQ threshold timeout values */ 491 for (i = 0; i < DPMAIF_HPC_MAX_TOTAL_NUM / 2; i++) { 492 value = FIELD_PREP(DPMAIF_DLQ_LOW_TIMEOUT_THRES_MKS, DPMAIF_DLQ_TIMEOUT_THRES_DF); 493 value |= FIELD_PREP(DPMAIF_DLQ_HIGH_TIMEOUT_THRES_MSK, 494 DPMAIF_DLQ_TIMEOUT_THRES_DF); 495 iowrite32(value, 496 hw_info->pcie_base + DPMAIF_AO_DL_DLQPIT_TIMEOUT1 + sizeof(u32) * i); 497 } 498} 499 500static void t7xx_dpmaif_hw_dlq_start_prs_thres_set(struct dpmaif_hw_info *hw_info) 501{ 502 iowrite32(DPMAIF_DLQ_PRS_THRES_DF, hw_info->pcie_base + DPMAIF_AO_DL_DLQPIT_TRIG_THRES); 503} 504 505static void t7xx_dpmaif_dl_dlq_hpc_hw_init(struct dpmaif_hw_info *hw_info) 506{ 507 t7xx_dpmaif_hw_hpc_cntl_set(hw_info); 508 t7xx_dpmaif_hw_agg_cfg_set(hw_info); 509 t7xx_dpmaif_hw_hash_bit_choose_set(hw_info); 510 t7xx_dpmaif_hw_mid_pit_timeout_thres_set(hw_info); 511 t7xx_dpmaif_hw_dlq_timeout_thres_set(hw_info); 512 t7xx_dpmaif_hw_dlq_start_prs_thres_set(hw_info); 513} 514 515static int t7xx_dpmaif_dl_bat_init_done(struct dpmaif_hw_info *hw_info, bool frg_en) 516{ 517 u32 value, dl_bat_init = 0; 518 int ret; 519 520 if (frg_en) 521 dl_bat_init = DPMAIF_DL_BAT_FRG_INIT; 522 523 dl_bat_init |= DPMAIF_DL_BAT_INIT_ALLSET; 524 dl_bat_init |= DPMAIF_DL_BAT_INIT_EN; 525 526 ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_BAT_INIT, 527 value, !(value & DPMAIF_DL_BAT_INIT_NOT_READY), 0, 528 DPMAIF_CHECK_INIT_TIMEOUT_US); 529 if (ret) { 530 dev_err(hw_info->dev, "Data plane modem DL BAT is not ready\n"); 531 return ret; 532 } 533 534 iowrite32(dl_bat_init, hw_info->pcie_base + DPMAIF_DL_BAT_INIT); 535 536 ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_BAT_INIT, 537 value, !(value & DPMAIF_DL_BAT_INIT_NOT_READY), 0, 538 DPMAIF_CHECK_INIT_TIMEOUT_US); 539 if (ret) 540 dev_err(hw_info->dev, "Data plane modem DL BAT initialization failed\n"); 541 542 return ret; 543} 544 545static void t7xx_dpmaif_dl_set_bat_base_addr(struct dpmaif_hw_info *hw_info, 546 dma_addr_t addr) 547{ 548 iowrite32(lower_32_bits(addr), hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON0); 549 iowrite32(upper_32_bits(addr), hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON3); 550} 551 552static void t7xx_dpmaif_dl_set_bat_size(struct dpmaif_hw_info *hw_info, unsigned int size) 553{ 554 unsigned int value; 555 556 value = ioread32(hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1); 557 value &= ~DPMAIF_BAT_SIZE_MSK; 558 value |= size & DPMAIF_BAT_SIZE_MSK; 559 iowrite32(value, hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1); 560} 561 562static void t7xx_dpmaif_dl_bat_en(struct dpmaif_hw_info *hw_info, bool enable) 563{ 564 unsigned int value; 565 566 value = ioread32(hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1); 567 568 if (enable) 569 value |= DPMAIF_BAT_EN_MSK; 570 else 571 value &= ~DPMAIF_BAT_EN_MSK; 572 573 iowrite32(value, hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1); 574} 575 576static void t7xx_dpmaif_dl_set_ao_bid_maxcnt(struct dpmaif_hw_info *hw_info) 577{ 578 unsigned int value; 579 580 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON0); 581 value &= ~DPMAIF_BAT_BID_MAXCNT_MSK; 582 value |= FIELD_PREP(DPMAIF_BAT_BID_MAXCNT_MSK, DPMAIF_HW_PKT_BIDCNT); 583 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON0); 584} 585 586static void t7xx_dpmaif_dl_set_ao_mtu(struct dpmaif_hw_info *hw_info) 587{ 588 iowrite32(DPMAIF_HW_MTU_SIZE, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON1); 589} 590 591static void t7xx_dpmaif_dl_set_ao_pit_chknum(struct dpmaif_hw_info *hw_info) 592{ 593 unsigned int value; 594 595 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2); 596 value &= ~DPMAIF_PIT_CHK_NUM_MSK; 597 value |= FIELD_PREP(DPMAIF_PIT_CHK_NUM_MSK, DPMAIF_HW_CHK_PIT_NUM); 598 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2); 599} 600 601static void t7xx_dpmaif_dl_set_ao_remain_minsz(struct dpmaif_hw_info *hw_info) 602{ 603 unsigned int value; 604 605 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON0); 606 value &= ~DPMAIF_BAT_REMAIN_MINSZ_MSK; 607 value |= FIELD_PREP(DPMAIF_BAT_REMAIN_MINSZ_MSK, 608 DPMAIF_HW_BAT_REMAIN / DPMAIF_BAT_REMAIN_SZ_BASE); 609 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON0); 610} 611 612static void t7xx_dpmaif_dl_set_ao_bat_bufsz(struct dpmaif_hw_info *hw_info) 613{ 614 unsigned int value; 615 616 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2); 617 value &= ~DPMAIF_BAT_BUF_SZ_MSK; 618 value |= FIELD_PREP(DPMAIF_BAT_BUF_SZ_MSK, 619 DPMAIF_HW_BAT_PKTBUF / DPMAIF_BAT_BUFFER_SZ_BASE); 620 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2); 621} 622 623static void t7xx_dpmaif_dl_set_ao_bat_rsv_length(struct dpmaif_hw_info *hw_info) 624{ 625 unsigned int value; 626 627 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2); 628 value &= ~DPMAIF_BAT_RSV_LEN_MSK; 629 value |= DPMAIF_HW_BAT_RSVLEN; 630 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PKTINFO_CON2); 631} 632 633static void t7xx_dpmaif_dl_set_pkt_alignment(struct dpmaif_hw_info *hw_info) 634{ 635 unsigned int value; 636 637 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES); 638 value &= ~DPMAIF_PKT_ALIGN_MSK; 639 value |= DPMAIF_PKT_ALIGN_EN; 640 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES); 641} 642 643static void t7xx_dpmaif_dl_set_pkt_checksum(struct dpmaif_hw_info *hw_info) 644{ 645 unsigned int value; 646 647 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES); 648 value |= DPMAIF_DL_PKT_CHECKSUM_EN; 649 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES); 650} 651 652static void t7xx_dpmaif_dl_set_ao_frg_check_thres(struct dpmaif_hw_info *hw_info) 653{ 654 unsigned int value; 655 656 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES); 657 value &= ~DPMAIF_FRG_CHECK_THRES_MSK; 658 value |= DPMAIF_HW_CHK_FRG_NUM; 659 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES); 660} 661 662static void t7xx_dpmaif_dl_set_ao_frg_bufsz(struct dpmaif_hw_info *hw_info) 663{ 664 unsigned int value; 665 666 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES); 667 value &= ~DPMAIF_FRG_BUF_SZ_MSK; 668 value |= FIELD_PREP(DPMAIF_FRG_BUF_SZ_MSK, 669 DPMAIF_HW_FRG_PKTBUF / DPMAIF_FRG_BUFFER_SZ_BASE); 670 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES); 671} 672 673static void t7xx_dpmaif_dl_frg_ao_en(struct dpmaif_hw_info *hw_info, bool enable) 674{ 675 unsigned int value; 676 677 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES); 678 679 if (enable) 680 value |= DPMAIF_FRG_EN_MSK; 681 else 682 value &= ~DPMAIF_FRG_EN_MSK; 683 684 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_FRG_THRES); 685} 686 687static void t7xx_dpmaif_dl_set_ao_bat_check_thres(struct dpmaif_hw_info *hw_info) 688{ 689 unsigned int value; 690 691 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES); 692 value &= ~DPMAIF_BAT_CHECK_THRES_MSK; 693 value |= FIELD_PREP(DPMAIF_BAT_CHECK_THRES_MSK, DPMAIF_HW_CHK_BAT_NUM); 694 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_RDY_CHK_THRES); 695} 696 697static void t7xx_dpmaif_dl_set_pit_seqnum(struct dpmaif_hw_info *hw_info) 698{ 699 unsigned int value; 700 701 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PIT_SEQ_END); 702 value &= ~DPMAIF_DL_PIT_SEQ_MSK; 703 value |= DPMAIF_DL_PIT_SEQ_VALUE; 704 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_DL_PIT_SEQ_END); 705} 706 707static void t7xx_dpmaif_dl_set_dlq_pit_base_addr(struct dpmaif_hw_info *hw_info, 708 dma_addr_t addr) 709{ 710 iowrite32(lower_32_bits(addr), hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON0); 711 iowrite32(upper_32_bits(addr), hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON4); 712} 713 714static void t7xx_dpmaif_dl_set_dlq_pit_size(struct dpmaif_hw_info *hw_info, unsigned int size) 715{ 716 unsigned int value; 717 718 value = ioread32(hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON1); 719 value &= ~DPMAIF_PIT_SIZE_MSK; 720 value |= size & DPMAIF_PIT_SIZE_MSK; 721 iowrite32(value, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON1); 722 iowrite32(0, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON2); 723 iowrite32(0, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON3); 724 iowrite32(0, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON5); 725 iowrite32(0, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON6); 726} 727 728static void t7xx_dpmaif_dl_dlq_pit_en(struct dpmaif_hw_info *hw_info) 729{ 730 unsigned int value; 731 732 value = ioread32(hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON3); 733 value |= DPMAIF_DLQPIT_EN_MSK; 734 iowrite32(value, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT_CON3); 735} 736 737static void t7xx_dpmaif_dl_dlq_pit_init_done(struct dpmaif_hw_info *hw_info, 738 unsigned int pit_idx) 739{ 740 unsigned int dl_pit_init; 741 int timeout; 742 u32 value; 743 744 dl_pit_init = DPMAIF_DL_PIT_INIT_ALLSET; 745 dl_pit_init |= (pit_idx << DPMAIF_DLQPIT_CHAN_OFS); 746 dl_pit_init |= DPMAIF_DL_PIT_INIT_EN; 747 748 timeout = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT, 749 value, !(value & DPMAIF_DL_PIT_INIT_NOT_READY), 750 DPMAIF_CHECK_DELAY_US, 751 DPMAIF_CHECK_INIT_TIMEOUT_US); 752 if (timeout) { 753 dev_err(hw_info->dev, "Data plane modem DL PIT is not ready\n"); 754 return; 755 } 756 757 iowrite32(dl_pit_init, hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT); 758 timeout = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_DLQPIT_INIT, 759 value, !(value & DPMAIF_DL_PIT_INIT_NOT_READY), 760 DPMAIF_CHECK_DELAY_US, 761 DPMAIF_CHECK_INIT_TIMEOUT_US); 762 if (timeout) 763 dev_err(hw_info->dev, "Data plane modem DL PIT initialization failed\n"); 764} 765 766static void t7xx_dpmaif_config_dlq_pit_hw(struct dpmaif_hw_info *hw_info, unsigned int q_num, 767 struct dpmaif_dl *dl_que) 768{ 769 t7xx_dpmaif_dl_set_dlq_pit_base_addr(hw_info, dl_que->pit_base); 770 t7xx_dpmaif_dl_set_dlq_pit_size(hw_info, dl_que->pit_size_cnt); 771 t7xx_dpmaif_dl_dlq_pit_en(hw_info); 772 t7xx_dpmaif_dl_dlq_pit_init_done(hw_info, q_num); 773} 774 775static void t7xx_dpmaif_config_all_dlq_hw(struct dpmaif_hw_info *hw_info) 776{ 777 int i; 778 779 for (i = 0; i < DPMAIF_RXQ_NUM; i++) 780 t7xx_dpmaif_config_dlq_pit_hw(hw_info, i, &hw_info->dl_que[i]); 781} 782 783static void t7xx_dpmaif_dl_all_q_en(struct dpmaif_hw_info *hw_info, bool enable) 784{ 785 u32 dl_bat_init, value; 786 int timeout; 787 788 value = ioread32(hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1); 789 790 if (enable) 791 value |= DPMAIF_BAT_EN_MSK; 792 else 793 value &= ~DPMAIF_BAT_EN_MSK; 794 795 iowrite32(value, hw_info->pcie_base + DPMAIF_DL_BAT_INIT_CON1); 796 dl_bat_init = DPMAIF_DL_BAT_INIT_ONLY_ENABLE_BIT; 797 dl_bat_init |= DPMAIF_DL_BAT_INIT_EN; 798 799 timeout = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_BAT_INIT, 800 value, !(value & DPMAIF_DL_BAT_INIT_NOT_READY), 0, 801 DPMAIF_CHECK_TIMEOUT_US); 802 if (timeout) 803 dev_err(hw_info->dev, "Timeout updating BAT setting to HW\n"); 804 805 iowrite32(dl_bat_init, hw_info->pcie_base + DPMAIF_DL_BAT_INIT); 806 timeout = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_BAT_INIT, 807 value, !(value & DPMAIF_DL_BAT_INIT_NOT_READY), 0, 808 DPMAIF_CHECK_TIMEOUT_US); 809 if (timeout) 810 dev_err(hw_info->dev, "Data plane modem DL BAT is not ready\n"); 811} 812 813static int t7xx_dpmaif_config_dlq_hw(struct dpmaif_hw_info *hw_info) 814{ 815 struct dpmaif_dl *dl_que; 816 int ret; 817 818 t7xx_dpmaif_dl_dlq_hpc_hw_init(hw_info); 819 820 dl_que = &hw_info->dl_que[0]; /* All queues share one BAT/frag BAT table */ 821 if (!dl_que->que_started) 822 return -EBUSY; 823 824 t7xx_dpmaif_dl_set_ao_remain_minsz(hw_info); 825 t7xx_dpmaif_dl_set_ao_bat_bufsz(hw_info); 826 t7xx_dpmaif_dl_set_ao_frg_bufsz(hw_info); 827 t7xx_dpmaif_dl_set_ao_bat_rsv_length(hw_info); 828 t7xx_dpmaif_dl_set_ao_bid_maxcnt(hw_info); 829 t7xx_dpmaif_dl_set_pkt_alignment(hw_info); 830 t7xx_dpmaif_dl_set_pit_seqnum(hw_info); 831 t7xx_dpmaif_dl_set_ao_mtu(hw_info); 832 t7xx_dpmaif_dl_set_ao_pit_chknum(hw_info); 833 t7xx_dpmaif_dl_set_ao_bat_check_thres(hw_info); 834 t7xx_dpmaif_dl_set_ao_frg_check_thres(hw_info); 835 t7xx_dpmaif_dl_frg_ao_en(hw_info, true); 836 837 t7xx_dpmaif_dl_set_bat_base_addr(hw_info, dl_que->frg_base); 838 t7xx_dpmaif_dl_set_bat_size(hw_info, dl_que->frg_size_cnt); 839 t7xx_dpmaif_dl_bat_en(hw_info, true); 840 841 ret = t7xx_dpmaif_dl_bat_init_done(hw_info, true); 842 if (ret) 843 return ret; 844 845 t7xx_dpmaif_dl_set_bat_base_addr(hw_info, dl_que->bat_base); 846 t7xx_dpmaif_dl_set_bat_size(hw_info, dl_que->bat_size_cnt); 847 t7xx_dpmaif_dl_bat_en(hw_info, false); 848 849 ret = t7xx_dpmaif_dl_bat_init_done(hw_info, false); 850 if (ret) 851 return ret; 852 853 /* Init PIT (two PIT table) */ 854 t7xx_dpmaif_config_all_dlq_hw(hw_info); 855 t7xx_dpmaif_dl_all_q_en(hw_info, true); 856 t7xx_dpmaif_dl_set_pkt_checksum(hw_info); 857 return 0; 858} 859 860static void t7xx_dpmaif_ul_update_drb_size(struct dpmaif_hw_info *hw_info, 861 unsigned int q_num, unsigned int size) 862{ 863 unsigned int value; 864 865 value = ioread32(hw_info->pcie_base + DPMAIF_UL_DRBSIZE_ADDRH_n(q_num)); 866 value &= ~DPMAIF_DRB_SIZE_MSK; 867 value |= size & DPMAIF_DRB_SIZE_MSK; 868 iowrite32(value, hw_info->pcie_base + DPMAIF_UL_DRBSIZE_ADDRH_n(q_num)); 869} 870 871static void t7xx_dpmaif_ul_update_drb_base_addr(struct dpmaif_hw_info *hw_info, 872 unsigned int q_num, dma_addr_t addr) 873{ 874 iowrite32(lower_32_bits(addr), hw_info->pcie_base + DPMAIF_ULQSAR_n(q_num)); 875 iowrite32(upper_32_bits(addr), hw_info->pcie_base + DPMAIF_UL_DRB_ADDRH_n(q_num)); 876} 877 878static void t7xx_dpmaif_ul_rdy_en(struct dpmaif_hw_info *hw_info, 879 unsigned int q_num, bool ready) 880{ 881 u32 value; 882 883 value = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0); 884 885 if (ready) 886 value |= BIT(q_num); 887 else 888 value &= ~BIT(q_num); 889 890 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0); 891} 892 893static void t7xx_dpmaif_ul_arb_en(struct dpmaif_hw_info *hw_info, 894 unsigned int q_num, bool enable) 895{ 896 u32 value; 897 898 value = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0); 899 900 if (enable) 901 value |= BIT(q_num + 8); 902 else 903 value &= ~BIT(q_num + 8); 904 905 iowrite32(value, hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0); 906} 907 908static void t7xx_dpmaif_config_ulq_hw(struct dpmaif_hw_info *hw_info) 909{ 910 struct dpmaif_ul *ul_que; 911 int i; 912 913 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 914 ul_que = &hw_info->ul_que[i]; 915 if (ul_que->que_started) { 916 t7xx_dpmaif_ul_update_drb_size(hw_info, i, ul_que->drb_size_cnt * 917 DPMAIF_UL_DRB_SIZE_WORD); 918 t7xx_dpmaif_ul_update_drb_base_addr(hw_info, i, ul_que->drb_base); 919 t7xx_dpmaif_ul_rdy_en(hw_info, i, true); 920 t7xx_dpmaif_ul_arb_en(hw_info, i, true); 921 } else { 922 t7xx_dpmaif_ul_arb_en(hw_info, i, false); 923 } 924 } 925} 926 927static int t7xx_dpmaif_hw_init_done(struct dpmaif_hw_info *hw_info) 928{ 929 u32 ap_cfg; 930 int ret; 931 932 ap_cfg = ioread32(hw_info->pcie_base + DPMAIF_AP_OVERWRITE_CFG); 933 ap_cfg |= DPMAIF_SRAM_SYNC; 934 iowrite32(ap_cfg, hw_info->pcie_base + DPMAIF_AP_OVERWRITE_CFG); 935 936 ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_AP_OVERWRITE_CFG, 937 ap_cfg, !(ap_cfg & DPMAIF_SRAM_SYNC), 0, 938 DPMAIF_CHECK_TIMEOUT_US); 939 if (ret) 940 return ret; 941 942 iowrite32(DPMAIF_UL_INIT_DONE, hw_info->pcie_base + DPMAIF_AO_UL_INIT_SET); 943 iowrite32(DPMAIF_DL_INIT_DONE, hw_info->pcie_base + DPMAIF_AO_DL_INIT_SET); 944 return 0; 945} 946 947static bool t7xx_dpmaif_dl_idle_check(struct dpmaif_hw_info *hw_info) 948{ 949 u32 dpmaif_dl_is_busy = ioread32(hw_info->pcie_base + DPMAIF_DL_CHK_BUSY); 950 951 return !(dpmaif_dl_is_busy & DPMAIF_DL_IDLE_STS); 952} 953 954static void t7xx_dpmaif_ul_all_q_en(struct dpmaif_hw_info *hw_info, bool enable) 955{ 956 u32 ul_arb_en = ioread32(hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0); 957 958 if (enable) 959 ul_arb_en |= DPMAIF_UL_ALL_QUE_ARB_EN; 960 else 961 ul_arb_en &= ~DPMAIF_UL_ALL_QUE_ARB_EN; 962 963 iowrite32(ul_arb_en, hw_info->pcie_base + DPMAIF_AO_UL_CHNL_ARB0); 964} 965 966static bool t7xx_dpmaif_ul_idle_check(struct dpmaif_hw_info *hw_info) 967{ 968 u32 dpmaif_ul_is_busy = ioread32(hw_info->pcie_base + DPMAIF_UL_CHK_BUSY); 969 970 return !(dpmaif_ul_is_busy & DPMAIF_UL_IDLE_STS); 971} 972 973void t7xx_dpmaif_ul_update_hw_drb_cnt(struct dpmaif_hw_info *hw_info, unsigned int q_num, 974 unsigned int drb_entry_cnt) 975{ 976 u32 ul_update, value; 977 int err; 978 979 ul_update = drb_entry_cnt & DPMAIF_UL_ADD_COUNT_MASK; 980 ul_update |= DPMAIF_UL_ADD_UPDATE; 981 982 err = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_ULQ_ADD_DESC_CH_n(q_num), 983 value, !(value & DPMAIF_UL_ADD_NOT_READY), 0, 984 DPMAIF_CHECK_TIMEOUT_US); 985 if (err) { 986 dev_err(hw_info->dev, "UL add is not ready\n"); 987 return; 988 } 989 990 iowrite32(ul_update, hw_info->pcie_base + DPMAIF_ULQ_ADD_DESC_CH_n(q_num)); 991 992 err = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_ULQ_ADD_DESC_CH_n(q_num), 993 value, !(value & DPMAIF_UL_ADD_NOT_READY), 0, 994 DPMAIF_CHECK_TIMEOUT_US); 995 if (err) 996 dev_err(hw_info->dev, "Timeout updating UL add\n"); 997} 998 999unsigned int t7xx_dpmaif_ul_get_rd_idx(struct dpmaif_hw_info *hw_info, unsigned int q_num) 1000{ 1001 unsigned int value = ioread32(hw_info->pcie_base + DPMAIF_ULQ_STA0_n(q_num)); 1002 1003 return FIELD_GET(DPMAIF_UL_DRB_RIDX_MSK, value) / DPMAIF_UL_DRB_SIZE_WORD; 1004} 1005 1006int t7xx_dpmaif_dlq_add_pit_remain_cnt(struct dpmaif_hw_info *hw_info, unsigned int dlq_pit_idx, 1007 unsigned int pit_remain_cnt) 1008{ 1009 u32 dl_update, value; 1010 int ret; 1011 1012 dl_update = pit_remain_cnt & DPMAIF_PIT_REM_CNT_MSK; 1013 dl_update |= DPMAIF_DL_ADD_UPDATE | (dlq_pit_idx << DPMAIF_ADD_DLQ_PIT_CHAN_OFS); 1014 1015 ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_DLQPIT_ADD, 1016 value, !(value & DPMAIF_DL_ADD_NOT_READY), 0, 1017 DPMAIF_CHECK_TIMEOUT_US); 1018 if (ret) { 1019 dev_err(hw_info->dev, "Data plane modem is not ready to add dlq\n"); 1020 return ret; 1021 } 1022 1023 iowrite32(dl_update, hw_info->pcie_base + DPMAIF_DL_DLQPIT_ADD); 1024 1025 ret = ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_DLQPIT_ADD, 1026 value, !(value & DPMAIF_DL_ADD_NOT_READY), 0, 1027 DPMAIF_CHECK_TIMEOUT_US); 1028 if (ret) { 1029 dev_err(hw_info->dev, "Data plane modem add dlq failed\n"); 1030 return ret; 1031 } 1032 1033 return 0; 1034} 1035 1036unsigned int t7xx_dpmaif_dl_dlq_pit_get_wr_idx(struct dpmaif_hw_info *hw_info, 1037 unsigned int dlq_pit_idx) 1038{ 1039 u32 value; 1040 1041 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_DLQ_WR_IDX + 1042 dlq_pit_idx * DLQ_PIT_IDX_SIZE); 1043 return value & DPMAIF_DL_RD_WR_IDX_MSK; 1044} 1045 1046static int t7xx_dl_add_timedout(struct dpmaif_hw_info *hw_info) 1047{ 1048 u32 value; 1049 1050 return ioread32_poll_timeout_atomic(hw_info->pcie_base + DPMAIF_DL_BAT_ADD, 1051 value, !(value & DPMAIF_DL_ADD_NOT_READY), 0, 1052 DPMAIF_CHECK_TIMEOUT_US); 1053} 1054 1055int t7xx_dpmaif_dl_snd_hw_bat_cnt(struct dpmaif_hw_info *hw_info, unsigned int bat_entry_cnt) 1056{ 1057 unsigned int value; 1058 1059 if (t7xx_dl_add_timedout(hw_info)) { 1060 dev_err(hw_info->dev, "DL add BAT not ready\n"); 1061 return -EBUSY; 1062 } 1063 1064 value = bat_entry_cnt & DPMAIF_DL_ADD_COUNT_MASK; 1065 value |= DPMAIF_DL_ADD_UPDATE; 1066 iowrite32(value, hw_info->pcie_base + DPMAIF_DL_BAT_ADD); 1067 1068 if (t7xx_dl_add_timedout(hw_info)) { 1069 dev_err(hw_info->dev, "DL add BAT timeout\n"); 1070 return -EBUSY; 1071 } 1072 1073 return 0; 1074} 1075 1076unsigned int t7xx_dpmaif_dl_get_bat_rd_idx(struct dpmaif_hw_info *hw_info, unsigned int q_num) 1077{ 1078 u32 value; 1079 1080 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_BAT_RD_IDX); 1081 return value & DPMAIF_DL_RD_WR_IDX_MSK; 1082} 1083 1084unsigned int t7xx_dpmaif_dl_get_bat_wr_idx(struct dpmaif_hw_info *hw_info, unsigned int q_num) 1085{ 1086 u32 value; 1087 1088 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_BAT_WR_IDX); 1089 return value & DPMAIF_DL_RD_WR_IDX_MSK; 1090} 1091 1092int t7xx_dpmaif_dl_snd_hw_frg_cnt(struct dpmaif_hw_info *hw_info, unsigned int frg_entry_cnt) 1093{ 1094 unsigned int value; 1095 1096 if (t7xx_dl_add_timedout(hw_info)) { 1097 dev_err(hw_info->dev, "Data plane modem is not ready to add frag DLQ\n"); 1098 return -EBUSY; 1099 } 1100 1101 value = frg_entry_cnt & DPMAIF_DL_ADD_COUNT_MASK; 1102 value |= DPMAIF_DL_FRG_ADD_UPDATE | DPMAIF_DL_ADD_UPDATE; 1103 iowrite32(value, hw_info->pcie_base + DPMAIF_DL_BAT_ADD); 1104 1105 if (t7xx_dl_add_timedout(hw_info)) { 1106 dev_err(hw_info->dev, "Data plane modem add frag DLQ failed"); 1107 return -EBUSY; 1108 } 1109 1110 return 0; 1111} 1112 1113unsigned int t7xx_dpmaif_dl_get_frg_rd_idx(struct dpmaif_hw_info *hw_info, unsigned int q_num) 1114{ 1115 u32 value; 1116 1117 value = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_FRGBAT_RD_IDX); 1118 return value & DPMAIF_DL_RD_WR_IDX_MSK; 1119} 1120 1121static void t7xx_dpmaif_set_queue_property(struct dpmaif_hw_info *hw_info, 1122 struct dpmaif_hw_params *init_para) 1123{ 1124 struct dpmaif_dl *dl_que; 1125 struct dpmaif_ul *ul_que; 1126 int i; 1127 1128 for (i = 0; i < DPMAIF_RXQ_NUM; i++) { 1129 dl_que = &hw_info->dl_que[i]; 1130 dl_que->bat_base = init_para->pkt_bat_base_addr[i]; 1131 dl_que->bat_size_cnt = init_para->pkt_bat_size_cnt[i]; 1132 dl_que->pit_base = init_para->pit_base_addr[i]; 1133 dl_que->pit_size_cnt = init_para->pit_size_cnt[i]; 1134 dl_que->frg_base = init_para->frg_bat_base_addr[i]; 1135 dl_que->frg_size_cnt = init_para->frg_bat_size_cnt[i]; 1136 dl_que->que_started = true; 1137 } 1138 1139 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 1140 ul_que = &hw_info->ul_que[i]; 1141 ul_que->drb_base = init_para->drb_base_addr[i]; 1142 ul_que->drb_size_cnt = init_para->drb_size_cnt[i]; 1143 ul_que->que_started = true; 1144 } 1145} 1146 1147/** 1148 * t7xx_dpmaif_hw_stop_all_txq() - Stop all TX queues. 1149 * @hw_info: Pointer to struct hw_info. 1150 * 1151 * Disable HW UL queues. Checks busy UL queues to go to idle 1152 * with an attempt count of 1000000. 1153 * 1154 * Return: 1155 * * 0 - Success 1156 * * -ETIMEDOUT - Timed out checking busy queues 1157 */ 1158int t7xx_dpmaif_hw_stop_all_txq(struct dpmaif_hw_info *hw_info) 1159{ 1160 int count = 0; 1161 1162 t7xx_dpmaif_ul_all_q_en(hw_info, false); 1163 while (t7xx_dpmaif_ul_idle_check(hw_info)) { 1164 if (++count >= DPMAIF_MAX_CHECK_COUNT) { 1165 dev_err(hw_info->dev, "Failed to stop TX, status: 0x%x\n", 1166 ioread32(hw_info->pcie_base + DPMAIF_UL_CHK_BUSY)); 1167 return -ETIMEDOUT; 1168 } 1169 } 1170 1171 return 0; 1172} 1173 1174/** 1175 * t7xx_dpmaif_hw_stop_all_rxq() - Stop all RX queues. 1176 * @hw_info: Pointer to struct hw_info. 1177 * 1178 * Disable HW DL queue. Checks busy UL queues to go to idle 1179 * with an attempt count of 1000000. 1180 * Check that HW PIT write index equals read index with the same 1181 * attempt count. 1182 * 1183 * Return: 1184 * * 0 - Success. 1185 * * -ETIMEDOUT - Timed out checking busy queues. 1186 */ 1187int t7xx_dpmaif_hw_stop_all_rxq(struct dpmaif_hw_info *hw_info) 1188{ 1189 unsigned int wr_idx, rd_idx; 1190 int count = 0; 1191 1192 t7xx_dpmaif_dl_all_q_en(hw_info, false); 1193 while (t7xx_dpmaif_dl_idle_check(hw_info)) { 1194 if (++count >= DPMAIF_MAX_CHECK_COUNT) { 1195 dev_err(hw_info->dev, "Failed to stop RX, status: 0x%x\n", 1196 ioread32(hw_info->pcie_base + DPMAIF_DL_CHK_BUSY)); 1197 return -ETIMEDOUT; 1198 } 1199 } 1200 1201 /* Check middle PIT sync done */ 1202 count = 0; 1203 do { 1204 wr_idx = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PIT_WR_IDX); 1205 wr_idx &= DPMAIF_DL_RD_WR_IDX_MSK; 1206 rd_idx = ioread32(hw_info->pcie_base + DPMAIF_AO_DL_PIT_RD_IDX); 1207 rd_idx &= DPMAIF_DL_RD_WR_IDX_MSK; 1208 1209 if (wr_idx == rd_idx) 1210 return 0; 1211 } while (++count < DPMAIF_MAX_CHECK_COUNT); 1212 1213 dev_err(hw_info->dev, "Check middle PIT sync fail\n"); 1214 return -ETIMEDOUT; 1215} 1216 1217void t7xx_dpmaif_start_hw(struct dpmaif_hw_info *hw_info) 1218{ 1219 t7xx_dpmaif_ul_all_q_en(hw_info, true); 1220 t7xx_dpmaif_dl_all_q_en(hw_info, true); 1221} 1222 1223/** 1224 * t7xx_dpmaif_hw_init() - Initialize HW data path API. 1225 * @hw_info: Pointer to struct hw_info. 1226 * @init_param: Pointer to struct dpmaif_hw_params. 1227 * 1228 * Configures port mode, clock config, HW interrupt initialization, and HW queue. 1229 * 1230 * Return: 1231 * * 0 - Success. 1232 * * -ERROR - Error code from failure sub-initializations. 1233 */ 1234int t7xx_dpmaif_hw_init(struct dpmaif_hw_info *hw_info, struct dpmaif_hw_params *init_param) 1235{ 1236 int ret; 1237 1238 ret = t7xx_dpmaif_hw_config(hw_info); 1239 if (ret) { 1240 dev_err(hw_info->dev, "DPMAIF HW config failed\n"); 1241 return ret; 1242 } 1243 1244 ret = t7xx_dpmaif_init_intr(hw_info); 1245 if (ret) { 1246 dev_err(hw_info->dev, "DPMAIF HW interrupts init failed\n"); 1247 return ret; 1248 } 1249 1250 t7xx_dpmaif_set_queue_property(hw_info, init_param); 1251 t7xx_dpmaif_pcie_dpmaif_sign(hw_info); 1252 t7xx_dpmaif_dl_performance(hw_info); 1253 1254 ret = t7xx_dpmaif_config_dlq_hw(hw_info); 1255 if (ret) { 1256 dev_err(hw_info->dev, "DPMAIF HW dlq config failed\n"); 1257 return ret; 1258 } 1259 1260 t7xx_dpmaif_config_ulq_hw(hw_info); 1261 1262 ret = t7xx_dpmaif_hw_init_done(hw_info); 1263 if (ret) 1264 dev_err(hw_info->dev, "DPMAIF HW queue init failed\n"); 1265 1266 return ret; 1267} 1268 1269bool t7xx_dpmaif_ul_clr_done(struct dpmaif_hw_info *hw_info, unsigned int qno) 1270{ 1271 u32 intr_status; 1272 1273 intr_status = ioread32(hw_info->pcie_base + DPMAIF_AP_L2TISAR0); 1274 intr_status &= BIT(DP_UL_INT_DONE_OFFSET + qno); 1275 if (intr_status) { 1276 iowrite32(intr_status, hw_info->pcie_base + DPMAIF_AP_L2TISAR0); 1277 return true; 1278 } 1279 1280 return false; 1281} 1282