1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2/* 3 * Copyright (C) 2003-2014, 2018-2022 Intel Corporation 4 * Copyright (C) 2015-2016 Intel Deutschland GmbH 5 */ 6#include <linux/delay.h> 7#include <linux/device.h> 8#include <linux/export.h> 9 10#include "iwl-drv.h" 11#include "iwl-io.h" 12#include "iwl-csr.h" 13#include "iwl-debug.h" 14#include "iwl-prph.h" 15#include "iwl-fh.h" 16 17void iwl_write8(struct iwl_trans *trans, u32 ofs, u8 val) 18{ 19 trace_iwlwifi_dev_iowrite8(trans->dev, ofs, val); 20 iwl_trans_write8(trans, ofs, val); 21} 22IWL_EXPORT_SYMBOL(iwl_write8); 23 24void iwl_write32(struct iwl_trans *trans, u32 ofs, u32 val) 25{ 26 trace_iwlwifi_dev_iowrite32(trans->dev, ofs, val); 27 iwl_trans_write32(trans, ofs, val); 28} 29IWL_EXPORT_SYMBOL(iwl_write32); 30 31void iwl_write64(struct iwl_trans *trans, u64 ofs, u64 val) 32{ 33 trace_iwlwifi_dev_iowrite64(trans->dev, ofs, val); 34 iwl_trans_write32(trans, ofs, lower_32_bits(val)); 35 iwl_trans_write32(trans, ofs + 4, upper_32_bits(val)); 36} 37IWL_EXPORT_SYMBOL(iwl_write64); 38 39u32 iwl_read32(struct iwl_trans *trans, u32 ofs) 40{ 41 u32 val = iwl_trans_read32(trans, ofs); 42 43 trace_iwlwifi_dev_ioread32(trans->dev, ofs, val); 44 return val; 45} 46IWL_EXPORT_SYMBOL(iwl_read32); 47 48#define IWL_POLL_INTERVAL 10 /* microseconds */ 49 50int iwl_poll_bit(struct iwl_trans *trans, u32 addr, 51 u32 bits, u32 mask, int timeout) 52{ 53 int t = 0; 54 55 do { 56 if ((iwl_read32(trans, addr) & mask) == (bits & mask)) 57 return t; 58 udelay(IWL_POLL_INTERVAL); 59 t += IWL_POLL_INTERVAL; 60 } while (t < timeout); 61 62 return -ETIMEDOUT; 63} 64IWL_EXPORT_SYMBOL(iwl_poll_bit); 65 66u32 iwl_read_direct32(struct iwl_trans *trans, u32 reg) 67{ 68 if (iwl_trans_grab_nic_access(trans)) { 69 u32 value = iwl_read32(trans, reg); 70 71 iwl_trans_release_nic_access(trans); 72 return value; 73 } 74 75 /* return as if we have a HW timeout/failure */ 76 return 0x5a5a5a5a; 77} 78IWL_EXPORT_SYMBOL(iwl_read_direct32); 79 80void iwl_write_direct32(struct iwl_trans *trans, u32 reg, u32 value) 81{ 82 if (iwl_trans_grab_nic_access(trans)) { 83 iwl_write32(trans, reg, value); 84 iwl_trans_release_nic_access(trans); 85 } 86} 87IWL_EXPORT_SYMBOL(iwl_write_direct32); 88 89void iwl_write_direct64(struct iwl_trans *trans, u64 reg, u64 value) 90{ 91 if (iwl_trans_grab_nic_access(trans)) { 92 iwl_write64(trans, reg, value); 93 iwl_trans_release_nic_access(trans); 94 } 95} 96IWL_EXPORT_SYMBOL(iwl_write_direct64); 97 98int iwl_poll_direct_bit(struct iwl_trans *trans, u32 addr, u32 mask, 99 int timeout) 100{ 101 int t = 0; 102 103 do { 104 if ((iwl_read_direct32(trans, addr) & mask) == mask) 105 return t; 106 udelay(IWL_POLL_INTERVAL); 107 t += IWL_POLL_INTERVAL; 108 } while (t < timeout); 109 110 return -ETIMEDOUT; 111} 112IWL_EXPORT_SYMBOL(iwl_poll_direct_bit); 113 114u32 iwl_read_prph_no_grab(struct iwl_trans *trans, u32 ofs) 115{ 116 u32 val = iwl_trans_read_prph(trans, ofs); 117 trace_iwlwifi_dev_ioread_prph32(trans->dev, ofs, val); 118 return val; 119} 120IWL_EXPORT_SYMBOL(iwl_read_prph_no_grab); 121 122void iwl_write_prph_no_grab(struct iwl_trans *trans, u32 ofs, u32 val) 123{ 124 trace_iwlwifi_dev_iowrite_prph32(trans->dev, ofs, val); 125 iwl_trans_write_prph(trans, ofs, val); 126} 127IWL_EXPORT_SYMBOL(iwl_write_prph_no_grab); 128 129void iwl_write_prph64_no_grab(struct iwl_trans *trans, u64 ofs, u64 val) 130{ 131 trace_iwlwifi_dev_iowrite_prph64(trans->dev, ofs, val); 132 iwl_write_prph_no_grab(trans, ofs, val & 0xffffffff); 133 iwl_write_prph_no_grab(trans, ofs + 4, val >> 32); 134} 135IWL_EXPORT_SYMBOL(iwl_write_prph64_no_grab); 136 137u32 iwl_read_prph(struct iwl_trans *trans, u32 ofs) 138{ 139 if (iwl_trans_grab_nic_access(trans)) { 140 u32 val = iwl_read_prph_no_grab(trans, ofs); 141 142 iwl_trans_release_nic_access(trans); 143 144 return val; 145 } 146 147 /* return as if we have a HW timeout/failure */ 148 return 0x5a5a5a5a; 149} 150IWL_EXPORT_SYMBOL(iwl_read_prph); 151 152void iwl_write_prph_delay(struct iwl_trans *trans, u32 ofs, u32 val, u32 delay_ms) 153{ 154 if (iwl_trans_grab_nic_access(trans)) { 155 mdelay(delay_ms); 156 iwl_write_prph_no_grab(trans, ofs, val); 157 iwl_trans_release_nic_access(trans); 158 } 159} 160IWL_EXPORT_SYMBOL(iwl_write_prph_delay); 161 162int iwl_poll_prph_bit(struct iwl_trans *trans, u32 addr, 163 u32 bits, u32 mask, int timeout) 164{ 165 int t = 0; 166 167 do { 168 if ((iwl_read_prph(trans, addr) & mask) == (bits & mask)) 169 return t; 170 udelay(IWL_POLL_INTERVAL); 171 t += IWL_POLL_INTERVAL; 172 } while (t < timeout); 173 174 return -ETIMEDOUT; 175} 176 177void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask) 178{ 179 if (iwl_trans_grab_nic_access(trans)) { 180 iwl_write_prph_no_grab(trans, ofs, 181 iwl_read_prph_no_grab(trans, ofs) | 182 mask); 183 iwl_trans_release_nic_access(trans); 184 } 185} 186IWL_EXPORT_SYMBOL(iwl_set_bits_prph); 187 188void iwl_set_bits_mask_prph(struct iwl_trans *trans, u32 ofs, 189 u32 bits, u32 mask) 190{ 191 if (iwl_trans_grab_nic_access(trans)) { 192 iwl_write_prph_no_grab(trans, ofs, 193 (iwl_read_prph_no_grab(trans, ofs) & 194 mask) | bits); 195 iwl_trans_release_nic_access(trans); 196 } 197} 198IWL_EXPORT_SYMBOL(iwl_set_bits_mask_prph); 199 200void iwl_clear_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask) 201{ 202 u32 val; 203 204 if (iwl_trans_grab_nic_access(trans)) { 205 val = iwl_read_prph_no_grab(trans, ofs); 206 iwl_write_prph_no_grab(trans, ofs, (val & ~mask)); 207 iwl_trans_release_nic_access(trans); 208 } 209} 210IWL_EXPORT_SYMBOL(iwl_clear_bits_prph); 211 212void iwl_force_nmi(struct iwl_trans *trans) 213{ 214 if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_9000) 215 iwl_write_prph_delay(trans, DEVICE_SET_NMI_REG, 216 DEVICE_SET_NMI_VAL_DRV, 1); 217 else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210) 218 iwl_write_umac_prph(trans, UREG_NIC_SET_NMI_DRIVER, 219 UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER); 220 else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ) 221 iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6, 222 UREG_DOORBELL_TO_ISR6_NMI_BIT); 223 else 224 iwl_write32(trans, CSR_DOORBELL_VECTOR, 225 UREG_DOORBELL_TO_ISR6_NMI_BIT); 226} 227IWL_EXPORT_SYMBOL(iwl_force_nmi); 228 229static const char *get_rfh_string(int cmd) 230{ 231#define IWL_CMD(x) case x: return #x 232#define IWL_CMD_MQ(arg, reg, q) { if (arg == reg(q)) return #reg; } 233 234 int i; 235 236 for (i = 0; i < IWL_MAX_RX_HW_QUEUES; i++) { 237 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_BA_LSB, i); 238 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_WIDX, i); 239 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_RIDX, i); 240 IWL_CMD_MQ(cmd, RFH_Q_URBD_STTS_WPTR_LSB, i); 241 } 242 243 switch (cmd) { 244 IWL_CMD(RFH_RXF_DMA_CFG); 245 IWL_CMD(RFH_GEN_CFG); 246 IWL_CMD(RFH_GEN_STATUS); 247 IWL_CMD(FH_TSSR_TX_STATUS_REG); 248 IWL_CMD(FH_TSSR_TX_ERROR_REG); 249 default: 250 return "UNKNOWN"; 251 } 252#undef IWL_CMD_MQ 253} 254 255struct reg { 256 u32 addr; 257 bool is64; 258}; 259 260static int iwl_dump_rfh(struct iwl_trans *trans, char **buf) 261{ 262 int i, q; 263 int num_q = trans->num_rx_queues; 264 static const u32 rfh_tbl[] = { 265 RFH_RXF_DMA_CFG, 266 RFH_GEN_CFG, 267 RFH_GEN_STATUS, 268 FH_TSSR_TX_STATUS_REG, 269 FH_TSSR_TX_ERROR_REG, 270 }; 271 static const struct reg rfh_mq_tbl[] = { 272 { RFH_Q0_FRBDCB_BA_LSB, true }, 273 { RFH_Q0_FRBDCB_WIDX, false }, 274 { RFH_Q0_FRBDCB_RIDX, false }, 275 { RFH_Q0_URBD_STTS_WPTR_LSB, true }, 276 }; 277 278#ifdef CONFIG_IWLWIFI_DEBUGFS 279 if (buf) { 280 int pos = 0; 281 /* 282 * Register (up to 34 for name + 8 blank/q for MQ): 40 chars 283 * Colon + space: 2 characters 284 * 0X%08x: 10 characters 285 * New line: 1 character 286 * Total of 53 characters 287 */ 288 size_t bufsz = ARRAY_SIZE(rfh_tbl) * 53 + 289 ARRAY_SIZE(rfh_mq_tbl) * 53 * num_q + 40; 290 291 *buf = kmalloc(bufsz, GFP_KERNEL); 292 if (!*buf) 293 return -ENOMEM; 294 295 pos += scnprintf(*buf + pos, bufsz - pos, 296 "RFH register values:\n"); 297 298 for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++) 299 pos += scnprintf(*buf + pos, bufsz - pos, 300 "%40s: 0X%08x\n", 301 get_rfh_string(rfh_tbl[i]), 302 iwl_read_prph(trans, rfh_tbl[i])); 303 304 for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++) 305 for (q = 0; q < num_q; q++) { 306 u32 addr = rfh_mq_tbl[i].addr; 307 308 addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4); 309 pos += scnprintf(*buf + pos, bufsz - pos, 310 "%34s(q %2d): 0X%08x\n", 311 get_rfh_string(addr), q, 312 iwl_read_prph(trans, addr)); 313 } 314 315 return pos; 316 } 317#endif 318 319 IWL_ERR(trans, "RFH register values:\n"); 320 for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++) 321 IWL_ERR(trans, " %34s: 0X%08x\n", 322 get_rfh_string(rfh_tbl[i]), 323 iwl_read_prph(trans, rfh_tbl[i])); 324 325 for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++) 326 for (q = 0; q < num_q; q++) { 327 u32 addr = rfh_mq_tbl[i].addr; 328 329 addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4); 330 IWL_ERR(trans, " %34s(q %d): 0X%08x\n", 331 get_rfh_string(addr), q, 332 iwl_read_prph(trans, addr)); 333 } 334 335 return 0; 336} 337 338static const char *get_fh_string(int cmd) 339{ 340 switch (cmd) { 341 IWL_CMD(FH_RSCSR_CHNL0_STTS_WPTR_REG); 342 IWL_CMD(FH_RSCSR_CHNL0_RBDCB_BASE_REG); 343 IWL_CMD(FH_RSCSR_CHNL0_WPTR); 344 IWL_CMD(FH_MEM_RCSR_CHNL0_CONFIG_REG); 345 IWL_CMD(FH_MEM_RSSR_SHARED_CTRL_REG); 346 IWL_CMD(FH_MEM_RSSR_RX_STATUS_REG); 347 IWL_CMD(FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV); 348 IWL_CMD(FH_TSSR_TX_STATUS_REG); 349 IWL_CMD(FH_TSSR_TX_ERROR_REG); 350 default: 351 return "UNKNOWN"; 352 } 353#undef IWL_CMD 354} 355 356int iwl_dump_fh(struct iwl_trans *trans, char **buf) 357{ 358 int i; 359 static const u32 fh_tbl[] = { 360 FH_RSCSR_CHNL0_STTS_WPTR_REG, 361 FH_RSCSR_CHNL0_RBDCB_BASE_REG, 362 FH_RSCSR_CHNL0_WPTR, 363 FH_MEM_RCSR_CHNL0_CONFIG_REG, 364 FH_MEM_RSSR_SHARED_CTRL_REG, 365 FH_MEM_RSSR_RX_STATUS_REG, 366 FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV, 367 FH_TSSR_TX_STATUS_REG, 368 FH_TSSR_TX_ERROR_REG 369 }; 370 371 if (trans->trans_cfg->mq_rx_supported) 372 return iwl_dump_rfh(trans, buf); 373 374#ifdef CONFIG_IWLWIFI_DEBUGFS 375 if (buf) { 376 int pos = 0; 377 size_t bufsz = ARRAY_SIZE(fh_tbl) * 48 + 40; 378 379 *buf = kmalloc(bufsz, GFP_KERNEL); 380 if (!*buf) 381 return -ENOMEM; 382 383 pos += scnprintf(*buf + pos, bufsz - pos, 384 "FH register values:\n"); 385 386 for (i = 0; i < ARRAY_SIZE(fh_tbl); i++) 387 pos += scnprintf(*buf + pos, bufsz - pos, 388 " %34s: 0X%08x\n", 389 get_fh_string(fh_tbl[i]), 390 iwl_read_direct32(trans, fh_tbl[i])); 391 392 return pos; 393 } 394#endif 395 396 IWL_ERR(trans, "FH register values:\n"); 397 for (i = 0; i < ARRAY_SIZE(fh_tbl); i++) 398 IWL_ERR(trans, " %34s: 0X%08x\n", 399 get_fh_string(fh_tbl[i]), 400 iwl_read_direct32(trans, fh_tbl[i])); 401 402 return 0; 403} 404 405#define IWL_HOST_MON_BLOCK_PEMON 0x00 406#define IWL_HOST_MON_BLOCK_HIPM 0x22 407 408#define IWL_HOST_MON_BLOCK_PEMON_VEC0 0x00 409#define IWL_HOST_MON_BLOCK_PEMON_VEC1 0x01 410#define IWL_HOST_MON_BLOCK_PEMON_WFPM 0x06 411 412static void iwl_dump_host_monitor_block(struct iwl_trans *trans, 413 u32 block, u32 vec, u32 iter) 414{ 415 int i; 416 417 IWL_ERR(trans, "Host monitor block 0x%x vector 0x%x\n", block, vec); 418 iwl_write32(trans, CSR_MONITOR_CFG_REG, (block << 8) | vec); 419 for (i = 0; i < iter; i++) 420 IWL_ERR(trans, " value [iter %d]: 0x%08x\n", 421 i, iwl_read32(trans, CSR_MONITOR_STATUS_REG)); 422} 423 424static void iwl_dump_host_monitor(struct iwl_trans *trans) 425{ 426 switch (trans->trans_cfg->device_family) { 427 case IWL_DEVICE_FAMILY_22000: 428 case IWL_DEVICE_FAMILY_AX210: 429 IWL_ERR(trans, "CSR_RESET = 0x%x\n", 430 iwl_read32(trans, CSR_RESET)); 431 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON, 432 IWL_HOST_MON_BLOCK_PEMON_VEC0, 15); 433 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON, 434 IWL_HOST_MON_BLOCK_PEMON_VEC1, 15); 435 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON, 436 IWL_HOST_MON_BLOCK_PEMON_WFPM, 15); 437 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_HIPM, 438 IWL_HOST_MON_BLOCK_PEMON_VEC0, 1); 439 break; 440 default: 441 /* not supported yet */ 442 return; 443 } 444} 445 446int iwl_finish_nic_init(struct iwl_trans *trans) 447{ 448 const struct iwl_cfg_trans_params *cfg_trans = trans->trans_cfg; 449 u32 poll_ready; 450 int err; 451 452 if (cfg_trans->bisr_workaround) { 453 /* ensure the TOP FSM isn't still in previous reset */ 454 mdelay(2); 455 } 456 457 /* 458 * Set "initialization complete" bit to move adapter from 459 * D0U* --> D0A* (powered-up active) state. 460 */ 461 if (cfg_trans->device_family >= IWL_DEVICE_FAMILY_BZ) { 462 iwl_set_bit(trans, CSR_GP_CNTRL, 463 CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY | 464 CSR_GP_CNTRL_REG_FLAG_MAC_INIT); 465 poll_ready = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS; 466 } else { 467 iwl_set_bit(trans, CSR_GP_CNTRL, 468 CSR_GP_CNTRL_REG_FLAG_INIT_DONE); 469 poll_ready = CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY; 470 } 471 472 if (cfg_trans->device_family == IWL_DEVICE_FAMILY_8000) 473 udelay(2); 474 475 /* 476 * Wait for clock stabilization; once stabilized, access to 477 * device-internal resources is supported, e.g. iwl_write_prph() 478 * and accesses to uCode SRAM. 479 */ 480 err = iwl_poll_bit(trans, CSR_GP_CNTRL, poll_ready, poll_ready, 25000); 481 if (err < 0) { 482 IWL_DEBUG_INFO(trans, "Failed to wake NIC\n"); 483 484 iwl_dump_host_monitor(trans); 485 } 486 487 if (cfg_trans->bisr_workaround) { 488 /* ensure BISR shift has finished */ 489 udelay(200); 490 } 491 492 return err < 0 ? err : 0; 493} 494IWL_EXPORT_SYMBOL(iwl_finish_nic_init); 495 496void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr, 497 u32 sw_err_bit) 498{ 499 unsigned long timeout = jiffies + IWL_TRANS_NMI_TIMEOUT; 500 bool interrupts_enabled = test_bit(STATUS_INT_ENABLED, &trans->status); 501 502 /* if the interrupts were already disabled, there is no point in 503 * calling iwl_disable_interrupts 504 */ 505 if (interrupts_enabled) 506 iwl_trans_interrupts(trans, false); 507 508 iwl_force_nmi(trans); 509 while (time_after(timeout, jiffies)) { 510 u32 inta_hw = iwl_read32(trans, inta_addr); 511 512 /* Error detected by uCode */ 513 if (inta_hw & sw_err_bit) { 514 /* Clear causes register */ 515 iwl_write32(trans, inta_addr, inta_hw & sw_err_bit); 516 break; 517 } 518 519 mdelay(1); 520 } 521 522 /* enable interrupts only if there were already enabled before this 523 * function to avoid a case were the driver enable interrupts before 524 * proper configurations were made 525 */ 526 if (interrupts_enabled) 527 iwl_trans_interrupts(trans, true); 528 529 iwl_trans_fw_error(trans, false); 530} 531