1219820Sjeff// SPDX-License-Identifier: GPL-2.0+ 2219820Sjeff/* 3219820Sjeff * Rockchip AXI PCIe host controller driver 4219820Sjeff * 5219820Sjeff * Copyright (c) 2016 Rockchip, Inc. 6219820Sjeff * 7219820Sjeff * Author: Shawn Lin <shawn.lin@rock-chips.com> 8219820Sjeff * Wenrui Li <wenrui.li@rock-chips.com> 9219820Sjeff * 10219820Sjeff * Bits taken from Synopsys DesignWare Host controller driver and 11219820Sjeff * ARM PCI Host generic driver. 12219820Sjeff */ 13219820Sjeff 14219820Sjeff#include <linux/bitrev.h> 15219820Sjeff#include <linux/clk.h> 16219820Sjeff#include <linux/delay.h> 17219820Sjeff#include <linux/gpio/consumer.h> 18219820Sjeff#include <linux/init.h> 19219820Sjeff#include <linux/interrupt.h> 20219820Sjeff#include <linux/iopoll.h> 21219820Sjeff#include <linux/irq.h> 22219820Sjeff#include <linux/irqchip/chained_irq.h> 23219820Sjeff#include <linux/irqdomain.h> 24219820Sjeff#include <linux/kernel.h> 25219820Sjeff#include <linux/mfd/syscon.h> 26219820Sjeff#include <linux/module.h> 27219820Sjeff#include <linux/of.h> 28219820Sjeff#include <linux/of_pci.h> 29219820Sjeff#include <linux/pci.h> 30219820Sjeff#include <linux/pci_ids.h> 31219820Sjeff#include <linux/phy/phy.h> 32219820Sjeff#include <linux/platform_device.h> 33219820Sjeff#include <linux/reset.h> 34219820Sjeff#include <linux/regmap.h> 35219820Sjeff 36219820Sjeff#include "../pci.h" 37219820Sjeff#include "pcie-rockchip.h" 38219820Sjeff 39219820Sjeffstatic void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) 40219820Sjeff{ 41219820Sjeff u32 status; 42219820Sjeff 43219820Sjeff status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 44219820Sjeff status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); 45219820Sjeff rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 46219820Sjeff} 47219820Sjeff 48219820Sjeffstatic void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) 49219820Sjeff{ 50219820Sjeff u32 status; 51219820Sjeff 52219820Sjeff status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 53219820Sjeff status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; 54219820Sjeff rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 55219820Sjeff} 56219820Sjeff 57219820Sjeffstatic void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) 58219820Sjeff{ 59219820Sjeff u32 val; 60219820Sjeff 61219820Sjeff /* Update Tx credit maximum update interval */ 62219820Sjeff val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1); 63219820Sjeff val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK; 64219820Sjeff val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */ 65219820Sjeff rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1); 66219820Sjeff} 67219820Sjeff 68219820Sjeffstatic int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip, 69219820Sjeff struct pci_bus *bus, int dev) 70219820Sjeff{ 71219820Sjeff /* 72219820Sjeff * Access only one slot on each root port. 73219820Sjeff * Do not read more than one device on the bus directly attached 74219820Sjeff * to RC's downstream side. 75219820Sjeff */ 76219820Sjeff if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent)) 77219820Sjeff return dev == 0; 78219820Sjeff 79219820Sjeff return 1; 80219820Sjeff} 81219820Sjeff 82219820Sjeffstatic u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip) 83219820Sjeff{ 84219820Sjeff u32 val; 85219820Sjeff u8 map; 86219820Sjeff 87219820Sjeff if (rockchip->legacy_phy) 88219820Sjeff return GENMASK(MAX_LANE_NUM - 1, 0); 89219820Sjeff 90219820Sjeff val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP); 91219820Sjeff map = val & PCIE_CORE_LANE_MAP_MASK; 92219820Sjeff 93219820Sjeff /* The link may be using a reverse-indexed mapping. */ 94219820Sjeff if (val & PCIE_CORE_LANE_MAP_REVERSE) 95219820Sjeff map = bitrev8(map) >> 4; 96219820Sjeff 97219820Sjeff return map; 98219820Sjeff} 99219820Sjeff 100219820Sjeffstatic int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip, 101219820Sjeff int where, int size, u32 *val) 102219820Sjeff{ 103219820Sjeff void __iomem *addr; 104219820Sjeff 105219820Sjeff addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where; 106219820Sjeff 107219820Sjeff if (!IS_ALIGNED((uintptr_t)addr, size)) { 108219820Sjeff *val = 0; 109219820Sjeff return PCIBIOS_BAD_REGISTER_NUMBER; 110219820Sjeff } 111219820Sjeff 112219820Sjeff if (size == 4) { 113219820Sjeff *val = readl(addr); 114219820Sjeff } else if (size == 2) { 115219820Sjeff *val = readw(addr); 116219820Sjeff } else if (size == 1) { 117219820Sjeff *val = readb(addr); 118219820Sjeff } else { 119219820Sjeff *val = 0; 120219820Sjeff return PCIBIOS_BAD_REGISTER_NUMBER; 121219820Sjeff } 122219820Sjeff return PCIBIOS_SUCCESSFUL; 123219820Sjeff} 124219820Sjeff 125219820Sjeffstatic int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip, 126219820Sjeff int where, int size, u32 val) 127219820Sjeff{ 128219820Sjeff u32 mask, tmp, offset; 129219820Sjeff void __iomem *addr; 130219820Sjeff 131219820Sjeff offset = where & ~0x3; 132219820Sjeff addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset; 133219820Sjeff 134219820Sjeff if (size == 4) { 135219820Sjeff writel(val, addr); 136219820Sjeff return PCIBIOS_SUCCESSFUL; 137219820Sjeff } 138219820Sjeff 139219820Sjeff mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 140219820Sjeff 141219820Sjeff /* 142219820Sjeff * N.B. This read/modify/write isn't safe in general because it can 143219820Sjeff * corrupt RW1C bits in adjacent registers. But the hardware 144219820Sjeff * doesn't support smaller writes. 145219820Sjeff */ 146219820Sjeff tmp = readl(addr) & mask; 147219820Sjeff tmp |= val << ((where & 0x3) * 8); 148219820Sjeff writel(tmp, addr); 149219820Sjeff 150219820Sjeff return PCIBIOS_SUCCESSFUL; 151219820Sjeff} 152219820Sjeff 153219820Sjeffstatic int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip, 154219820Sjeff struct pci_bus *bus, u32 devfn, 155219820Sjeff int where, int size, u32 *val) 156219820Sjeff{ 157219820Sjeff void __iomem *addr; 158219820Sjeff 159 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); 160 161 if (!IS_ALIGNED((uintptr_t)addr, size)) { 162 *val = 0; 163 return PCIBIOS_BAD_REGISTER_NUMBER; 164 } 165 166 if (pci_is_root_bus(bus->parent)) 167 rockchip_pcie_cfg_configuration_accesses(rockchip, 168 AXI_WRAPPER_TYPE0_CFG); 169 else 170 rockchip_pcie_cfg_configuration_accesses(rockchip, 171 AXI_WRAPPER_TYPE1_CFG); 172 173 if (size == 4) { 174 *val = readl(addr); 175 } else if (size == 2) { 176 *val = readw(addr); 177 } else if (size == 1) { 178 *val = readb(addr); 179 } else { 180 *val = 0; 181 return PCIBIOS_BAD_REGISTER_NUMBER; 182 } 183 return PCIBIOS_SUCCESSFUL; 184} 185 186static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip, 187 struct pci_bus *bus, u32 devfn, 188 int where, int size, u32 val) 189{ 190 void __iomem *addr; 191 192 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); 193 194 if (!IS_ALIGNED((uintptr_t)addr, size)) 195 return PCIBIOS_BAD_REGISTER_NUMBER; 196 197 if (pci_is_root_bus(bus->parent)) 198 rockchip_pcie_cfg_configuration_accesses(rockchip, 199 AXI_WRAPPER_TYPE0_CFG); 200 else 201 rockchip_pcie_cfg_configuration_accesses(rockchip, 202 AXI_WRAPPER_TYPE1_CFG); 203 204 if (size == 4) 205 writel(val, addr); 206 else if (size == 2) 207 writew(val, addr); 208 else if (size == 1) 209 writeb(val, addr); 210 else 211 return PCIBIOS_BAD_REGISTER_NUMBER; 212 213 return PCIBIOS_SUCCESSFUL; 214} 215 216static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, 217 int size, u32 *val) 218{ 219 struct rockchip_pcie *rockchip = bus->sysdata; 220 221 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 222 return PCIBIOS_DEVICE_NOT_FOUND; 223 224 if (pci_is_root_bus(bus)) 225 return rockchip_pcie_rd_own_conf(rockchip, where, size, val); 226 227 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, 228 val); 229} 230 231static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn, 232 int where, int size, u32 val) 233{ 234 struct rockchip_pcie *rockchip = bus->sysdata; 235 236 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 237 return PCIBIOS_DEVICE_NOT_FOUND; 238 239 if (pci_is_root_bus(bus)) 240 return rockchip_pcie_wr_own_conf(rockchip, where, size, val); 241 242 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, 243 val); 244} 245 246static struct pci_ops rockchip_pcie_ops = { 247 .read = rockchip_pcie_rd_conf, 248 .write = rockchip_pcie_wr_conf, 249}; 250 251static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) 252{ 253 int curr; 254 u32 status, scale, power; 255 256 if (IS_ERR(rockchip->vpcie3v3)) 257 return; 258 259 /* 260 * Set RC's captured slot power limit and scale if 261 * vpcie3v3 available. The default values are both zero 262 * which means the software should set these two according 263 * to the actual power supply. 264 */ 265 curr = regulator_get_current_limit(rockchip->vpcie3v3); 266 if (curr <= 0) 267 return; 268 269 scale = 3; /* 0.001x */ 270 curr = curr / 1000; /* convert to mA */ 271 power = (curr * 3300) / 1000; /* milliwatt */ 272 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) { 273 if (!scale) { 274 dev_warn(rockchip->dev, "invalid power supply\n"); 275 return; 276 } 277 scale--; 278 power = power / 10; 279 } 280 281 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR); 282 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) | 283 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT); 284 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR); 285} 286 287/** 288 * rockchip_pcie_host_init_port - Initialize hardware 289 * @rockchip: PCIe port information 290 */ 291static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) 292{ 293 struct device *dev = rockchip->dev; 294 int err, i = MAX_LANE_NUM; 295 u32 status; 296 297 gpiod_set_value_cansleep(rockchip->ep_gpio, 0); 298 299 err = rockchip_pcie_init_port(rockchip); 300 if (err) 301 return err; 302 303 /* Fix the transmitted FTS count desired to exit from L0s. */ 304 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); 305 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | 306 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); 307 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1); 308 309 rockchip_pcie_set_power_limit(rockchip); 310 311 /* Set RC's clock architecture as common clock */ 312 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 313 status |= PCI_EXP_LNKSTA_SLC << 16; 314 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 315 316 /* Set RC's RCB to 128 */ 317 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 318 status |= PCI_EXP_LNKCTL_RCB; 319 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 320 321 /* Enable Gen1 training */ 322 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 323 PCIE_CLIENT_CONFIG); 324 325 gpiod_set_value_cansleep(rockchip->ep_gpio, 1); 326 327 /* 500ms timeout value should be enough for Gen1/2 training */ 328 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, 329 status, PCIE_LINK_UP(status), 20, 330 500 * USEC_PER_MSEC); 331 if (err) { 332 dev_err(dev, "PCIe link training gen1 timeout!\n"); 333 goto err_power_off_phy; 334 } 335 336 if (rockchip->link_gen == 2) { 337 /* 338 * Enable retrain for gen2. This should be configured only after 339 * gen1 finished. 340 */ 341 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 342 status |= PCI_EXP_LNKCTL_RL; 343 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 344 345 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, 346 status, PCIE_LINK_IS_GEN2(status), 20, 347 500 * USEC_PER_MSEC); 348 if (err) 349 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n"); 350 } 351 352 /* Check the final link width from negotiated lane counter from MGMT */ 353 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); 354 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> 355 PCIE_CORE_PL_CONF_LANE_SHIFT); 356 dev_dbg(dev, "current link width is x%d\n", status); 357 358 /* Power off unused lane(s) */ 359 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip); 360 for (i = 0; i < MAX_LANE_NUM; i++) { 361 if (!(rockchip->lanes_map & BIT(i))) { 362 dev_dbg(dev, "idling lane %d\n", i); 363 phy_power_off(rockchip->phys[i]); 364 } 365 } 366 367 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID, 368 PCIE_CORE_CONFIG_VENDOR); 369 rockchip_pcie_write(rockchip, 370 PCI_CLASS_BRIDGE_PCI_NORMAL << 8, 371 PCIE_RC_CONFIG_RID_CCR); 372 373 /* Clear THP cap's next cap pointer to remove L1 substate cap */ 374 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP); 375 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK; 376 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP); 377 378 /* Clear L0s from RC's link cap */ 379 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) { 380 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP); 381 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S; 382 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP); 383 } 384 385 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR); 386 status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK; 387 status |= PCIE_RC_CONFIG_DCSR_MPS_256; 388 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR); 389 390 return 0; 391err_power_off_phy: 392 while (i--) 393 phy_power_off(rockchip->phys[i]); 394 i = MAX_LANE_NUM; 395 while (i--) 396 phy_exit(rockchip->phys[i]); 397 return err; 398} 399 400static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) 401{ 402 struct rockchip_pcie *rockchip = arg; 403 struct device *dev = rockchip->dev; 404 u32 reg; 405 u32 sub_reg; 406 407 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 408 if (reg & PCIE_CLIENT_INT_LOCAL) { 409 dev_dbg(dev, "local interrupt received\n"); 410 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS); 411 if (sub_reg & PCIE_CORE_INT_PRFPE) 412 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n"); 413 414 if (sub_reg & PCIE_CORE_INT_CRFPE) 415 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n"); 416 417 if (sub_reg & PCIE_CORE_INT_RRPE) 418 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n"); 419 420 if (sub_reg & PCIE_CORE_INT_PRFO) 421 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n"); 422 423 if (sub_reg & PCIE_CORE_INT_CRFO) 424 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n"); 425 426 if (sub_reg & PCIE_CORE_INT_RT) 427 dev_dbg(dev, "replay timer timed out\n"); 428 429 if (sub_reg & PCIE_CORE_INT_RTR) 430 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n"); 431 432 if (sub_reg & PCIE_CORE_INT_PE) 433 dev_dbg(dev, "phy error detected on receive side\n"); 434 435 if (sub_reg & PCIE_CORE_INT_MTR) 436 dev_dbg(dev, "malformed TLP received from the link\n"); 437 438 if (sub_reg & PCIE_CORE_INT_UCR) 439 dev_dbg(dev, "malformed TLP received from the link\n"); 440 441 if (sub_reg & PCIE_CORE_INT_FCE) 442 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n"); 443 444 if (sub_reg & PCIE_CORE_INT_CT) 445 dev_dbg(dev, "a request timed out waiting for completion\n"); 446 447 if (sub_reg & PCIE_CORE_INT_UTC) 448 dev_dbg(dev, "unmapped TC error\n"); 449 450 if (sub_reg & PCIE_CORE_INT_MMVC) 451 dev_dbg(dev, "MSI mask register changes\n"); 452 453 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS); 454 } else if (reg & PCIE_CLIENT_INT_PHY) { 455 dev_dbg(dev, "phy link changes\n"); 456 rockchip_pcie_update_txcredit_mui(rockchip); 457 rockchip_pcie_clr_bw_int(rockchip); 458 } 459 460 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL, 461 PCIE_CLIENT_INT_STATUS); 462 463 return IRQ_HANDLED; 464} 465 466static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg) 467{ 468 struct rockchip_pcie *rockchip = arg; 469 struct device *dev = rockchip->dev; 470 u32 reg; 471 472 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 473 if (reg & PCIE_CLIENT_INT_LEGACY_DONE) 474 dev_dbg(dev, "legacy done interrupt received\n"); 475 476 if (reg & PCIE_CLIENT_INT_MSG) 477 dev_dbg(dev, "message done interrupt received\n"); 478 479 if (reg & PCIE_CLIENT_INT_HOT_RST) 480 dev_dbg(dev, "hot reset interrupt received\n"); 481 482 if (reg & PCIE_CLIENT_INT_DPA) 483 dev_dbg(dev, "dpa interrupt received\n"); 484 485 if (reg & PCIE_CLIENT_INT_FATAL_ERR) 486 dev_dbg(dev, "fatal error interrupt received\n"); 487 488 if (reg & PCIE_CLIENT_INT_NFATAL_ERR) 489 dev_dbg(dev, "no fatal error interrupt received\n"); 490 491 if (reg & PCIE_CLIENT_INT_CORR_ERR) 492 dev_dbg(dev, "correctable error interrupt received\n"); 493 494 if (reg & PCIE_CLIENT_INT_PHY) 495 dev_dbg(dev, "phy interrupt received\n"); 496 497 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE | 498 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST | 499 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR | 500 PCIE_CLIENT_INT_NFATAL_ERR | 501 PCIE_CLIENT_INT_CORR_ERR | 502 PCIE_CLIENT_INT_PHY), 503 PCIE_CLIENT_INT_STATUS); 504 505 return IRQ_HANDLED; 506} 507 508static void rockchip_pcie_intx_handler(struct irq_desc *desc) 509{ 510 struct irq_chip *chip = irq_desc_get_chip(desc); 511 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); 512 struct device *dev = rockchip->dev; 513 u32 reg; 514 u32 hwirq; 515 int ret; 516 517 chained_irq_enter(chip, desc); 518 519 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 520 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT; 521 522 while (reg) { 523 hwirq = ffs(reg) - 1; 524 reg &= ~BIT(hwirq); 525 526 ret = generic_handle_domain_irq(rockchip->irq_domain, hwirq); 527 if (ret) 528 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq); 529 } 530 531 chained_irq_exit(chip, desc); 532} 533 534static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip) 535{ 536 int irq, err; 537 struct device *dev = rockchip->dev; 538 struct platform_device *pdev = to_platform_device(dev); 539 540 irq = platform_get_irq_byname(pdev, "sys"); 541 if (irq < 0) 542 return irq; 543 544 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler, 545 IRQF_SHARED, "pcie-sys", rockchip); 546 if (err) { 547 dev_err(dev, "failed to request PCIe subsystem IRQ\n"); 548 return err; 549 } 550 551 irq = platform_get_irq_byname(pdev, "legacy"); 552 if (irq < 0) 553 return irq; 554 555 irq_set_chained_handler_and_data(irq, 556 rockchip_pcie_intx_handler, 557 rockchip); 558 559 irq = platform_get_irq_byname(pdev, "client"); 560 if (irq < 0) 561 return irq; 562 563 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler, 564 IRQF_SHARED, "pcie-client", rockchip); 565 if (err) { 566 dev_err(dev, "failed to request PCIe client IRQ\n"); 567 return err; 568 } 569 570 return 0; 571} 572 573/** 574 * rockchip_pcie_parse_host_dt - Parse Device Tree 575 * @rockchip: PCIe port information 576 * 577 * Return: '0' on success and error value on failure 578 */ 579static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) 580{ 581 struct device *dev = rockchip->dev; 582 int err; 583 584 err = rockchip_pcie_parse_dt(rockchip); 585 if (err) 586 return err; 587 588 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); 589 if (IS_ERR(rockchip->vpcie12v)) { 590 if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) 591 return PTR_ERR(rockchip->vpcie12v); 592 dev_info(dev, "no vpcie12v regulator found\n"); 593 } 594 595 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); 596 if (IS_ERR(rockchip->vpcie3v3)) { 597 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) 598 return PTR_ERR(rockchip->vpcie3v3); 599 dev_info(dev, "no vpcie3v3 regulator found\n"); 600 } 601 602 rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8"); 603 if (IS_ERR(rockchip->vpcie1v8)) 604 return PTR_ERR(rockchip->vpcie1v8); 605 606 rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9"); 607 if (IS_ERR(rockchip->vpcie0v9)) 608 return PTR_ERR(rockchip->vpcie0v9); 609 610 return 0; 611} 612 613static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip) 614{ 615 struct device *dev = rockchip->dev; 616 int err; 617 618 if (!IS_ERR(rockchip->vpcie12v)) { 619 err = regulator_enable(rockchip->vpcie12v); 620 if (err) { 621 dev_err(dev, "fail to enable vpcie12v regulator\n"); 622 goto err_out; 623 } 624 } 625 626 if (!IS_ERR(rockchip->vpcie3v3)) { 627 err = regulator_enable(rockchip->vpcie3v3); 628 if (err) { 629 dev_err(dev, "fail to enable vpcie3v3 regulator\n"); 630 goto err_disable_12v; 631 } 632 } 633 634 err = regulator_enable(rockchip->vpcie1v8); 635 if (err) { 636 dev_err(dev, "fail to enable vpcie1v8 regulator\n"); 637 goto err_disable_3v3; 638 } 639 640 err = regulator_enable(rockchip->vpcie0v9); 641 if (err) { 642 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 643 goto err_disable_1v8; 644 } 645 646 return 0; 647 648err_disable_1v8: 649 regulator_disable(rockchip->vpcie1v8); 650err_disable_3v3: 651 if (!IS_ERR(rockchip->vpcie3v3)) 652 regulator_disable(rockchip->vpcie3v3); 653err_disable_12v: 654 if (!IS_ERR(rockchip->vpcie12v)) 655 regulator_disable(rockchip->vpcie12v); 656err_out: 657 return err; 658} 659 660static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip) 661{ 662 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) & 663 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK); 664 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT), 665 PCIE_CORE_INT_MASK); 666 667 rockchip_pcie_enable_bw_int(rockchip); 668} 669 670static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, 671 irq_hw_number_t hwirq) 672{ 673 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); 674 irq_set_chip_data(irq, domain->host_data); 675 676 return 0; 677} 678 679static const struct irq_domain_ops intx_domain_ops = { 680 .map = rockchip_pcie_intx_map, 681}; 682 683static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) 684{ 685 struct device *dev = rockchip->dev; 686 struct device_node *intc = of_get_next_child(dev->of_node, NULL); 687 688 if (!intc) { 689 dev_err(dev, "missing child interrupt-controller node\n"); 690 return -EINVAL; 691 } 692 693 rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX, 694 &intx_domain_ops, rockchip); 695 of_node_put(intc); 696 if (!rockchip->irq_domain) { 697 dev_err(dev, "failed to get a INTx IRQ domain\n"); 698 return -EINVAL; 699 } 700 701 return 0; 702} 703 704static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip, 705 int region_no, int type, u8 num_pass_bits, 706 u32 lower_addr, u32 upper_addr) 707{ 708 u32 ob_addr_0; 709 u32 ob_addr_1; 710 u32 ob_desc_0; 711 u32 aw_offset; 712 713 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM) 714 return -EINVAL; 715 if (num_pass_bits + 1 < 8) 716 return -EINVAL; 717 if (num_pass_bits > 63) 718 return -EINVAL; 719 if (region_no == 0) { 720 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits)) 721 return -EINVAL; 722 } 723 if (region_no != 0) { 724 if (AXI_REGION_SIZE < (2ULL << num_pass_bits)) 725 return -EINVAL; 726 } 727 728 aw_offset = (region_no << OB_REG_SIZE_SHIFT); 729 730 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS; 731 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR; 732 ob_addr_1 = upper_addr; 733 ob_desc_0 = (1 << 23 | type); 734 735 rockchip_pcie_write(rockchip, ob_addr_0, 736 PCIE_CORE_OB_REGION_ADDR0 + aw_offset); 737 rockchip_pcie_write(rockchip, ob_addr_1, 738 PCIE_CORE_OB_REGION_ADDR1 + aw_offset); 739 rockchip_pcie_write(rockchip, ob_desc_0, 740 PCIE_CORE_OB_REGION_DESC0 + aw_offset); 741 rockchip_pcie_write(rockchip, 0, 742 PCIE_CORE_OB_REGION_DESC1 + aw_offset); 743 744 return 0; 745} 746 747static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip, 748 int region_no, u8 num_pass_bits, 749 u32 lower_addr, u32 upper_addr) 750{ 751 u32 ib_addr_0; 752 u32 ib_addr_1; 753 u32 aw_offset; 754 755 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM) 756 return -EINVAL; 757 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED) 758 return -EINVAL; 759 if (num_pass_bits > 63) 760 return -EINVAL; 761 762 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT); 763 764 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS; 765 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR; 766 ib_addr_1 = upper_addr; 767 768 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset); 769 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset); 770 771 return 0; 772} 773 774static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip) 775{ 776 struct device *dev = rockchip->dev; 777 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 778 struct resource_entry *entry; 779 u64 pci_addr, size; 780 int offset; 781 int err; 782 int reg_no; 783 784 rockchip_pcie_cfg_configuration_accesses(rockchip, 785 AXI_WRAPPER_TYPE0_CFG); 786 entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); 787 if (!entry) 788 return -ENODEV; 789 790 size = resource_size(entry->res); 791 pci_addr = entry->res->start - entry->offset; 792 rockchip->msg_bus_addr = pci_addr; 793 794 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 795 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1, 796 AXI_WRAPPER_MEM_WRITE, 797 20 - 1, 798 pci_addr + (reg_no << 20), 799 0); 800 if (err) { 801 dev_err(dev, "program RC mem outbound ATU failed\n"); 802 return err; 803 } 804 } 805 806 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0); 807 if (err) { 808 dev_err(dev, "program RC mem inbound ATU failed\n"); 809 return err; 810 } 811 812 entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO); 813 if (!entry) 814 return -ENODEV; 815 816 /* store the register number offset to program RC io outbound ATU */ 817 offset = size >> 20; 818 819 size = resource_size(entry->res); 820 pci_addr = entry->res->start - entry->offset; 821 822 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 823 err = rockchip_pcie_prog_ob_atu(rockchip, 824 reg_no + 1 + offset, 825 AXI_WRAPPER_IO_WRITE, 826 20 - 1, 827 pci_addr + (reg_no << 20), 828 0); 829 if (err) { 830 dev_err(dev, "program RC io outbound ATU failed\n"); 831 return err; 832 } 833 } 834 835 /* assign message regions */ 836 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset, 837 AXI_WRAPPER_NOR_MSG, 838 20 - 1, 0, 0); 839 840 rockchip->msg_bus_addr += ((reg_no + offset) << 20); 841 return err; 842} 843 844static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip) 845{ 846 u32 value; 847 int err; 848 849 /* send PME_TURN_OFF message */ 850 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF); 851 852 /* read LTSSM and wait for falling into L2 link state */ 853 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0, 854 value, PCIE_LINK_IS_L2(value), 20, 855 jiffies_to_usecs(5 * HZ)); 856 if (err) { 857 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n"); 858 return err; 859 } 860 861 return 0; 862} 863 864static int rockchip_pcie_suspend_noirq(struct device *dev) 865{ 866 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 867 int ret; 868 869 /* disable core and cli int since we don't need to ack PME_ACK */ 870 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) | 871 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK); 872 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK); 873 874 ret = rockchip_pcie_wait_l2(rockchip); 875 if (ret) { 876 rockchip_pcie_enable_interrupts(rockchip); 877 return ret; 878 } 879 880 rockchip_pcie_deinit_phys(rockchip); 881 882 rockchip_pcie_disable_clocks(rockchip); 883 884 regulator_disable(rockchip->vpcie0v9); 885 886 return ret; 887} 888 889static int rockchip_pcie_resume_noirq(struct device *dev) 890{ 891 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 892 int err; 893 894 err = regulator_enable(rockchip->vpcie0v9); 895 if (err) { 896 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 897 return err; 898 } 899 900 err = rockchip_pcie_enable_clocks(rockchip); 901 if (err) 902 goto err_disable_0v9; 903 904 err = rockchip_pcie_host_init_port(rockchip); 905 if (err) 906 goto err_pcie_resume; 907 908 err = rockchip_pcie_cfg_atu(rockchip); 909 if (err) 910 goto err_err_deinit_port; 911 912 /* Need this to enter L1 again */ 913 rockchip_pcie_update_txcredit_mui(rockchip); 914 rockchip_pcie_enable_interrupts(rockchip); 915 916 return 0; 917 918err_err_deinit_port: 919 rockchip_pcie_deinit_phys(rockchip); 920err_pcie_resume: 921 rockchip_pcie_disable_clocks(rockchip); 922err_disable_0v9: 923 regulator_disable(rockchip->vpcie0v9); 924 return err; 925} 926 927static int rockchip_pcie_probe(struct platform_device *pdev) 928{ 929 struct rockchip_pcie *rockchip; 930 struct device *dev = &pdev->dev; 931 struct pci_host_bridge *bridge; 932 int err; 933 934 if (!dev->of_node) 935 return -ENODEV; 936 937 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip)); 938 if (!bridge) 939 return -ENOMEM; 940 941 rockchip = pci_host_bridge_priv(bridge); 942 943 platform_set_drvdata(pdev, rockchip); 944 rockchip->dev = dev; 945 rockchip->is_rc = true; 946 947 err = rockchip_pcie_parse_host_dt(rockchip); 948 if (err) 949 return err; 950 951 err = rockchip_pcie_enable_clocks(rockchip); 952 if (err) 953 return err; 954 955 err = rockchip_pcie_set_vpcie(rockchip); 956 if (err) { 957 dev_err(dev, "failed to set vpcie regulator\n"); 958 goto err_set_vpcie; 959 } 960 961 err = rockchip_pcie_host_init_port(rockchip); 962 if (err) 963 goto err_vpcie; 964 965 err = rockchip_pcie_init_irq_domain(rockchip); 966 if (err < 0) 967 goto err_deinit_port; 968 969 err = rockchip_pcie_cfg_atu(rockchip); 970 if (err) 971 goto err_remove_irq_domain; 972 973 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M); 974 if (!rockchip->msg_region) { 975 err = -ENOMEM; 976 goto err_remove_irq_domain; 977 } 978 979 bridge->sysdata = rockchip; 980 bridge->ops = &rockchip_pcie_ops; 981 982 err = rockchip_pcie_setup_irq(rockchip); 983 if (err) 984 goto err_remove_irq_domain; 985 986 rockchip_pcie_enable_interrupts(rockchip); 987 988 err = pci_host_probe(bridge); 989 if (err < 0) 990 goto err_remove_irq_domain; 991 992 return 0; 993 994err_remove_irq_domain: 995 irq_domain_remove(rockchip->irq_domain); 996err_deinit_port: 997 rockchip_pcie_deinit_phys(rockchip); 998err_vpcie: 999 if (!IS_ERR(rockchip->vpcie12v)) 1000 regulator_disable(rockchip->vpcie12v); 1001 if (!IS_ERR(rockchip->vpcie3v3)) 1002 regulator_disable(rockchip->vpcie3v3); 1003 regulator_disable(rockchip->vpcie1v8); 1004 regulator_disable(rockchip->vpcie0v9); 1005err_set_vpcie: 1006 rockchip_pcie_disable_clocks(rockchip); 1007 return err; 1008} 1009 1010static void rockchip_pcie_remove(struct platform_device *pdev) 1011{ 1012 struct device *dev = &pdev->dev; 1013 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 1014 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 1015 1016 pci_stop_root_bus(bridge->bus); 1017 pci_remove_root_bus(bridge->bus); 1018 irq_domain_remove(rockchip->irq_domain); 1019 1020 rockchip_pcie_deinit_phys(rockchip); 1021 1022 rockchip_pcie_disable_clocks(rockchip); 1023 1024 if (!IS_ERR(rockchip->vpcie12v)) 1025 regulator_disable(rockchip->vpcie12v); 1026 if (!IS_ERR(rockchip->vpcie3v3)) 1027 regulator_disable(rockchip->vpcie3v3); 1028 regulator_disable(rockchip->vpcie1v8); 1029 regulator_disable(rockchip->vpcie0v9); 1030} 1031 1032static const struct dev_pm_ops rockchip_pcie_pm_ops = { 1033 NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq, 1034 rockchip_pcie_resume_noirq) 1035}; 1036 1037static const struct of_device_id rockchip_pcie_of_match[] = { 1038 { .compatible = "rockchip,rk3399-pcie", }, 1039 {} 1040}; 1041MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match); 1042 1043static struct platform_driver rockchip_pcie_driver = { 1044 .driver = { 1045 .name = "rockchip-pcie", 1046 .of_match_table = rockchip_pcie_of_match, 1047 .pm = &rockchip_pcie_pm_ops, 1048 }, 1049 .probe = rockchip_pcie_probe, 1050 .remove_new = rockchip_pcie_remove, 1051}; 1052module_platform_driver(rockchip_pcie_driver); 1053 1054MODULE_AUTHOR("Rockchip Inc"); 1055MODULE_DESCRIPTION("Rockchip AXI PCIe driver"); 1056MODULE_LICENSE("GPL v2"); 1057