1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Microsemi Switchtec(tm) PCIe Management Driver 4 * Copyright (c) 2017, Microsemi Corporation 5 */ 6 7#include <linux/interrupt.h> 8#include <linux/io-64-nonatomic-lo-hi.h> 9#include <linux/delay.h> 10#include <linux/kthread.h> 11#include <linux/module.h> 12#include <linux/ntb.h> 13#include <linux/pci.h> 14#include <linux/switchtec.h> 15 16MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver"); 17MODULE_VERSION("0.1"); 18MODULE_LICENSE("GPL"); 19MODULE_AUTHOR("Microsemi Corporation"); 20 21static ulong max_mw_size = SZ_2M; 22module_param(max_mw_size, ulong, 0644); 23MODULE_PARM_DESC(max_mw_size, 24 "Max memory window size reported to the upper layer"); 25 26static bool use_lut_mws; 27module_param(use_lut_mws, bool, 0644); 28MODULE_PARM_DESC(use_lut_mws, 29 "Enable the use of the LUT based memory windows"); 30 31#define SWITCHTEC_NTB_MAGIC 0x45CC0001 32#define MAX_MWS 128 33 34struct shared_mw { 35 u32 magic; 36 u32 link_sta; 37 u32 partition_id; 38 u64 mw_sizes[MAX_MWS]; 39 u32 spad[128]; 40}; 41 42#define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry) 43#define LUT_SIZE SZ_64K 44 45struct switchtec_ntb { 46 struct ntb_dev ntb; 47 struct switchtec_dev *stdev; 48 49 int self_partition; 50 int peer_partition; 51 52 int doorbell_irq; 53 int message_irq; 54 55 struct ntb_info_regs __iomem *mmio_ntb; 56 struct ntb_ctrl_regs __iomem *mmio_ctrl; 57 struct ntb_dbmsg_regs __iomem *mmio_dbmsg; 58 struct ntb_ctrl_regs __iomem *mmio_self_ctrl; 59 struct ntb_ctrl_regs __iomem *mmio_peer_ctrl; 60 struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg; 61 struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg; 62 63 void __iomem *mmio_xlink_win; 64 65 struct shared_mw *self_shared; 66 struct shared_mw __iomem *peer_shared; 67 dma_addr_t self_shared_dma; 68 69 u64 db_mask; 70 u64 db_valid_mask; 71 int db_shift; 72 int db_peer_shift; 73 74 /* synchronize rmw access of db_mask and hw reg */ 75 spinlock_t db_mask_lock; 76 77 int nr_direct_mw; 78 int nr_lut_mw; 79 int nr_rsvd_luts; 80 int direct_mw_to_bar[MAX_DIRECT_MW]; 81 82 int peer_nr_direct_mw; 83 int peer_nr_lut_mw; 84 int peer_direct_mw_to_bar[MAX_DIRECT_MW]; 85 86 bool link_is_up; 87 enum ntb_speed link_speed; 88 enum ntb_width link_width; 89 struct work_struct check_link_status_work; 90 bool link_force_down; 91}; 92 93static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb) 94{ 95 return container_of(ntb, struct switchtec_ntb, ntb); 96} 97 98static int switchtec_ntb_part_op(struct switchtec_ntb *sndev, 99 struct ntb_ctrl_regs __iomem *ctl, 100 u32 op, int wait_status) 101{ 102 static const char * const op_text[] = { 103 [NTB_CTRL_PART_OP_LOCK] = "lock", 104 [NTB_CTRL_PART_OP_CFG] = "configure", 105 [NTB_CTRL_PART_OP_RESET] = "reset", 106 }; 107 108 int i; 109 u32 ps; 110 int status; 111 112 switch (op) { 113 case NTB_CTRL_PART_OP_LOCK: 114 status = NTB_CTRL_PART_STATUS_LOCKING; 115 break; 116 case NTB_CTRL_PART_OP_CFG: 117 status = NTB_CTRL_PART_STATUS_CONFIGURING; 118 break; 119 case NTB_CTRL_PART_OP_RESET: 120 status = NTB_CTRL_PART_STATUS_RESETTING; 121 break; 122 default: 123 return -EINVAL; 124 } 125 126 iowrite32(op, &ctl->partition_op); 127 128 for (i = 0; i < 1000; i++) { 129 if (msleep_interruptible(50) != 0) { 130 iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op); 131 return -EINTR; 132 } 133 134 ps = ioread32(&ctl->partition_status) & 0xFFFF; 135 136 if (ps != status) 137 break; 138 } 139 140 if (ps == wait_status) 141 return 0; 142 143 if (ps == status) { 144 dev_err(&sndev->stdev->dev, 145 "Timed out while performing %s (%d). (%08x)\n", 146 op_text[op], op, 147 ioread32(&ctl->partition_status)); 148 149 return -ETIMEDOUT; 150 } 151 152 return -EIO; 153} 154 155static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx, 156 u32 val) 157{ 158 if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg)) 159 return -EINVAL; 160 161 iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg); 162 163 return 0; 164} 165 166static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx) 167{ 168 struct switchtec_ntb *sndev = ntb_sndev(ntb); 169 int nr_direct_mw = sndev->peer_nr_direct_mw; 170 int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts; 171 172 if (pidx != NTB_DEF_PEER_IDX) 173 return -EINVAL; 174 175 if (!use_lut_mws) 176 nr_lut_mw = 0; 177 178 return nr_direct_mw + nr_lut_mw; 179} 180 181static int lut_index(struct switchtec_ntb *sndev, int mw_idx) 182{ 183 return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts; 184} 185 186static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx) 187{ 188 return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts; 189} 190 191static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, 192 int widx, resource_size_t *addr_align, 193 resource_size_t *size_align, 194 resource_size_t *size_max) 195{ 196 struct switchtec_ntb *sndev = ntb_sndev(ntb); 197 int lut; 198 resource_size_t size; 199 200 if (pidx != NTB_DEF_PEER_IDX) 201 return -EINVAL; 202 203 lut = widx >= sndev->peer_nr_direct_mw; 204 size = ioread64(&sndev->peer_shared->mw_sizes[widx]); 205 206 if (size == 0) 207 return -EINVAL; 208 209 if (addr_align) 210 *addr_align = lut ? size : SZ_4K; 211 212 if (size_align) 213 *size_align = lut ? size : SZ_4K; 214 215 if (size_max) 216 *size_max = size; 217 218 return 0; 219} 220 221static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx) 222{ 223 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 224 int bar = sndev->peer_direct_mw_to_bar[idx]; 225 u32 ctl_val; 226 227 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 228 ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN; 229 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 230 iowrite32(0, &ctl->bar_entry[bar].win_size); 231 iowrite32(0, &ctl->bar_ext_entry[bar].win_size); 232 iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr); 233} 234 235static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx) 236{ 237 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 238 239 iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]); 240} 241 242static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx, 243 dma_addr_t addr, resource_size_t size) 244{ 245 int xlate_pos = ilog2(size); 246 int bar = sndev->peer_direct_mw_to_bar[idx]; 247 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 248 u32 ctl_val; 249 250 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 251 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; 252 253 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 254 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000), 255 &ctl->bar_entry[bar].win_size); 256 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size); 257 iowrite64(sndev->self_partition | addr, 258 &ctl->bar_entry[bar].xlate_addr); 259} 260 261static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx, 262 dma_addr_t addr, resource_size_t size) 263{ 264 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 265 266 iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr), 267 &ctl->lut_entry[peer_lut_index(sndev, idx)]); 268} 269 270static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, 271 dma_addr_t addr, resource_size_t size) 272{ 273 struct switchtec_ntb *sndev = ntb_sndev(ntb); 274 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 275 int xlate_pos = ilog2(size); 276 int nr_direct_mw = sndev->peer_nr_direct_mw; 277 int rc; 278 279 if (pidx != NTB_DEF_PEER_IDX) 280 return -EINVAL; 281 282 dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n", 283 widx, pidx, &addr, &size); 284 285 if (widx >= switchtec_ntb_mw_count(ntb, pidx)) 286 return -EINVAL; 287 288 if (size != 0 && xlate_pos < 12) 289 return -EINVAL; 290 291 if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) { 292 /* 293 * In certain circumstances we can get a buffer that is 294 * not aligned to its size. (Most of the time 295 * dma_alloc_coherent ensures this). This can happen when 296 * using large buffers allocated by the CMA 297 * (see CMA_CONFIG_ALIGNMENT) 298 */ 299 dev_err(&sndev->stdev->dev, 300 "ERROR: Memory window address is not aligned to its size!\n"); 301 return -EINVAL; 302 } 303 304 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 305 NTB_CTRL_PART_STATUS_LOCKED); 306 if (rc) 307 return rc; 308 309 if (size == 0) { 310 if (widx < nr_direct_mw) 311 switchtec_ntb_mw_clr_direct(sndev, widx); 312 else 313 switchtec_ntb_mw_clr_lut(sndev, widx); 314 } else { 315 if (widx < nr_direct_mw) 316 switchtec_ntb_mw_set_direct(sndev, widx, addr, size); 317 else 318 switchtec_ntb_mw_set_lut(sndev, widx, addr, size); 319 } 320 321 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 322 NTB_CTRL_PART_STATUS_NORMAL); 323 324 if (rc == -EIO) { 325 dev_err(&sndev->stdev->dev, 326 "Hardware reported an error configuring mw %d: %08x\n", 327 widx, ioread32(&ctl->bar_error)); 328 329 if (widx < nr_direct_mw) 330 switchtec_ntb_mw_clr_direct(sndev, widx); 331 else 332 switchtec_ntb_mw_clr_lut(sndev, widx); 333 334 switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 335 NTB_CTRL_PART_STATUS_NORMAL); 336 } 337 338 return rc; 339} 340 341static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb) 342{ 343 struct switchtec_ntb *sndev = ntb_sndev(ntb); 344 int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts; 345 346 return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0); 347} 348 349static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev, 350 int idx, phys_addr_t *base, 351 resource_size_t *size) 352{ 353 int bar = sndev->direct_mw_to_bar[idx]; 354 size_t offset = 0; 355 356 if (bar < 0) 357 return -EINVAL; 358 359 if (idx == 0) { 360 /* 361 * This is the direct BAR shared with the LUTs 362 * which means the actual window will be offset 363 * by the size of all the LUT entries. 364 */ 365 366 offset = LUT_SIZE * sndev->nr_lut_mw; 367 } 368 369 if (base) 370 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; 371 372 if (size) { 373 *size = pci_resource_len(sndev->ntb.pdev, bar) - offset; 374 if (offset && *size > offset) 375 *size = offset; 376 377 if (*size > max_mw_size) 378 *size = max_mw_size; 379 } 380 381 return 0; 382} 383 384static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev, 385 int idx, phys_addr_t *base, 386 resource_size_t *size) 387{ 388 int bar = sndev->direct_mw_to_bar[0]; 389 int offset; 390 391 offset = LUT_SIZE * lut_index(sndev, idx); 392 393 if (base) 394 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; 395 396 if (size) 397 *size = LUT_SIZE; 398 399 return 0; 400} 401 402static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 403 phys_addr_t *base, 404 resource_size_t *size) 405{ 406 struct switchtec_ntb *sndev = ntb_sndev(ntb); 407 408 if (idx < sndev->nr_direct_mw) 409 return switchtec_ntb_direct_get_addr(sndev, idx, base, size); 410 else if (idx < switchtec_ntb_peer_mw_count(ntb)) 411 return switchtec_ntb_lut_get_addr(sndev, idx, base, size); 412 else 413 return -EINVAL; 414} 415 416static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev, 417 int partition, 418 enum ntb_speed *speed, 419 enum ntb_width *width) 420{ 421 struct switchtec_dev *stdev = sndev->stdev; 422 struct part_cfg_regs __iomem *part_cfg = 423 &stdev->mmio_part_cfg_all[partition]; 424 425 u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF; 426 u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]); 427 428 if (speed) 429 *speed = (linksta >> 16) & 0xF; 430 431 if (width) 432 *width = (linksta >> 20) & 0x3F; 433} 434 435static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev) 436{ 437 enum ntb_speed self_speed, peer_speed; 438 enum ntb_width self_width, peer_width; 439 440 if (!sndev->link_is_up) { 441 sndev->link_speed = NTB_SPEED_NONE; 442 sndev->link_width = NTB_WIDTH_NONE; 443 return; 444 } 445 446 switchtec_ntb_part_link_speed(sndev, sndev->self_partition, 447 &self_speed, &self_width); 448 switchtec_ntb_part_link_speed(sndev, sndev->peer_partition, 449 &peer_speed, &peer_width); 450 451 sndev->link_speed = min(self_speed, peer_speed); 452 sndev->link_width = min(self_width, peer_width); 453} 454 455static int crosslink_is_enabled(struct switchtec_ntb *sndev) 456{ 457 struct ntb_info_regs __iomem *inf = sndev->mmio_ntb; 458 459 return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled); 460} 461 462static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev) 463{ 464 int i; 465 u32 msg_map = 0; 466 467 if (!crosslink_is_enabled(sndev)) 468 return; 469 470 for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) { 471 int m = i | sndev->self_partition << 2; 472 473 msg_map |= m << i * 8; 474 } 475 476 iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map); 477 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 478 &sndev->mmio_peer_dbmsg->odb_mask); 479} 480 481enum switchtec_msg { 482 LINK_MESSAGE = 0, 483 MSG_LINK_UP = 1, 484 MSG_LINK_DOWN = 2, 485 MSG_CHECK_LINK = 3, 486 MSG_LINK_FORCE_DOWN = 4, 487}; 488 489static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev); 490 491static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev) 492{ 493 int link_sta; 494 int old = sndev->link_is_up; 495 496 link_sta = sndev->self_shared->link_sta; 497 if (link_sta) { 498 u64 peer = ioread64(&sndev->peer_shared->magic); 499 500 if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC) 501 link_sta = peer >> 32; 502 else 503 link_sta = 0; 504 } 505 506 sndev->link_is_up = link_sta; 507 switchtec_ntb_set_link_speed(sndev); 508 509 if (link_sta != old) { 510 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK); 511 ntb_link_event(&sndev->ntb); 512 dev_info(&sndev->stdev->dev, "ntb link %s\n", 513 link_sta ? "up" : "down"); 514 515 if (link_sta) 516 crosslink_init_dbmsgs(sndev); 517 } 518} 519 520static void check_link_status_work(struct work_struct *work) 521{ 522 struct switchtec_ntb *sndev; 523 524 sndev = container_of(work, struct switchtec_ntb, 525 check_link_status_work); 526 527 if (sndev->link_force_down) { 528 sndev->link_force_down = false; 529 switchtec_ntb_reinit_peer(sndev); 530 531 if (sndev->link_is_up) { 532 sndev->link_is_up = 0; 533 ntb_link_event(&sndev->ntb); 534 dev_info(&sndev->stdev->dev, "ntb link forced down\n"); 535 } 536 537 return; 538 } 539 540 switchtec_ntb_link_status_update(sndev); 541} 542 543static void switchtec_ntb_check_link(struct switchtec_ntb *sndev, 544 enum switchtec_msg msg) 545{ 546 if (msg == MSG_LINK_FORCE_DOWN) 547 sndev->link_force_down = true; 548 549 schedule_work(&sndev->check_link_status_work); 550} 551 552static void switchtec_ntb_link_notification(struct switchtec_dev *stdev) 553{ 554 struct switchtec_ntb *sndev = stdev->sndev; 555 556 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 557} 558 559static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb, 560 enum ntb_speed *speed, 561 enum ntb_width *width) 562{ 563 struct switchtec_ntb *sndev = ntb_sndev(ntb); 564 565 if (speed) 566 *speed = sndev->link_speed; 567 if (width) 568 *width = sndev->link_width; 569 570 return sndev->link_is_up; 571} 572 573static int switchtec_ntb_link_enable(struct ntb_dev *ntb, 574 enum ntb_speed max_speed, 575 enum ntb_width max_width) 576{ 577 struct switchtec_ntb *sndev = ntb_sndev(ntb); 578 579 dev_dbg(&sndev->stdev->dev, "enabling link\n"); 580 581 sndev->self_shared->link_sta = 1; 582 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); 583 584 switchtec_ntb_link_status_update(sndev); 585 586 return 0; 587} 588 589static int switchtec_ntb_link_disable(struct ntb_dev *ntb) 590{ 591 struct switchtec_ntb *sndev = ntb_sndev(ntb); 592 593 dev_dbg(&sndev->stdev->dev, "disabling link\n"); 594 595 sndev->self_shared->link_sta = 0; 596 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN); 597 598 switchtec_ntb_link_status_update(sndev); 599 600 return 0; 601} 602 603static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb) 604{ 605 struct switchtec_ntb *sndev = ntb_sndev(ntb); 606 607 return sndev->db_valid_mask; 608} 609 610static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb) 611{ 612 return 1; 613} 614 615static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 616{ 617 struct switchtec_ntb *sndev = ntb_sndev(ntb); 618 619 if (db_vector < 0 || db_vector > 1) 620 return 0; 621 622 return sndev->db_valid_mask; 623} 624 625static u64 switchtec_ntb_db_read(struct ntb_dev *ntb) 626{ 627 u64 ret; 628 struct switchtec_ntb *sndev = ntb_sndev(ntb); 629 630 ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift; 631 632 return ret & sndev->db_valid_mask; 633} 634 635static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 636{ 637 struct switchtec_ntb *sndev = ntb_sndev(ntb); 638 639 iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb); 640 641 return 0; 642} 643 644static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 645{ 646 unsigned long irqflags; 647 struct switchtec_ntb *sndev = ntb_sndev(ntb); 648 649 if (db_bits & ~sndev->db_valid_mask) 650 return -EINVAL; 651 652 spin_lock_irqsave(&sndev->db_mask_lock, irqflags); 653 654 sndev->db_mask |= db_bits << sndev->db_shift; 655 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 656 657 spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); 658 659 return 0; 660} 661 662static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 663{ 664 unsigned long irqflags; 665 struct switchtec_ntb *sndev = ntb_sndev(ntb); 666 667 if (db_bits & ~sndev->db_valid_mask) 668 return -EINVAL; 669 670 spin_lock_irqsave(&sndev->db_mask_lock, irqflags); 671 672 sndev->db_mask &= ~(db_bits << sndev->db_shift); 673 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 674 675 spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); 676 677 return 0; 678} 679 680static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb) 681{ 682 struct switchtec_ntb *sndev = ntb_sndev(ntb); 683 684 return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask; 685} 686 687static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb, 688 phys_addr_t *db_addr, 689 resource_size_t *db_size, 690 u64 *db_data, 691 int db_bit) 692{ 693 struct switchtec_ntb *sndev = ntb_sndev(ntb); 694 unsigned long offset; 695 696 if (unlikely(db_bit >= BITS_PER_LONG_LONG)) 697 return -EINVAL; 698 699 offset = (unsigned long)sndev->mmio_peer_dbmsg->odb - 700 (unsigned long)sndev->stdev->mmio; 701 702 offset += sndev->db_shift / 8; 703 704 if (db_addr) 705 *db_addr = pci_resource_start(ntb->pdev, 0) + offset; 706 if (db_size) 707 *db_size = sizeof(u32); 708 if (db_data) 709 *db_data = BIT_ULL(db_bit) << sndev->db_peer_shift; 710 711 return 0; 712} 713 714static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 715{ 716 struct switchtec_ntb *sndev = ntb_sndev(ntb); 717 718 iowrite64(db_bits << sndev->db_peer_shift, 719 &sndev->mmio_peer_dbmsg->odb); 720 721 return 0; 722} 723 724static int switchtec_ntb_spad_count(struct ntb_dev *ntb) 725{ 726 struct switchtec_ntb *sndev = ntb_sndev(ntb); 727 728 return ARRAY_SIZE(sndev->self_shared->spad); 729} 730 731static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx) 732{ 733 struct switchtec_ntb *sndev = ntb_sndev(ntb); 734 735 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) 736 return 0; 737 738 if (!sndev->self_shared) 739 return 0; 740 741 return sndev->self_shared->spad[idx]; 742} 743 744static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val) 745{ 746 struct switchtec_ntb *sndev = ntb_sndev(ntb); 747 748 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) 749 return -EINVAL; 750 751 if (!sndev->self_shared) 752 return -EIO; 753 754 sndev->self_shared->spad[idx] = val; 755 756 return 0; 757} 758 759static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, 760 int sidx) 761{ 762 struct switchtec_ntb *sndev = ntb_sndev(ntb); 763 764 if (pidx != NTB_DEF_PEER_IDX) 765 return -EINVAL; 766 767 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) 768 return 0; 769 770 if (!sndev->peer_shared) 771 return 0; 772 773 return ioread32(&sndev->peer_shared->spad[sidx]); 774} 775 776static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, 777 int sidx, u32 val) 778{ 779 struct switchtec_ntb *sndev = ntb_sndev(ntb); 780 781 if (pidx != NTB_DEF_PEER_IDX) 782 return -EINVAL; 783 784 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) 785 return -EINVAL; 786 787 if (!sndev->peer_shared) 788 return -EIO; 789 790 iowrite32(val, &sndev->peer_shared->spad[sidx]); 791 792 return 0; 793} 794 795static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, 796 int sidx, phys_addr_t *spad_addr) 797{ 798 struct switchtec_ntb *sndev = ntb_sndev(ntb); 799 unsigned long offset; 800 801 if (pidx != NTB_DEF_PEER_IDX) 802 return -EINVAL; 803 804 offset = (unsigned long)&sndev->peer_shared->spad[sidx] - 805 (unsigned long)sndev->stdev->mmio; 806 807 if (spad_addr) 808 *spad_addr = pci_resource_start(ntb->pdev, 0) + offset; 809 810 return 0; 811} 812 813static const struct ntb_dev_ops switchtec_ntb_ops = { 814 .mw_count = switchtec_ntb_mw_count, 815 .mw_get_align = switchtec_ntb_mw_get_align, 816 .mw_set_trans = switchtec_ntb_mw_set_trans, 817 .peer_mw_count = switchtec_ntb_peer_mw_count, 818 .peer_mw_get_addr = switchtec_ntb_peer_mw_get_addr, 819 .link_is_up = switchtec_ntb_link_is_up, 820 .link_enable = switchtec_ntb_link_enable, 821 .link_disable = switchtec_ntb_link_disable, 822 .db_valid_mask = switchtec_ntb_db_valid_mask, 823 .db_vector_count = switchtec_ntb_db_vector_count, 824 .db_vector_mask = switchtec_ntb_db_vector_mask, 825 .db_read = switchtec_ntb_db_read, 826 .db_clear = switchtec_ntb_db_clear, 827 .db_set_mask = switchtec_ntb_db_set_mask, 828 .db_clear_mask = switchtec_ntb_db_clear_mask, 829 .db_read_mask = switchtec_ntb_db_read_mask, 830 .peer_db_addr = switchtec_ntb_peer_db_addr, 831 .peer_db_set = switchtec_ntb_peer_db_set, 832 .spad_count = switchtec_ntb_spad_count, 833 .spad_read = switchtec_ntb_spad_read, 834 .spad_write = switchtec_ntb_spad_write, 835 .peer_spad_read = switchtec_ntb_peer_spad_read, 836 .peer_spad_write = switchtec_ntb_peer_spad_write, 837 .peer_spad_addr = switchtec_ntb_peer_spad_addr, 838}; 839 840static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev) 841{ 842 u64 tpart_vec; 843 int self; 844 u64 part_map; 845 846 sndev->ntb.pdev = sndev->stdev->pdev; 847 sndev->ntb.topo = NTB_TOPO_SWITCH; 848 sndev->ntb.ops = &switchtec_ntb_ops; 849 850 INIT_WORK(&sndev->check_link_status_work, check_link_status_work); 851 sndev->link_force_down = false; 852 853 sndev->self_partition = sndev->stdev->partition; 854 855 sndev->mmio_ntb = sndev->stdev->mmio_ntb; 856 857 self = sndev->self_partition; 858 tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high); 859 tpart_vec <<= 32; 860 tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low); 861 862 part_map = ioread64(&sndev->mmio_ntb->ep_map); 863 tpart_vec &= part_map; 864 part_map &= ~(1 << sndev->self_partition); 865 866 if (!tpart_vec) { 867 if (sndev->stdev->partition_count != 2) { 868 dev_err(&sndev->stdev->dev, 869 "ntb target partition not defined\n"); 870 return -ENODEV; 871 } 872 873 if (!part_map) { 874 dev_err(&sndev->stdev->dev, 875 "peer partition is not NT partition\n"); 876 return -ENODEV; 877 } 878 879 sndev->peer_partition = __ffs64(part_map); 880 } else { 881 if (__ffs64(tpart_vec) != (fls64(tpart_vec) - 1)) { 882 dev_err(&sndev->stdev->dev, 883 "ntb driver only supports 1 pair of 1-1 ntb mapping\n"); 884 return -ENODEV; 885 } 886 887 sndev->peer_partition = __ffs64(tpart_vec); 888 if (!(part_map & (1ULL << sndev->peer_partition))) { 889 dev_err(&sndev->stdev->dev, 890 "ntb target partition is not NT partition\n"); 891 return -ENODEV; 892 } 893 } 894 895 dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n", 896 sndev->self_partition, sndev->stdev->partition_count); 897 898 sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb + 899 SWITCHTEC_NTB_REG_CTRL_OFFSET; 900 sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb + 901 SWITCHTEC_NTB_REG_DBMSG_OFFSET; 902 903 sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition]; 904 sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition]; 905 sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition]; 906 sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg; 907 908 return 0; 909} 910 911static int config_rsvd_lut_win(struct switchtec_ntb *sndev, 912 struct ntb_ctrl_regs __iomem *ctl, 913 int lut_idx, int partition, u64 addr) 914{ 915 int peer_bar = sndev->peer_direct_mw_to_bar[0]; 916 u32 ctl_val; 917 int rc; 918 919 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 920 NTB_CTRL_PART_STATUS_LOCKED); 921 if (rc) 922 return rc; 923 924 ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl); 925 ctl_val &= 0xFF; 926 ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN; 927 ctl_val |= ilog2(LUT_SIZE) << 8; 928 ctl_val |= (sndev->nr_lut_mw - 1) << 14; 929 iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl); 930 931 iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr), 932 &ctl->lut_entry[lut_idx]); 933 934 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 935 NTB_CTRL_PART_STATUS_NORMAL); 936 if (rc) { 937 u32 bar_error, lut_error; 938 939 bar_error = ioread32(&ctl->bar_error); 940 lut_error = ioread32(&ctl->lut_error); 941 dev_err(&sndev->stdev->dev, 942 "Error setting up reserved lut window: %08x / %08x\n", 943 bar_error, lut_error); 944 return rc; 945 } 946 947 return 0; 948} 949 950static int config_req_id_table(struct switchtec_ntb *sndev, 951 struct ntb_ctrl_regs __iomem *mmio_ctrl, 952 int *req_ids, int count) 953{ 954 int i, rc = 0; 955 u32 error; 956 u32 proxy_id; 957 958 if (ioread16(&mmio_ctrl->req_id_table_size) < count) { 959 dev_err(&sndev->stdev->dev, 960 "Not enough requester IDs available.\n"); 961 return -EFAULT; 962 } 963 964 rc = switchtec_ntb_part_op(sndev, mmio_ctrl, 965 NTB_CTRL_PART_OP_LOCK, 966 NTB_CTRL_PART_STATUS_LOCKED); 967 if (rc) 968 return rc; 969 970 for (i = 0; i < count; i++) { 971 iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN, 972 &mmio_ctrl->req_id_table[i]); 973 974 proxy_id = ioread32(&mmio_ctrl->req_id_table[i]); 975 dev_dbg(&sndev->stdev->dev, 976 "Requester ID %02X:%02X.%X -> BB:%02X.%X\n", 977 req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F, 978 req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F, 979 (proxy_id >> 1) & 0x7); 980 } 981 982 rc = switchtec_ntb_part_op(sndev, mmio_ctrl, 983 NTB_CTRL_PART_OP_CFG, 984 NTB_CTRL_PART_STATUS_NORMAL); 985 986 if (rc == -EIO) { 987 error = ioread32(&mmio_ctrl->req_id_error); 988 dev_err(&sndev->stdev->dev, 989 "Error setting up the requester ID table: %08x\n", 990 error); 991 } 992 993 return 0; 994} 995 996static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx, 997 u64 *mw_addrs, int mw_count) 998{ 999 int rc, i; 1000 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl; 1001 u64 addr; 1002 size_t size, offset; 1003 int bar; 1004 int xlate_pos; 1005 u32 ctl_val; 1006 1007 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 1008 NTB_CTRL_PART_STATUS_LOCKED); 1009 if (rc) 1010 return rc; 1011 1012 for (i = 0; i < sndev->nr_lut_mw; i++) { 1013 if (i == ntb_lut_idx) 1014 continue; 1015 1016 addr = mw_addrs[0] + LUT_SIZE * i; 1017 1018 iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) | 1019 addr), 1020 &ctl->lut_entry[i]); 1021 } 1022 1023 sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count); 1024 1025 for (i = 0; i < sndev->nr_direct_mw; i++) { 1026 bar = sndev->direct_mw_to_bar[i]; 1027 offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0; 1028 addr = mw_addrs[i] + offset; 1029 size = pci_resource_len(sndev->ntb.pdev, bar) - offset; 1030 xlate_pos = ilog2(size); 1031 1032 if (offset && size > offset) 1033 size = offset; 1034 1035 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 1036 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; 1037 1038 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 1039 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000), 1040 &ctl->bar_entry[bar].win_size); 1041 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size); 1042 iowrite64(sndev->peer_partition | addr, 1043 &ctl->bar_entry[bar].xlate_addr); 1044 } 1045 1046 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 1047 NTB_CTRL_PART_STATUS_NORMAL); 1048 if (rc) { 1049 u32 bar_error, lut_error; 1050 1051 bar_error = ioread32(&ctl->bar_error); 1052 lut_error = ioread32(&ctl->lut_error); 1053 dev_err(&sndev->stdev->dev, 1054 "Error setting up cross link windows: %08x / %08x\n", 1055 bar_error, lut_error); 1056 return rc; 1057 } 1058 1059 return 0; 1060} 1061 1062static int crosslink_setup_req_ids(struct switchtec_ntb *sndev, 1063 struct ntb_ctrl_regs __iomem *mmio_ctrl) 1064{ 1065 int req_ids[16]; 1066 int i; 1067 u32 proxy_id; 1068 1069 for (i = 0; i < ARRAY_SIZE(req_ids); i++) { 1070 proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]); 1071 1072 if (!(proxy_id & NTB_CTRL_REQ_ID_EN)) 1073 break; 1074 1075 req_ids[i] = ((proxy_id >> 1) & 0xFF); 1076 } 1077 1078 return config_req_id_table(sndev, mmio_ctrl, req_ids, i); 1079} 1080 1081/* 1082 * In crosslink configuration there is a virtual partition in the 1083 * middle of the two switches. The BARs in this partition have to be 1084 * enumerated and assigned addresses. 1085 */ 1086static int crosslink_enum_partition(struct switchtec_ntb *sndev, 1087 u64 *bar_addrs) 1088{ 1089 struct part_cfg_regs __iomem *part_cfg = 1090 &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition]; 1091 u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF; 1092 struct pff_csr_regs __iomem *mmio_pff = 1093 &sndev->stdev->mmio_pff_csr[pff]; 1094 const u64 bar_space = 0x1000000000LL; 1095 u64 bar_addr; 1096 int bar_cnt = 0; 1097 int i; 1098 1099 iowrite16(0x6, &mmio_pff->pcicmd); 1100 1101 for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) { 1102 iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]); 1103 bar_addr = ioread64(&mmio_pff->pci_bar64[i]); 1104 bar_addr &= ~0xf; 1105 1106 dev_dbg(&sndev->stdev->dev, 1107 "Crosslink BAR%d addr: %llx\n", 1108 i*2, bar_addr); 1109 1110 if (bar_addr != bar_space * i) 1111 continue; 1112 1113 bar_addrs[bar_cnt++] = bar_addr; 1114 } 1115 1116 return bar_cnt; 1117} 1118 1119static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev) 1120{ 1121 int rc; 1122 int bar = sndev->direct_mw_to_bar[0]; 1123 const int ntb_lut_idx = 1; 1124 u64 bar_addrs[6]; 1125 u64 addr; 1126 int offset; 1127 int bar_cnt; 1128 1129 if (!crosslink_is_enabled(sndev)) 1130 return 0; 1131 1132 dev_info(&sndev->stdev->dev, "Using crosslink configuration\n"); 1133 sndev->ntb.topo = NTB_TOPO_CROSSLINK; 1134 1135 bar_cnt = crosslink_enum_partition(sndev, bar_addrs); 1136 if (bar_cnt < sndev->nr_direct_mw + 1) { 1137 dev_err(&sndev->stdev->dev, 1138 "Error enumerating crosslink partition\n"); 1139 return -EINVAL; 1140 } 1141 1142 addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET + 1143 SWITCHTEC_NTB_REG_DBMSG_OFFSET + 1144 sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition); 1145 1146 offset = addr & (LUT_SIZE - 1); 1147 addr -= offset; 1148 1149 rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx, 1150 sndev->peer_partition, addr); 1151 if (rc) 1152 return rc; 1153 1154 rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1], 1155 bar_cnt - 1); 1156 if (rc) 1157 return rc; 1158 1159 rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl); 1160 if (rc) 1161 return rc; 1162 1163 sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar, 1164 LUT_SIZE, LUT_SIZE); 1165 if (!sndev->mmio_xlink_win) { 1166 rc = -ENOMEM; 1167 return rc; 1168 } 1169 1170 sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset; 1171 sndev->nr_rsvd_luts++; 1172 1173 crosslink_init_dbmsgs(sndev); 1174 1175 return 0; 1176} 1177 1178static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev) 1179{ 1180 if (sndev->mmio_xlink_win) 1181 pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win); 1182} 1183 1184static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl) 1185{ 1186 int i; 1187 int cnt = 0; 1188 1189 for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) { 1190 u32 r = ioread32(&ctrl->bar_entry[i].ctl); 1191 1192 if (r & NTB_CTRL_BAR_VALID) 1193 map[cnt++] = i; 1194 } 1195 1196 return cnt; 1197} 1198 1199static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) 1200{ 1201 sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar, 1202 sndev->mmio_self_ctrl); 1203 1204 sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); 1205 sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); 1206 1207 dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n", 1208 sndev->nr_direct_mw, sndev->nr_lut_mw); 1209 1210 sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar, 1211 sndev->mmio_peer_ctrl); 1212 1213 sndev->peer_nr_lut_mw = 1214 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); 1215 sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); 1216 1217 dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n", 1218 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); 1219 1220} 1221 1222/* 1223 * There are 64 doorbells in the switch hardware but this is 1224 * shared among all partitions. So we must split them in half 1225 * (32 for each partition). However, the message interrupts are 1226 * also shared with the top 4 doorbells so we just limit this to 1227 * 28 doorbells per partition. 1228 * 1229 * In crosslink mode, each side has it's own dbmsg register so 1230 * they can each use all 60 of the available doorbells. 1231 */ 1232static void switchtec_ntb_init_db(struct switchtec_ntb *sndev) 1233{ 1234 sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL; 1235 1236 if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) { 1237 sndev->db_shift = 0; 1238 sndev->db_peer_shift = 0; 1239 sndev->db_valid_mask = sndev->db_mask; 1240 } else if (sndev->self_partition < sndev->peer_partition) { 1241 sndev->db_shift = 0; 1242 sndev->db_peer_shift = 32; 1243 sndev->db_valid_mask = 0x0FFFFFFF; 1244 } else { 1245 sndev->db_shift = 32; 1246 sndev->db_peer_shift = 0; 1247 sndev->db_valid_mask = 0x0FFFFFFF; 1248 } 1249 1250 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 1251 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 1252 &sndev->mmio_peer_dbmsg->odb_mask); 1253 1254 dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n", 1255 sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask); 1256} 1257 1258static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev) 1259{ 1260 int i; 1261 u32 msg_map = 0; 1262 1263 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 1264 int m = i | sndev->peer_partition << 2; 1265 1266 msg_map |= m << i * 8; 1267 } 1268 1269 iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map); 1270 1271 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) 1272 iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK, 1273 &sndev->mmio_self_dbmsg->imsg[i]); 1274} 1275 1276static int 1277switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev) 1278{ 1279 int req_ids[2]; 1280 1281 /* 1282 * Root Complex Requester ID (which is 0:00.0) 1283 */ 1284 req_ids[0] = 0; 1285 1286 /* 1287 * Host Bridge Requester ID (as read from the mmap address) 1288 */ 1289 req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id); 1290 1291 return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids, 1292 ARRAY_SIZE(req_ids)); 1293} 1294 1295static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) 1296{ 1297 int i; 1298 1299 memset(sndev->self_shared, 0, LUT_SIZE); 1300 sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC; 1301 sndev->self_shared->partition_id = sndev->stdev->partition; 1302 1303 for (i = 0; i < sndev->nr_direct_mw; i++) { 1304 int bar = sndev->direct_mw_to_bar[i]; 1305 resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar); 1306 1307 if (i == 0) 1308 sz = min_t(resource_size_t, sz, 1309 LUT_SIZE * sndev->nr_lut_mw); 1310 1311 sndev->self_shared->mw_sizes[i] = sz; 1312 } 1313 1314 for (i = 0; i < sndev->nr_lut_mw; i++) { 1315 int idx = sndev->nr_direct_mw + i; 1316 1317 sndev->self_shared->mw_sizes[idx] = LUT_SIZE; 1318 } 1319} 1320 1321static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev) 1322{ 1323 int self_bar = sndev->direct_mw_to_bar[0]; 1324 int rc; 1325 1326 sndev->nr_rsvd_luts++; 1327 sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev, 1328 LUT_SIZE, 1329 &sndev->self_shared_dma, 1330 GFP_KERNEL); 1331 if (!sndev->self_shared) { 1332 dev_err(&sndev->stdev->dev, 1333 "unable to allocate memory for shared mw\n"); 1334 return -ENOMEM; 1335 } 1336 1337 switchtec_ntb_init_shared(sndev); 1338 1339 rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0, 1340 sndev->self_partition, 1341 sndev->self_shared_dma); 1342 if (rc) 1343 goto unalloc_and_exit; 1344 1345 sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE); 1346 if (!sndev->peer_shared) { 1347 rc = -ENOMEM; 1348 goto unalloc_and_exit; 1349 } 1350 1351 dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n"); 1352 return 0; 1353 1354unalloc_and_exit: 1355 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1356 sndev->self_shared, sndev->self_shared_dma); 1357 1358 return rc; 1359} 1360 1361static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev) 1362{ 1363 if (sndev->peer_shared) 1364 pci_iounmap(sndev->stdev->pdev, sndev->peer_shared); 1365 1366 if (sndev->self_shared) 1367 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1368 sndev->self_shared, 1369 sndev->self_shared_dma); 1370 sndev->nr_rsvd_luts--; 1371} 1372 1373static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev) 1374{ 1375 struct switchtec_ntb *sndev = dev; 1376 1377 dev_dbg(&sndev->stdev->dev, "doorbell\n"); 1378 1379 ntb_db_event(&sndev->ntb, 0); 1380 1381 return IRQ_HANDLED; 1382} 1383 1384static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev) 1385{ 1386 int i; 1387 struct switchtec_ntb *sndev = dev; 1388 1389 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 1390 u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]); 1391 1392 if (msg & NTB_DBMSG_IMSG_STATUS) { 1393 dev_dbg(&sndev->stdev->dev, "message: %d %08x\n", 1394 i, (u32)msg); 1395 iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status); 1396 1397 if (i == LINK_MESSAGE) 1398 switchtec_ntb_check_link(sndev, msg); 1399 } 1400 } 1401 1402 return IRQ_HANDLED; 1403} 1404 1405static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev) 1406{ 1407 int i; 1408 int rc; 1409 int doorbell_irq = 0; 1410 int message_irq = 0; 1411 int event_irq; 1412 int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map); 1413 1414 event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number); 1415 1416 while (doorbell_irq == event_irq) 1417 doorbell_irq++; 1418 while (message_irq == doorbell_irq || 1419 message_irq == event_irq) 1420 message_irq++; 1421 1422 dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n", 1423 event_irq, doorbell_irq, message_irq); 1424 1425 for (i = 0; i < idb_vecs - 4; i++) 1426 iowrite8(doorbell_irq, 1427 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 1428 1429 for (; i < idb_vecs; i++) 1430 iowrite8(message_irq, 1431 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 1432 1433 sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq); 1434 sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq); 1435 1436 rc = request_irq(sndev->doorbell_irq, 1437 switchtec_ntb_doorbell_isr, 0, 1438 "switchtec_ntb_doorbell", sndev); 1439 if (rc) 1440 return rc; 1441 1442 rc = request_irq(sndev->message_irq, 1443 switchtec_ntb_message_isr, 0, 1444 "switchtec_ntb_message", sndev); 1445 if (rc) { 1446 free_irq(sndev->doorbell_irq, sndev); 1447 return rc; 1448 } 1449 1450 return 0; 1451} 1452 1453static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev) 1454{ 1455 free_irq(sndev->doorbell_irq, sndev); 1456 free_irq(sndev->message_irq, sndev); 1457} 1458 1459static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev) 1460{ 1461 int rc; 1462 1463 if (crosslink_is_enabled(sndev)) 1464 return 0; 1465 1466 dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n"); 1467 rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0, 1468 sndev->self_partition, 1469 sndev->self_shared_dma); 1470 return rc; 1471} 1472 1473static int switchtec_ntb_add(struct device *dev) 1474{ 1475 struct switchtec_dev *stdev = to_stdev(dev); 1476 struct switchtec_ntb *sndev; 1477 int rc; 1478 1479 stdev->sndev = NULL; 1480 1481 if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8)) 1482 return -ENODEV; 1483 1484 sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev)); 1485 if (!sndev) 1486 return -ENOMEM; 1487 1488 sndev->stdev = stdev; 1489 rc = switchtec_ntb_init_sndev(sndev); 1490 if (rc) 1491 goto free_and_exit; 1492 1493 switchtec_ntb_init_mw(sndev); 1494 1495 rc = switchtec_ntb_init_req_id_table(sndev); 1496 if (rc) 1497 goto free_and_exit; 1498 1499 rc = switchtec_ntb_init_crosslink(sndev); 1500 if (rc) 1501 goto free_and_exit; 1502 1503 switchtec_ntb_init_db(sndev); 1504 switchtec_ntb_init_msgs(sndev); 1505 1506 rc = switchtec_ntb_init_shared_mw(sndev); 1507 if (rc) 1508 goto deinit_crosslink; 1509 1510 rc = switchtec_ntb_init_db_msg_irq(sndev); 1511 if (rc) 1512 goto deinit_shared_and_exit; 1513 1514 /* 1515 * If this host crashed, the other host may think the link is 1516 * still up. Tell them to force it down (it will go back up 1517 * once we register the ntb device). 1518 */ 1519 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN); 1520 1521 rc = ntb_register_device(&sndev->ntb); 1522 if (rc) 1523 goto deinit_and_exit; 1524 1525 stdev->sndev = sndev; 1526 stdev->link_notifier = switchtec_ntb_link_notification; 1527 dev_info(dev, "NTB device registered\n"); 1528 1529 return 0; 1530 1531deinit_and_exit: 1532 switchtec_ntb_deinit_db_msg_irq(sndev); 1533deinit_shared_and_exit: 1534 switchtec_ntb_deinit_shared_mw(sndev); 1535deinit_crosslink: 1536 switchtec_ntb_deinit_crosslink(sndev); 1537free_and_exit: 1538 kfree(sndev); 1539 dev_err(dev, "failed to register ntb device: %d\n", rc); 1540 return rc; 1541} 1542 1543static void switchtec_ntb_remove(struct device *dev) 1544{ 1545 struct switchtec_dev *stdev = to_stdev(dev); 1546 struct switchtec_ntb *sndev = stdev->sndev; 1547 1548 if (!sndev) 1549 return; 1550 1551 stdev->link_notifier = NULL; 1552 stdev->sndev = NULL; 1553 ntb_unregister_device(&sndev->ntb); 1554 switchtec_ntb_deinit_db_msg_irq(sndev); 1555 switchtec_ntb_deinit_shared_mw(sndev); 1556 switchtec_ntb_deinit_crosslink(sndev); 1557 kfree(sndev); 1558 dev_info(dev, "ntb device unregistered\n"); 1559} 1560 1561static struct class_interface switchtec_interface = { 1562 .add_dev = switchtec_ntb_add, 1563 .remove_dev = switchtec_ntb_remove, 1564}; 1565 1566static int __init switchtec_ntb_init(void) 1567{ 1568 switchtec_interface.class = switchtec_class; 1569 return class_interface_register(&switchtec_interface); 1570} 1571module_init(switchtec_ntb_init); 1572 1573static void __exit switchtec_ntb_exit(void) 1574{ 1575 class_interface_unregister(&switchtec_interface); 1576} 1577module_exit(switchtec_ntb_exit); 1578