1/*- 2 * Copyright (c) 2017-2019 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27/* 28 * The Non-Transparent Bridge (NTB) is a device that allows you to connect 29 * two or more systems using a PCI-e links, providing remote memory access. 30 * 31 * This module contains a driver for NTBs in PLX/Avago/Broadcom PCIe bridges. 32 */ 33 34#include <sys/param.h> 35#include <sys/kernel.h> 36#include <sys/systm.h> 37#include <sys/bus.h> 38#include <sys/interrupt.h> 39#include <sys/module.h> 40#include <sys/rman.h> 41#include <sys/sysctl.h> 42#include <sys/taskqueue.h> 43#include <sys/tree.h> 44#include <vm/vm.h> 45#include <vm/pmap.h> 46#include <machine/bus.h> 47#include <machine/intr_machdep.h> 48#include <machine/resource.h> 49#include <dev/pci/pcireg.h> 50#include <dev/pci/pcivar.h> 51#include <dev/iommu/iommu.h> 52 53#include "../ntb.h" 54 55#define PLX_MAX_BARS 4 /* There are at most 4 data BARs. */ 56#define PLX_NUM_SPAD 8 /* There are 8 scratchpads. */ 57#define PLX_NUM_SPAD_PATT 4 /* Use test pattern as 4 more. */ 58#define PLX_NUM_DB 16 /* There are 16 doorbells. */ 59#define PLX_MAX_SPLIT 128 /* Allow are at most 128 splits. */ 60 61struct ntb_plx_mw_info { 62 int mw_bar; 63 int mw_64bit; 64 int mw_rid; 65 struct resource *mw_res; 66 vm_paddr_t mw_pbase; 67 caddr_t mw_vbase; 68 vm_size_t mw_size; 69 struct { 70 vm_memattr_t mw_map_mode; 71 bus_addr_t mw_xlat_addr; 72 bus_size_t mw_xlat_size; 73 } splits[PLX_MAX_SPLIT]; 74}; 75 76struct ntb_plx_softc { 77 /* ntb.c context. Do not move! Must go first! */ 78 void *ntb_store; 79 80 device_t dev; 81 struct resource *conf_res; 82 int conf_rid; 83 u_int ntx; /* NTx number within chip. */ 84 u_int link; /* Link v/s Virtual side. */ 85 u_int port; /* Port number within chip. */ 86 u_int alut; /* A-LUT is enabled for NTx */ 87 u_int split; /* split BAR2 into 2^x parts */ 88 89 int int_rid; 90 struct resource *int_res; 91 void *int_tag; 92 93 struct ntb_plx_mw_info mw_info[PLX_MAX_BARS]; 94 int mw_count; /* Number of memory windows. */ 95 96 int spad_count1; /* Number of standard spads. */ 97 int spad_count2; /* Number of extra spads. */ 98 uint32_t spad_off1; /* Offset of our spads. */ 99 uint32_t spad_off2; /* Offset of our extra spads. */ 100 uint32_t spad_offp1; /* Offset of peer spads. */ 101 uint32_t spad_offp2; /* Offset of peer extra spads. */ 102 103 /* Parameters of window shared with peer config access in B2B mode. */ 104 int b2b_mw; /* Shared window number. */ 105 uint64_t b2b_off; /* Offset in shared window. */ 106}; 107 108#define PLX_NT0_BASE 0x3E000 109#define PLX_NT1_BASE 0x3C000 110#define PLX_NTX_BASE(sc) ((sc)->ntx ? PLX_NT1_BASE : PLX_NT0_BASE) 111#define PLX_NTX_LINK_OFFSET 0x01000 112 113/* Bases of NTx our/peer interface registers */ 114#define PLX_NTX_OUR_BASE(sc) \ 115 (PLX_NTX_BASE(sc) + ((sc)->link ? PLX_NTX_LINK_OFFSET : 0)) 116#define PLX_NTX_PEER_BASE(sc) \ 117 (PLX_NTX_BASE(sc) + ((sc)->link ? 0 : PLX_NTX_LINK_OFFSET)) 118 119/* Read/write NTx our interface registers */ 120#define NTX_READ(sc, reg) \ 121 bus_read_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg)) 122#define NTX_WRITE(sc, reg, val) \ 123 bus_write_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg), (val)) 124 125/* Read/write NTx peer interface registers */ 126#define PNTX_READ(sc, reg) \ 127 bus_read_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg)) 128#define PNTX_WRITE(sc, reg, val) \ 129 bus_write_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg), (val)) 130 131/* Read/write B2B NTx registers */ 132#define BNTX_READ(sc, reg) \ 133 bus_read_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \ 134 PLX_NTX_BASE(sc) + (reg)) 135#define BNTX_WRITE(sc, reg, val) \ 136 bus_write_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \ 137 PLX_NTX_BASE(sc) + (reg), (val)) 138 139#define PLX_PORT_BASE(p) ((p) << 12) 140#define PLX_STATION_PORT_BASE(sc) PLX_PORT_BASE((sc)->port & ~7) 141 142#define PLX_PORT_CONTROL(sc) (PLX_STATION_PORT_BASE(sc) + 0x208) 143 144static int ntb_plx_init(device_t dev); 145static int ntb_plx_detach(device_t dev); 146static int ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx); 147 148static int 149ntb_plx_probe(device_t dev) 150{ 151 152 switch (pci_get_devid(dev)) { 153 case 0x87a010b5: 154 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Link"); 155 return (BUS_PROBE_DEFAULT); 156 case 0x87a110b5: 157 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Link"); 158 return (BUS_PROBE_DEFAULT); 159 case 0x87b010b5: 160 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Virtual"); 161 return (BUS_PROBE_DEFAULT); 162 case 0x87b110b5: 163 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Virtual"); 164 return (BUS_PROBE_DEFAULT); 165 } 166 return (ENXIO); 167} 168 169static int 170ntb_plx_init(device_t dev) 171{ 172 struct ntb_plx_softc *sc = device_get_softc(dev); 173 struct ntb_plx_mw_info *mw; 174 uint64_t val64; 175 int i; 176 uint32_t val; 177 178 if (sc->b2b_mw >= 0) { 179 /* Set peer BAR0/1 size and address for B2B NTx access. */ 180 mw = &sc->mw_info[sc->b2b_mw]; 181 if (mw->mw_64bit) { 182 PNTX_WRITE(sc, 0xe4, 0x3); /* 64-bit */ 183 val64 = 0x2000000000000000 * mw->mw_bar | 0x4; 184 PNTX_WRITE(sc, PCIR_BAR(0), val64); 185 PNTX_WRITE(sc, PCIR_BAR(0) + 4, val64 >> 32); 186 } else { 187 PNTX_WRITE(sc, 0xe4, 0x2); /* 32-bit */ 188 val = 0x20000000 * mw->mw_bar; 189 PNTX_WRITE(sc, PCIR_BAR(0), val); 190 } 191 192 /* Set Virtual to Link address translation for B2B. */ 193 for (i = 0; i < sc->mw_count; i++) { 194 mw = &sc->mw_info[i]; 195 if (mw->mw_64bit) { 196 val64 = 0x2000000000000000 * mw->mw_bar; 197 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val64); 198 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, val64 >> 32); 199 } else { 200 val = 0x20000000 * mw->mw_bar; 201 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val); 202 } 203 } 204 205 /* Make sure Virtual to Link A-LUT is disabled. */ 206 if (sc->alut) 207 PNTX_WRITE(sc, 0xc94, 0); 208 209 /* Enable all Link Interface LUT entries for peer. */ 210 for (i = 0; i < 32; i += 2) { 211 PNTX_WRITE(sc, 0xdb4 + i * 2, 212 0x00010001 | ((i + 1) << 19) | (i << 3)); 213 } 214 } 215 216 /* 217 * Enable Virtual Interface LUT entry 0 for 0:0.*. 218 * entry 1 for our Requester ID reported by the chip, 219 * entries 2-5 for 0/64/128/192:4.* of I/OAT DMA engines. 220 * XXX: Its a hack, we can't know all DMA engines, but this covers all 221 * I/OAT of Xeon E5/E7 at least from Sandy Bridge till Skylake I saw. 222 */ 223 val = (NTX_READ(sc, 0xc90) << 16) | 0x00010001; 224 NTX_WRITE(sc, sc->link ? 0xdb4 : 0xd94, val); 225 NTX_WRITE(sc, sc->link ? 0xdb8 : 0xd98, 0x40210021); 226 NTX_WRITE(sc, sc->link ? 0xdbc : 0xd9c, 0xc0218021); 227 228 /* Set Link to Virtual address translation. */ 229 for (i = 0; i < sc->mw_count; i++) 230 ntb_plx_mw_set_trans_internal(dev, i); 231 232 pci_enable_busmaster(dev); 233 if (sc->b2b_mw >= 0) 234 PNTX_WRITE(sc, PCIR_COMMAND, PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 235 236 return (0); 237} 238 239static void 240ntb_plx_isr(void *arg) 241{ 242 device_t dev = arg; 243 struct ntb_plx_softc *sc = device_get_softc(dev); 244 uint32_t val; 245 246 ntb_db_event((device_t)arg, 0); 247 248 if (sc->link) /* Link Interface has no Link Error registers. */ 249 return; 250 251 val = NTX_READ(sc, 0xfe0); 252 if (val == 0) 253 return; 254 NTX_WRITE(sc, 0xfe0, val); 255 if (val & 1) 256 device_printf(dev, "Correctable Error\n"); 257 if (val & 2) 258 device_printf(dev, "Uncorrectable Error\n"); 259 if (val & 4) { 260 /* DL_Down resets link side registers, have to reinit. */ 261 ntb_plx_init(dev); 262 ntb_link_event(dev); 263 } 264 if (val & 8) 265 device_printf(dev, "Uncorrectable Error Message Drop\n"); 266} 267 268static int 269ntb_plx_setup_intr(device_t dev) 270{ 271 struct ntb_plx_softc *sc = device_get_softc(dev); 272 int error; 273 274 /* 275 * XXX: This hardware supports MSI, but I found it unusable. 276 * It generates new MSI only when doorbell register goes from 277 * zero, but does not generate it when another bit is set or on 278 * partial clear. It makes operation very racy and unreliable. 279 * The data book mentions some mask juggling magic to workaround 280 * that, but I failed to make it work. 281 */ 282 sc->int_rid = 0; 283 sc->int_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 284 &sc->int_rid, RF_SHAREABLE|RF_ACTIVE); 285 if (sc->int_res == NULL) { 286 device_printf(dev, "bus_alloc_resource failed\n"); 287 return (ENOMEM); 288 } 289 error = bus_setup_intr(dev, sc->int_res, INTR_MPSAFE | INTR_TYPE_MISC, 290 NULL, ntb_plx_isr, dev, &sc->int_tag); 291 if (error != 0) { 292 device_printf(dev, "bus_setup_intr failed: %d\n", error); 293 return (error); 294 } 295 296 if (!sc->link) { /* Link Interface has no Link Error registers. */ 297 NTX_WRITE(sc, 0xfe0, 0xf); /* Clear link interrupts. */ 298 NTX_WRITE(sc, 0xfe4, 0x0); /* Unmask link interrupts. */ 299 } 300 return (0); 301} 302 303static void 304ntb_plx_teardown_intr(device_t dev) 305{ 306 struct ntb_plx_softc *sc = device_get_softc(dev); 307 308 if (!sc->link) /* Link Interface has no Link Error registers. */ 309 NTX_WRITE(sc, 0xfe4, 0xf); /* Mask link interrupts. */ 310 311 if (sc->int_res) { 312 bus_teardown_intr(dev, sc->int_res, sc->int_tag); 313 bus_release_resource(dev, SYS_RES_IRQ, sc->int_rid, 314 sc->int_res); 315 } 316} 317 318static int 319ntb_plx_attach(device_t dev) 320{ 321 struct ntb_plx_softc *sc = device_get_softc(dev); 322 struct ntb_plx_mw_info *mw; 323 int error = 0, i, j; 324 uint32_t val; 325 char buf[32]; 326 327 /* Identify what we are (what side of what NTx). */ 328 sc->dev = dev; 329 val = pci_read_config(dev, 0xc8c, 4); 330 sc->ntx = (val & 1) != 0; 331 sc->link = (val & 0x80000000) != 0; 332 333 /* Get access to whole 256KB of chip configuration space via BAR0/1. */ 334 sc->conf_rid = PCIR_BAR(0); 335 sc->conf_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 336 &sc->conf_rid, RF_ACTIVE); 337 if (sc->conf_res == NULL) { 338 device_printf(dev, "Can't allocate configuration BAR.\n"); 339 return (ENXIO); 340 } 341 342 /* 343 * The device occupies whole bus. In translated TLP slot field 344 * keeps LUT index (original bus/slot), function is passed through. 345 */ 346 bus_dma_iommu_set_buswide(dev); 347 348 /* Identify chip port we are connected to. */ 349 val = bus_read_4(sc->conf_res, 0x360); 350 sc->port = (val >> ((sc->ntx == 0) ? 8 : 16)) & 0x1f; 351 352 /* Detect A-LUT enable and size. */ 353 val >>= 30; 354 sc->alut = (val == 0x3) ? 1 : ((val & (1 << sc->ntx)) ? 2 : 0); 355 if (sc->alut) 356 device_printf(dev, "%u A-LUT entries\n", 128 * sc->alut); 357 358 /* Find configured memory windows at BAR2-5. */ 359 sc->mw_count = 0; 360 for (i = 2; i <= 5; i++) { 361 mw = &sc->mw_info[sc->mw_count]; 362 mw->mw_bar = i; 363 mw->mw_rid = PCIR_BAR(mw->mw_bar); 364 mw->mw_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 365 &mw->mw_rid, RF_ACTIVE); 366 if (mw->mw_res == NULL) 367 continue; 368 mw->mw_pbase = rman_get_start(mw->mw_res); 369 mw->mw_size = rman_get_size(mw->mw_res); 370 mw->mw_vbase = rman_get_virtual(mw->mw_res); 371 for (j = 0; j < PLX_MAX_SPLIT; j++) 372 mw->splits[j].mw_map_mode = VM_MEMATTR_UNCACHEABLE; 373 sc->mw_count++; 374 375 /* Skip over adjacent BAR for 64-bit BARs. */ 376 val = pci_read_config(dev, PCIR_BAR(mw->mw_bar), 4); 377 if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) { 378 mw->mw_64bit = 1; 379 i++; 380 } 381 } 382 383 /* Try to identify B2B mode. */ 384 i = 1; 385 snprintf(buf, sizeof(buf), "hint.%s.%d.b2b", device_get_name(dev), 386 device_get_unit(dev)); 387 TUNABLE_INT_FETCH(buf, &i); 388 if (sc->link) { 389 device_printf(dev, "NTB-to-Root Port mode (Link Interface)\n"); 390 sc->b2b_mw = -1; 391 } else if (i == 0) { 392 device_printf(dev, "NTB-to-Root Port mode (Virtual Interface)\n"); 393 sc->b2b_mw = -1; 394 } else { 395 device_printf(dev, "NTB-to-NTB (back-to-back) mode\n"); 396 397 /* We need at least one memory window for B2B peer access. */ 398 if (sc->mw_count == 0) { 399 device_printf(dev, "No memory window BARs enabled.\n"); 400 error = ENXIO; 401 goto out; 402 } 403 sc->b2b_mw = sc->mw_count - 1; 404 405 /* Use half of the window for B2B, but no less then 1MB. */ 406 mw = &sc->mw_info[sc->b2b_mw]; 407 if (mw->mw_size >= 2 * 1024 * 1024) 408 sc->b2b_off = mw->mw_size / 2; 409 else 410 sc->b2b_off = 0; 411 } 412 413 snprintf(buf, sizeof(buf), "hint.%s.%d.split", device_get_name(dev), 414 device_get_unit(dev)); 415 TUNABLE_INT_FETCH(buf, &sc->split); 416 if (sc->split > 7) { 417 device_printf(dev, "Split value is too high (%u)\n", sc->split); 418 sc->split = 0; 419 } else if (sc->split > 0 && sc->alut == 0) { 420 device_printf(dev, "Can't split with disabled A-LUT\n"); 421 sc->split = 0; 422 } else if (sc->split > 0 && (sc->mw_count == 0 || sc->mw_info[0].mw_bar != 2)) { 423 device_printf(dev, "Can't split disabled BAR2\n"); 424 sc->split = 0; 425 } else if (sc->split > 0 && (sc->b2b_mw == 0 && sc->b2b_off == 0)) { 426 device_printf(dev, "Can't split BAR2 consumed by B2B\n"); 427 sc->split = 0; 428 } else if (sc->split > 0) { 429 device_printf(dev, "Splitting BAR2 into %d memory windows\n", 430 1 << sc->split); 431 } 432 433 /* 434 * Use Physical Layer User Test Pattern as additional scratchpad. 435 * Make sure they are present and enabled by writing to them. 436 * XXX: Its a hack, but standard 8 registers are not enough. 437 */ 438 sc->spad_offp1 = sc->spad_off1 = PLX_NTX_OUR_BASE(sc) + 0xc6c; 439 sc->spad_offp2 = sc->spad_off2 = PLX_PORT_BASE(sc->ntx * 8) + 0x20c; 440 if (sc->b2b_mw >= 0) { 441 /* In NTB-to-NTB mode each side has own scratchpads. */ 442 sc->spad_count1 = PLX_NUM_SPAD; 443 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678); 444 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678) 445 sc->spad_count2 = PLX_NUM_SPAD_PATT; 446 } else { 447 /* Otherwise we have share scratchpads with the peer. */ 448 if (sc->link) { 449 sc->spad_off1 += PLX_NUM_SPAD / 2 * 4; 450 sc->spad_off2 += PLX_NUM_SPAD_PATT / 2 * 4; 451 } else { 452 sc->spad_offp1 += PLX_NUM_SPAD / 2 * 4; 453 sc->spad_offp2 += PLX_NUM_SPAD_PATT / 2 * 4; 454 } 455 sc->spad_count1 = PLX_NUM_SPAD / 2; 456 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678); 457 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678) 458 sc->spad_count2 = PLX_NUM_SPAD_PATT / 2; 459 } 460 461 /* Apply static part of NTB configuration. */ 462 ntb_plx_init(dev); 463 464 /* Allocate and setup interrupts. */ 465 error = ntb_plx_setup_intr(dev); 466 if (error) 467 goto out; 468 469 /* Attach children to this controller */ 470 error = ntb_register_device(dev); 471 472out: 473 if (error != 0) 474 ntb_plx_detach(dev); 475 return (error); 476} 477 478static int 479ntb_plx_detach(device_t dev) 480{ 481 struct ntb_plx_softc *sc = device_get_softc(dev); 482 struct ntb_plx_mw_info *mw; 483 int i; 484 485 /* Detach & delete all children */ 486 ntb_unregister_device(dev); 487 488 /* Disable and free interrupts. */ 489 ntb_plx_teardown_intr(dev); 490 491 /* Free memory resources. */ 492 for (i = 0; i < sc->mw_count; i++) { 493 mw = &sc->mw_info[i]; 494 bus_release_resource(dev, SYS_RES_MEMORY, mw->mw_rid, 495 mw->mw_res); 496 } 497 bus_release_resource(dev, SYS_RES_MEMORY, sc->conf_rid, sc->conf_res); 498 return (0); 499} 500 501static int 502ntb_plx_port_number(device_t dev) 503{ 504 struct ntb_plx_softc *sc = device_get_softc(dev); 505 506 return (sc->link ? 1 : 0); 507} 508 509static int 510ntb_plx_peer_port_count(device_t dev) 511{ 512 513 return (1); 514} 515 516static int 517ntb_plx_peer_port_number(device_t dev, int pidx) 518{ 519 struct ntb_plx_softc *sc = device_get_softc(dev); 520 521 if (pidx != 0) 522 return (-EINVAL); 523 524 return (sc->link ? 0 : 1); 525} 526 527static int 528ntb_plx_peer_port_idx(device_t dev, int port) 529{ 530 int peer_port; 531 532 peer_port = ntb_plx_peer_port_number(dev, 0); 533 if (peer_port == -EINVAL || port != peer_port) 534 return (-EINVAL); 535 536 return (0); 537} 538 539static bool 540ntb_plx_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 541{ 542 uint16_t link; 543 544 link = pcie_read_config(dev, PCIER_LINK_STA, 2); 545 if (speed != NULL) 546 *speed = (link & PCIEM_LINK_STA_SPEED); 547 if (width != NULL) 548 *width = (link & PCIEM_LINK_STA_WIDTH) >> 4; 549 return ((link & PCIEM_LINK_STA_WIDTH) != 0); 550} 551 552static int 553ntb_plx_link_enable(device_t dev, enum ntb_speed speed __unused, 554 enum ntb_width width __unused) 555{ 556 struct ntb_plx_softc *sc = device_get_softc(dev); 557 uint32_t reg, val; 558 559 /* The fact that we see the Link Interface means link is enabled. */ 560 if (sc->link) { 561 ntb_link_event(dev); 562 return (0); 563 } 564 565 reg = PLX_PORT_CONTROL(sc); 566 val = bus_read_4(sc->conf_res, reg); 567 if ((val & (1 << (sc->port & 7))) == 0) { 568 /* If already enabled, generate fake link event and exit. */ 569 ntb_link_event(dev); 570 return (0); 571 } 572 val &= ~(1 << (sc->port & 7)); 573 bus_write_4(sc->conf_res, reg, val); 574 return (0); 575} 576 577static int 578ntb_plx_link_disable(device_t dev) 579{ 580 struct ntb_plx_softc *sc = device_get_softc(dev); 581 uint32_t reg, val; 582 583 /* Link disable for Link Interface would be suicidal. */ 584 if (sc->link) 585 return (0); 586 587 reg = PLX_PORT_CONTROL(sc); 588 val = bus_read_4(sc->conf_res, reg); 589 val |= (1 << (sc->port & 7)); 590 bus_write_4(sc->conf_res, reg, val); 591 return (0); 592} 593 594static bool 595ntb_plx_link_enabled(device_t dev) 596{ 597 struct ntb_plx_softc *sc = device_get_softc(dev); 598 uint32_t reg, val; 599 600 /* The fact that we see the Link Interface means link is enabled. */ 601 if (sc->link) 602 return (TRUE); 603 604 reg = PLX_PORT_CONTROL(sc); 605 val = bus_read_4(sc->conf_res, reg); 606 return ((val & (1 << (sc->port & 7))) == 0); 607} 608 609static uint8_t 610ntb_plx_mw_count(device_t dev) 611{ 612 struct ntb_plx_softc *sc = device_get_softc(dev); 613 uint8_t res; 614 615 res = sc->mw_count; 616 res += (1 << sc->split) - 1; 617 if (sc->b2b_mw >= 0 && sc->b2b_off == 0) 618 res--; /* B2B consumed whole window. */ 619 return (res); 620} 621 622static unsigned 623ntb_plx_user_mw_to_idx(struct ntb_plx_softc *sc, unsigned uidx, unsigned *sp) 624{ 625 unsigned t; 626 627 t = 1 << sc->split; 628 if (uidx < t) { 629 *sp = uidx; 630 return (0); 631 } 632 *sp = 0; 633 return (uidx - (t - 1)); 634} 635 636static int 637ntb_plx_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 638 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 639 bus_addr_t *plimit) 640{ 641 struct ntb_plx_softc *sc = device_get_softc(dev); 642 struct ntb_plx_mw_info *mw; 643 size_t off, ss; 644 unsigned sp, split; 645 646 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 647 if (mw_idx >= sc->mw_count) 648 return (EINVAL); 649 off = 0; 650 if (mw_idx == sc->b2b_mw) { 651 KASSERT(sc->b2b_off != 0, 652 ("user shouldn't get non-shared b2b mw")); 653 off = sc->b2b_off; 654 } 655 mw = &sc->mw_info[mw_idx]; 656 split = (mw->mw_bar == 2) ? sc->split : 0; 657 ss = (mw->mw_size - off) >> split; 658 659 /* Local to remote memory window parameters. */ 660 if (base != NULL) 661 *base = mw->mw_pbase + off + ss * sp; 662 if (vbase != NULL) 663 *vbase = mw->mw_vbase + off + ss * sp; 664 if (size != NULL) 665 *size = ss; 666 667 /* 668 * Remote to local memory window translation address alignment. 669 * Translation address has to be aligned to the BAR size, but A-LUT 670 * entries re-map addresses can be aligned to 1/128 or 1/256 of it. 671 * XXX: In B2B mode we can change BAR size (and so alignmet) live, 672 * but there is no way to report it here, so report safe value. 673 */ 674 if (align != NULL) { 675 if (sc->alut && mw->mw_bar == 2) 676 *align = (mw->mw_size - off) / 128 / sc->alut; 677 else 678 *align = mw->mw_size - off; 679 } 680 681 /* 682 * Remote to local memory window size alignment. 683 * The chip has no limit registers, but A-LUT, when available, allows 684 * access control with granularity of 1/128 or 1/256 of the BAR size. 685 * XXX: In B2B case we can change BAR size live, but there is no way 686 * to report it, so report half of the BAR size, that should be safe. 687 * In non-B2B case there is no control at all, so report the BAR size. 688 */ 689 if (align_size != NULL) { 690 if (sc->alut && mw->mw_bar == 2) 691 *align_size = (mw->mw_size - off) / 128 / sc->alut; 692 else if (sc->b2b_mw >= 0) 693 *align_size = (mw->mw_size - off) / 2; 694 else 695 *align_size = mw->mw_size - off; 696 } 697 698 /* Remote to local memory window translation address upper limit. */ 699 if (plimit != NULL) 700 *plimit = mw->mw_64bit ? BUS_SPACE_MAXADDR : 701 BUS_SPACE_MAXADDR_32BIT; 702 return (0); 703} 704 705static int 706ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx) 707{ 708 struct ntb_plx_softc *sc = device_get_softc(dev); 709 struct ntb_plx_mw_info *mw; 710 uint64_t addr, eaddr, off, size, bsize, esize, val64; 711 uint32_t val; 712 unsigned i, sp, split; 713 714 mw = &sc->mw_info[mw_idx]; 715 off = (mw_idx == sc->b2b_mw) ? sc->b2b_off : 0; 716 split = (mw->mw_bar == 2) ? sc->split : 0; 717 718 /* Get BAR size. In case of split or B2RP we can't change it. */ 719 if (split || sc->b2b_mw < 0) { 720 bsize = mw->mw_size - off; 721 } else { 722 bsize = mw->splits[0].mw_xlat_size; 723 if (!powerof2(bsize)) 724 bsize = 1LL << flsll(bsize); 725 if (bsize > 0 && bsize < 1024 * 1024) 726 bsize = 1024 * 1024; 727 } 728 729 /* 730 * While for B2B we can set any BAR size on a link side, for shared 731 * window we can't go above preconfigured size due to BAR address 732 * alignment requirements. 733 */ 734 if ((off & (bsize - 1)) != 0) 735 return (EINVAL); 736 737 /* In B2B mode set Link Interface BAR size/address. */ 738 if (sc->b2b_mw >= 0 && mw->mw_64bit) { 739 val64 = 0; 740 if (bsize > 0) 741 val64 = (~(bsize - 1) & ~0xfffff); 742 val64 |= 0xc; 743 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val64); 744 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4 + 4, val64 >> 32); 745 746 val64 = 0x2000000000000000 * mw->mw_bar + off; 747 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64); 748 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar) + 4, val64 >> 32); 749 } else if (sc->b2b_mw >= 0) { 750 val = 0; 751 if (bsize > 0) 752 val = (~(bsize - 1) & ~0xfffff); 753 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val); 754 755 val64 = 0x20000000 * mw->mw_bar + off; 756 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64); 757 } 758 759 /* Set BARs address translation */ 760 addr = split ? UINT64_MAX : mw->splits[0].mw_xlat_addr; 761 if (mw->mw_64bit) { 762 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr); 763 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, addr >> 32); 764 } else { 765 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr); 766 } 767 768 /* Configure and enable A-LUT if we need it. */ 769 size = split ? 0 : mw->splits[0].mw_xlat_size; 770 if (sc->alut && mw->mw_bar == 2 && (sc->split > 0 || 771 ((addr & (bsize - 1)) != 0 || size != bsize))) { 772 esize = bsize / (128 * sc->alut); 773 for (i = sp = 0; i < 128 * sc->alut; i++) { 774 if (i % (128 * sc->alut >> sc->split) == 0) { 775 eaddr = addr = mw->splits[sp].mw_xlat_addr; 776 size = mw->splits[sp++].mw_xlat_size; 777 } 778 val = sc->link ? 0 : 1; 779 if (sc->alut == 1) 780 val += 2 * sc->ntx; 781 val *= 0x1000 * sc->alut; 782 val += 0x38000 + i * 4 + (i >= 128 ? 0x0e00 : 0); 783 bus_write_4(sc->conf_res, val, eaddr); 784 bus_write_4(sc->conf_res, val + 0x400, eaddr >> 32); 785 bus_write_4(sc->conf_res, val + 0x800, 786 (eaddr < addr + size) ? 0x3 : 0); 787 eaddr += esize; 788 } 789 NTX_WRITE(sc, 0xc94, 0x10000000); 790 } else if (sc->alut && mw->mw_bar == 2) 791 NTX_WRITE(sc, 0xc94, 0); 792 793 return (0); 794} 795 796static int 797ntb_plx_mw_set_trans(device_t dev, unsigned mw_idx, bus_addr_t addr, size_t size) 798{ 799 struct ntb_plx_softc *sc = device_get_softc(dev); 800 struct ntb_plx_mw_info *mw; 801 unsigned sp; 802 803 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 804 if (mw_idx >= sc->mw_count) 805 return (EINVAL); 806 mw = &sc->mw_info[mw_idx]; 807 if (!mw->mw_64bit && 808 ((addr & UINT32_MAX) != addr || 809 ((addr + size) & UINT32_MAX) != (addr + size))) 810 return (ERANGE); 811 mw->splits[sp].mw_xlat_addr = addr; 812 mw->splits[sp].mw_xlat_size = size; 813 return (ntb_plx_mw_set_trans_internal(dev, mw_idx)); 814} 815 816static int 817ntb_plx_mw_clear_trans(device_t dev, unsigned mw_idx) 818{ 819 820 return (ntb_plx_mw_set_trans(dev, mw_idx, 0, 0)); 821} 822 823static int 824ntb_plx_mw_get_wc(device_t dev, unsigned mw_idx, vm_memattr_t *mode) 825{ 826 struct ntb_plx_softc *sc = device_get_softc(dev); 827 struct ntb_plx_mw_info *mw; 828 unsigned sp; 829 830 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 831 if (mw_idx >= sc->mw_count) 832 return (EINVAL); 833 mw = &sc->mw_info[mw_idx]; 834 *mode = mw->splits[sp].mw_map_mode; 835 return (0); 836} 837 838static int 839ntb_plx_mw_set_wc(device_t dev, unsigned mw_idx, vm_memattr_t mode) 840{ 841 struct ntb_plx_softc *sc = device_get_softc(dev); 842 struct ntb_plx_mw_info *mw; 843 uint64_t off, ss; 844 int rc; 845 unsigned sp, split; 846 847 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 848 if (mw_idx >= sc->mw_count) 849 return (EINVAL); 850 mw = &sc->mw_info[mw_idx]; 851 if (mw->splits[sp].mw_map_mode == mode) 852 return (0); 853 854 off = 0; 855 if (mw_idx == sc->b2b_mw) { 856 KASSERT(sc->b2b_off != 0, 857 ("user shouldn't get non-shared b2b mw")); 858 off = sc->b2b_off; 859 } 860 861 split = (mw->mw_bar == 2) ? sc->split : 0; 862 ss = (mw->mw_size - off) >> split; 863 rc = pmap_change_attr((vm_offset_t)mw->mw_vbase + off + ss * sp, 864 ss, mode); 865 if (rc == 0) 866 mw->splits[sp].mw_map_mode = mode; 867 return (rc); 868} 869 870static uint8_t 871ntb_plx_spad_count(device_t dev) 872{ 873 struct ntb_plx_softc *sc = device_get_softc(dev); 874 875 return (sc->spad_count1 + sc->spad_count2); 876} 877 878static int 879ntb_plx_spad_write(device_t dev, unsigned int idx, uint32_t val) 880{ 881 struct ntb_plx_softc *sc = device_get_softc(dev); 882 u_int off, t; 883 884 if (idx >= sc->spad_count1 + sc->spad_count2) 885 return (EINVAL); 886 887 if (idx < sc->spad_count1) { 888 off = sc->spad_off1 + idx * 4; 889 bus_write_4(sc->conf_res, off, val); 890 return (0); 891 } else { 892 off = sc->spad_off2 + (idx - sc->spad_count1) * 4; 893 /* 894 * For some reason when link goes down Test Pattern registers 895 * we use as additional scratchpad become read-only for about 896 * 100us. I see no explanation in specs, so just wait a bit. 897 */ 898 for (t = 0; t <= 1000; t++) { 899 bus_write_4(sc->conf_res, off, val); 900 if (bus_read_4(sc->conf_res, off) == val) 901 return (0); 902 DELAY(1); 903 } 904 device_printf(dev, 905 "Can't write Physical Layer User Test Pattern (0x%x)\n", 906 off); 907 return (EIO); 908 } 909} 910 911static void 912ntb_plx_spad_clear(device_t dev) 913{ 914 struct ntb_plx_softc *sc = device_get_softc(dev); 915 int i; 916 917 for (i = 0; i < sc->spad_count1 + sc->spad_count2; i++) 918 ntb_plx_spad_write(dev, i, 0); 919} 920 921static int 922ntb_plx_spad_read(device_t dev, unsigned int idx, uint32_t *val) 923{ 924 struct ntb_plx_softc *sc = device_get_softc(dev); 925 u_int off; 926 927 if (idx >= sc->spad_count1 + sc->spad_count2) 928 return (EINVAL); 929 930 if (idx < sc->spad_count1) 931 off = sc->spad_off1 + idx * 4; 932 else 933 off = sc->spad_off2 + (idx - sc->spad_count1) * 4; 934 *val = bus_read_4(sc->conf_res, off); 935 return (0); 936} 937 938static int 939ntb_plx_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 940{ 941 struct ntb_plx_softc *sc = device_get_softc(dev); 942 u_int off; 943 944 if (idx >= sc->spad_count1 + sc->spad_count2) 945 return (EINVAL); 946 947 if (idx < sc->spad_count1) 948 off = sc->spad_offp1 + idx * 4; 949 else 950 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4; 951 if (sc->b2b_mw >= 0) 952 bus_write_4(sc->mw_info[sc->b2b_mw].mw_res, off, val); 953 else 954 bus_write_4(sc->conf_res, off, val); 955 return (0); 956} 957 958static int 959ntb_plx_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 960{ 961 struct ntb_plx_softc *sc = device_get_softc(dev); 962 u_int off; 963 964 if (idx >= sc->spad_count1 + sc->spad_count2) 965 return (EINVAL); 966 967 if (idx < sc->spad_count1) 968 off = sc->spad_offp1 + idx * 4; 969 else 970 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4; 971 if (sc->b2b_mw >= 0) 972 *val = bus_read_4(sc->mw_info[sc->b2b_mw].mw_res, off); 973 else 974 *val = bus_read_4(sc->conf_res, off); 975 return (0); 976} 977 978static uint64_t 979ntb_plx_db_valid_mask(device_t dev) 980{ 981 982 return ((1LL << PLX_NUM_DB) - 1); 983} 984 985static int 986ntb_plx_db_vector_count(device_t dev) 987{ 988 989 return (1); 990} 991 992static uint64_t 993ntb_plx_db_vector_mask(device_t dev, uint32_t vector) 994{ 995 996 if (vector > 0) 997 return (0); 998 return ((1LL << PLX_NUM_DB) - 1); 999} 1000 1001static void 1002ntb_plx_db_clear(device_t dev, uint64_t bits) 1003{ 1004 struct ntb_plx_softc *sc = device_get_softc(dev); 1005 1006 NTX_WRITE(sc, sc->link ? 0xc60 : 0xc50, bits); 1007} 1008 1009static void 1010ntb_plx_db_clear_mask(device_t dev, uint64_t bits) 1011{ 1012 struct ntb_plx_softc *sc = device_get_softc(dev); 1013 1014 NTX_WRITE(sc, sc->link ? 0xc68 : 0xc58, bits); 1015} 1016 1017static uint64_t 1018ntb_plx_db_read(device_t dev) 1019{ 1020 struct ntb_plx_softc *sc = device_get_softc(dev); 1021 1022 return (NTX_READ(sc, sc->link ? 0xc5c : 0xc4c)); 1023} 1024 1025static void 1026ntb_plx_db_set_mask(device_t dev, uint64_t bits) 1027{ 1028 struct ntb_plx_softc *sc = device_get_softc(dev); 1029 1030 NTX_WRITE(sc, sc->link ? 0xc64 : 0xc54, bits); 1031} 1032 1033static int 1034ntb_plx_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 1035{ 1036 struct ntb_plx_softc *sc = device_get_softc(dev); 1037 struct ntb_plx_mw_info *mw; 1038 1039 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 1040 1041 if (sc->b2b_mw >= 0) { 1042 mw = &sc->mw_info[sc->b2b_mw]; 1043 *db_addr = (uint64_t)mw->mw_pbase + PLX_NTX_BASE(sc) + 0xc4c; 1044 } else { 1045 *db_addr = rman_get_start(sc->conf_res) + PLX_NTX_BASE(sc); 1046 *db_addr += sc->link ? 0xc4c : 0xc5c; 1047 } 1048 *db_size = 4; 1049 return (0); 1050} 1051 1052static void 1053ntb_plx_peer_db_set(device_t dev, uint64_t bit) 1054{ 1055 struct ntb_plx_softc *sc = device_get_softc(dev); 1056 1057 if (sc->b2b_mw >= 0) 1058 BNTX_WRITE(sc, 0xc4c, bit); 1059 else 1060 NTX_WRITE(sc, sc->link ? 0xc4c : 0xc5c, bit); 1061} 1062 1063static device_method_t ntb_plx_methods[] = { 1064 /* Device interface */ 1065 DEVMETHOD(device_probe, ntb_plx_probe), 1066 DEVMETHOD(device_attach, ntb_plx_attach), 1067 DEVMETHOD(device_detach, ntb_plx_detach), 1068 /* Bus interface */ 1069 DEVMETHOD(bus_child_location, ntb_child_location), 1070 DEVMETHOD(bus_print_child, ntb_print_child), 1071 DEVMETHOD(bus_get_dma_tag, ntb_get_dma_tag), 1072 /* NTB interface */ 1073 DEVMETHOD(ntb_port_number, ntb_plx_port_number), 1074 DEVMETHOD(ntb_peer_port_count, ntb_plx_peer_port_count), 1075 DEVMETHOD(ntb_peer_port_number, ntb_plx_peer_port_number), 1076 DEVMETHOD(ntb_peer_port_idx, ntb_plx_peer_port_idx), 1077 DEVMETHOD(ntb_link_is_up, ntb_plx_link_is_up), 1078 DEVMETHOD(ntb_link_enable, ntb_plx_link_enable), 1079 DEVMETHOD(ntb_link_disable, ntb_plx_link_disable), 1080 DEVMETHOD(ntb_link_enabled, ntb_plx_link_enabled), 1081 DEVMETHOD(ntb_mw_count, ntb_plx_mw_count), 1082 DEVMETHOD(ntb_mw_get_range, ntb_plx_mw_get_range), 1083 DEVMETHOD(ntb_mw_set_trans, ntb_plx_mw_set_trans), 1084 DEVMETHOD(ntb_mw_clear_trans, ntb_plx_mw_clear_trans), 1085 DEVMETHOD(ntb_mw_get_wc, ntb_plx_mw_get_wc), 1086 DEVMETHOD(ntb_mw_set_wc, ntb_plx_mw_set_wc), 1087 DEVMETHOD(ntb_spad_count, ntb_plx_spad_count), 1088 DEVMETHOD(ntb_spad_clear, ntb_plx_spad_clear), 1089 DEVMETHOD(ntb_spad_write, ntb_plx_spad_write), 1090 DEVMETHOD(ntb_spad_read, ntb_plx_spad_read), 1091 DEVMETHOD(ntb_peer_spad_write, ntb_plx_peer_spad_write), 1092 DEVMETHOD(ntb_peer_spad_read, ntb_plx_peer_spad_read), 1093 DEVMETHOD(ntb_db_valid_mask, ntb_plx_db_valid_mask), 1094 DEVMETHOD(ntb_db_vector_count, ntb_plx_db_vector_count), 1095 DEVMETHOD(ntb_db_vector_mask, ntb_plx_db_vector_mask), 1096 DEVMETHOD(ntb_db_clear, ntb_plx_db_clear), 1097 DEVMETHOD(ntb_db_clear_mask, ntb_plx_db_clear_mask), 1098 DEVMETHOD(ntb_db_read, ntb_plx_db_read), 1099 DEVMETHOD(ntb_db_set_mask, ntb_plx_db_set_mask), 1100 DEVMETHOD(ntb_peer_db_addr, ntb_plx_peer_db_addr), 1101 DEVMETHOD(ntb_peer_db_set, ntb_plx_peer_db_set), 1102 DEVMETHOD_END 1103}; 1104 1105static DEFINE_CLASS_0(ntb_hw, ntb_plx_driver, ntb_plx_methods, 1106 sizeof(struct ntb_plx_softc)); 1107DRIVER_MODULE(ntb_hw_plx, pci, ntb_plx_driver, NULL, NULL); 1108MODULE_DEPEND(ntb_hw_plx, ntb, 1, 1, 1); 1109MODULE_VERSION(ntb_hw_plx, 1); 1110