1/*- 2 * Copyright (c) 2017-2019 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27/* 28 * The Non-Transparent Bridge (NTB) is a device that allows you to connect 29 * two or more systems using a PCI-e links, providing remote memory access. 30 * 31 * This module contains a driver for NTBs in PLX/Avago/Broadcom PCIe bridges. 32 */ 33 34#include <sys/cdefs.h> 35__FBSDID("$FreeBSD: stable/11/sys/dev/ntb/ntb_hw/ntb_hw_plx.c 355152 2019-11-28 00:41:42Z mav $"); 36 37#include <sys/param.h> 38#include <sys/kernel.h> 39#include <sys/systm.h> 40#include <sys/bus.h> 41#include <sys/interrupt.h> 42#include <sys/module.h> 43#include <sys/rman.h> 44#include <sys/sysctl.h> 45#include <vm/vm.h> 46#include <vm/pmap.h> 47#include <machine/bus.h> 48#include <machine/intr_machdep.h> 49#include <machine/resource.h> 50#include <dev/pci/pcireg.h> 51#include <dev/pci/pcivar.h> 52 53#include "../ntb.h" 54 55#define PLX_MAX_BARS 4 /* There are at most 4 data BARs. */ 56#define PLX_NUM_SPAD 8 /* There are 8 scratchpads. */ 57#define PLX_NUM_SPAD_PATT 4 /* Use test pattern as 4 more. */ 58#define PLX_NUM_DB 16 /* There are 16 doorbells. */ 59#define PLX_MAX_SPLIT 128 /* Allow are at most 128 splits. */ 60 61struct ntb_plx_mw_info { 62 int mw_bar; 63 int mw_64bit; 64 int mw_rid; 65 struct resource *mw_res; 66 vm_paddr_t mw_pbase; 67 caddr_t mw_vbase; 68 vm_size_t mw_size; 69 struct { 70 vm_memattr_t mw_map_mode; 71 bus_addr_t mw_xlat_addr; 72 bus_size_t mw_xlat_size; 73 } splits[PLX_MAX_SPLIT]; 74}; 75 76struct ntb_plx_softc { 77 /* ntb.c context. Do not move! Must go first! */ 78 void *ntb_store; 79 80 device_t dev; 81 struct resource *conf_res; 82 int conf_rid; 83 u_int ntx; /* NTx number within chip. */ 84 u_int link; /* Link v/s Virtual side. */ 85 u_int port; /* Port number within chip. */ 86 u_int alut; /* A-LUT is enabled for NTx */ 87 u_int split; /* split BAR2 into 2^x parts */ 88 89 int int_rid; 90 struct resource *int_res; 91 void *int_tag; 92 93 struct ntb_plx_mw_info mw_info[PLX_MAX_BARS]; 94 int mw_count; /* Number of memory windows. */ 95 96 int spad_count1; /* Number of standard spads. */ 97 int spad_count2; /* Number of extra spads. */ 98 uint32_t spad_off1; /* Offset of our spads. */ 99 uint32_t spad_off2; /* Offset of our extra spads. */ 100 uint32_t spad_offp1; /* Offset of peer spads. */ 101 uint32_t spad_offp2; /* Offset of peer extra spads. */ 102 103 /* Parameters of window shared with peer config access in B2B mode. */ 104 int b2b_mw; /* Shared window number. */ 105 uint64_t b2b_off; /* Offset in shared window. */ 106}; 107 108#define PLX_NT0_BASE 0x3E000 109#define PLX_NT1_BASE 0x3C000 110#define PLX_NTX_BASE(sc) ((sc)->ntx ? PLX_NT1_BASE : PLX_NT0_BASE) 111#define PLX_NTX_LINK_OFFSET 0x01000 112 113/* Bases of NTx our/peer interface registers */ 114#define PLX_NTX_OUR_BASE(sc) \ 115 (PLX_NTX_BASE(sc) + ((sc)->link ? PLX_NTX_LINK_OFFSET : 0)) 116#define PLX_NTX_PEER_BASE(sc) \ 117 (PLX_NTX_BASE(sc) + ((sc)->link ? 0 : PLX_NTX_LINK_OFFSET)) 118 119/* Read/write NTx our interface registers */ 120#define NTX_READ(sc, reg) \ 121 bus_read_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg)) 122#define NTX_WRITE(sc, reg, val) \ 123 bus_write_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg), (val)) 124 125/* Read/write NTx peer interface registers */ 126#define PNTX_READ(sc, reg) \ 127 bus_read_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg)) 128#define PNTX_WRITE(sc, reg, val) \ 129 bus_write_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg), (val)) 130 131/* Read/write B2B NTx registers */ 132#define BNTX_READ(sc, reg) \ 133 bus_read_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \ 134 PLX_NTX_BASE(sc) + (reg)) 135#define BNTX_WRITE(sc, reg, val) \ 136 bus_write_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \ 137 PLX_NTX_BASE(sc) + (reg), (val)) 138 139#define PLX_PORT_BASE(p) ((p) << 12) 140#define PLX_STATION_PORT_BASE(sc) PLX_PORT_BASE((sc)->port & ~7) 141 142#define PLX_PORT_CONTROL(sc) (PLX_STATION_PORT_BASE(sc) + 0x208) 143 144static int ntb_plx_init(device_t dev); 145static int ntb_plx_detach(device_t dev); 146static int ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx); 147 148static int 149ntb_plx_probe(device_t dev) 150{ 151 152 switch (pci_get_devid(dev)) { 153 case 0x87a010b5: 154 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Link"); 155 return (BUS_PROBE_DEFAULT); 156 case 0x87a110b5: 157 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Link"); 158 return (BUS_PROBE_DEFAULT); 159 case 0x87b010b5: 160 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Virtual"); 161 return (BUS_PROBE_DEFAULT); 162 case 0x87b110b5: 163 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Virtual"); 164 return (BUS_PROBE_DEFAULT); 165 } 166 return (ENXIO); 167} 168 169static int 170ntb_plx_init(device_t dev) 171{ 172 struct ntb_plx_softc *sc = device_get_softc(dev); 173 struct ntb_plx_mw_info *mw; 174 uint64_t val64; 175 int i; 176 uint32_t val; 177 178 if (sc->b2b_mw >= 0) { 179 /* Set peer BAR0/1 size and address for B2B NTx access. */ 180 mw = &sc->mw_info[sc->b2b_mw]; 181 if (mw->mw_64bit) { 182 PNTX_WRITE(sc, 0xe4, 0x3); /* 64-bit */ 183 val64 = 0x2000000000000000 * mw->mw_bar | 0x4; 184 PNTX_WRITE(sc, PCIR_BAR(0), val64); 185 PNTX_WRITE(sc, PCIR_BAR(0) + 4, val64 >> 32); 186 } else { 187 PNTX_WRITE(sc, 0xe4, 0x2); /* 32-bit */ 188 val = 0x20000000 * mw->mw_bar; 189 PNTX_WRITE(sc, PCIR_BAR(0), val); 190 } 191 192 /* Set Virtual to Link address translation for B2B. */ 193 for (i = 0; i < sc->mw_count; i++) { 194 mw = &sc->mw_info[i]; 195 if (mw->mw_64bit) { 196 val64 = 0x2000000000000000 * mw->mw_bar; 197 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val64); 198 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, val64 >> 32); 199 } else { 200 val = 0x20000000 * mw->mw_bar; 201 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val); 202 } 203 } 204 205 /* Make sure Virtual to Link A-LUT is disabled. */ 206 if (sc->alut) 207 PNTX_WRITE(sc, 0xc94, 0); 208 209 /* Enable all Link Interface LUT entries for peer. */ 210 for (i = 0; i < 32; i += 2) { 211 PNTX_WRITE(sc, 0xdb4 + i * 2, 212 0x00010001 | ((i + 1) << 19) | (i << 3)); 213 } 214 } 215 216 /* 217 * Enable Virtual Interface LUT entry 0 for 0:0.*. 218 * entry 1 for our Requester ID reported by the chip, 219 * entries 2-5 for 0/64/128/192:4.* of I/OAT DMA engines. 220 * XXX: Its a hack, we can't know all DMA engines, but this covers all 221 * I/OAT of Xeon E5/E7 at least from Sandy Bridge till Skylake I saw. 222 */ 223 val = (NTX_READ(sc, 0xc90) << 16) | 0x00010001; 224 NTX_WRITE(sc, sc->link ? 0xdb4 : 0xd94, val); 225 NTX_WRITE(sc, sc->link ? 0xdb8 : 0xd98, 0x40210021); 226 NTX_WRITE(sc, sc->link ? 0xdbc : 0xd9c, 0xc0218021); 227 228 /* Set Link to Virtual address translation. */ 229 for (i = 0; i < sc->mw_count; i++) 230 ntb_plx_mw_set_trans_internal(dev, i); 231 232 pci_enable_busmaster(dev); 233 if (sc->b2b_mw >= 0) 234 PNTX_WRITE(sc, PCIR_COMMAND, PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 235 236 return (0); 237} 238 239static void 240ntb_plx_isr(void *arg) 241{ 242 device_t dev = arg; 243 struct ntb_plx_softc *sc = device_get_softc(dev); 244 uint32_t val; 245 246 ntb_db_event((device_t)arg, 0); 247 248 if (sc->link) /* Link Interface has no Link Error registers. */ 249 return; 250 251 val = NTX_READ(sc, 0xfe0); 252 if (val == 0) 253 return; 254 NTX_WRITE(sc, 0xfe0, val); 255 if (val & 1) 256 device_printf(dev, "Correctable Error\n"); 257 if (val & 2) 258 device_printf(dev, "Uncorrectable Error\n"); 259 if (val & 4) { 260 /* DL_Down resets link side registers, have to reinit. */ 261 ntb_plx_init(dev); 262 ntb_link_event(dev); 263 } 264 if (val & 8) 265 device_printf(dev, "Uncorrectable Error Message Drop\n"); 266} 267 268static int 269ntb_plx_setup_intr(device_t dev) 270{ 271 struct ntb_plx_softc *sc = device_get_softc(dev); 272 int error; 273 274 /* 275 * XXX: This hardware supports MSI, but I found it unusable. 276 * It generates new MSI only when doorbell register goes from 277 * zero, but does not generate it when another bit is set or on 278 * partial clear. It makes operation very racy and unreliable. 279 * The data book mentions some mask juggling magic to workaround 280 * that, but I failed to make it work. 281 */ 282 sc->int_rid = 0; 283 sc->int_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 284 &sc->int_rid, RF_SHAREABLE|RF_ACTIVE); 285 if (sc->int_res == NULL) { 286 device_printf(dev, "bus_alloc_resource failed\n"); 287 return (ENOMEM); 288 } 289 error = bus_setup_intr(dev, sc->int_res, INTR_MPSAFE | INTR_TYPE_MISC, 290 NULL, ntb_plx_isr, dev, &sc->int_tag); 291 if (error != 0) { 292 device_printf(dev, "bus_setup_intr failed: %d\n", error); 293 return (error); 294 } 295 296 if (!sc->link) { /* Link Interface has no Link Error registers. */ 297 NTX_WRITE(sc, 0xfe0, 0xf); /* Clear link interrupts. */ 298 NTX_WRITE(sc, 0xfe4, 0x0); /* Unmask link interrupts. */ 299 } 300 return (0); 301} 302 303static void 304ntb_plx_teardown_intr(device_t dev) 305{ 306 struct ntb_plx_softc *sc = device_get_softc(dev); 307 308 if (!sc->link) /* Link Interface has no Link Error registers. */ 309 NTX_WRITE(sc, 0xfe4, 0xf); /* Mask link interrupts. */ 310 311 if (sc->int_res) { 312 bus_teardown_intr(dev, sc->int_res, sc->int_tag); 313 bus_release_resource(dev, SYS_RES_IRQ, sc->int_rid, 314 sc->int_res); 315 } 316} 317 318static int 319ntb_plx_attach(device_t dev) 320{ 321 struct ntb_plx_softc *sc = device_get_softc(dev); 322 struct ntb_plx_mw_info *mw; 323 int error = 0, i, j; 324 uint32_t val; 325 char buf[32]; 326 327 /* Identify what we are (what side of what NTx). */ 328 sc->dev = dev; 329 val = pci_read_config(dev, 0xc8c, 4); 330 sc->ntx = (val & 1) != 0; 331 sc->link = (val & 0x80000000) != 0; 332 333 /* Get access to whole 256KB of chip configuration space via BAR0/1. */ 334 sc->conf_rid = PCIR_BAR(0); 335 sc->conf_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 336 &sc->conf_rid, RF_ACTIVE); 337 if (sc->conf_res == NULL) { 338 device_printf(dev, "Can't allocate configuration BAR.\n"); 339 return (ENXIO); 340 } 341 342 /* Identify chip port we are connected to. */ 343 val = bus_read_4(sc->conf_res, 0x360); 344 sc->port = (val >> ((sc->ntx == 0) ? 8 : 16)) & 0x1f; 345 346 /* Detect A-LUT enable and size. */ 347 val >>= 30; 348 sc->alut = (val == 0x3) ? 1 : ((val & (1 << sc->ntx)) ? 2 : 0); 349 if (sc->alut) 350 device_printf(dev, "%u A-LUT entries\n", 128 * sc->alut); 351 352 /* Find configured memory windows at BAR2-5. */ 353 sc->mw_count = 0; 354 for (i = 2; i <= 5; i++) { 355 mw = &sc->mw_info[sc->mw_count]; 356 mw->mw_bar = i; 357 mw->mw_rid = PCIR_BAR(mw->mw_bar); 358 mw->mw_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 359 &mw->mw_rid, RF_ACTIVE); 360 if (mw->mw_res == NULL) 361 continue; 362 mw->mw_pbase = rman_get_start(mw->mw_res); 363 mw->mw_size = rman_get_size(mw->mw_res); 364 mw->mw_vbase = rman_get_virtual(mw->mw_res); 365 for (j = 0; j < PLX_MAX_SPLIT; j++) 366 mw->splits[j].mw_map_mode = VM_MEMATTR_UNCACHEABLE; 367 sc->mw_count++; 368 369 /* Skip over adjacent BAR for 64-bit BARs. */ 370 val = pci_read_config(dev, PCIR_BAR(mw->mw_bar), 4); 371 if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) { 372 mw->mw_64bit = 1; 373 i++; 374 } 375 } 376 377 /* Try to identify B2B mode. */ 378 i = 1; 379 snprintf(buf, sizeof(buf), "hint.%s.%d.b2b", device_get_name(dev), 380 device_get_unit(dev)); 381 TUNABLE_INT_FETCH(buf, &i); 382 if (sc->link) { 383 device_printf(dev, "NTB-to-Root Port mode (Link Interface)\n"); 384 sc->b2b_mw = -1; 385 } else if (i == 0) { 386 device_printf(dev, "NTB-to-Root Port mode (Virtual Interface)\n"); 387 sc->b2b_mw = -1; 388 } else { 389 device_printf(dev, "NTB-to-NTB (back-to-back) mode\n"); 390 391 /* We need at least one memory window for B2B peer access. */ 392 if (sc->mw_count == 0) { 393 device_printf(dev, "No memory window BARs enabled.\n"); 394 error = ENXIO; 395 goto out; 396 } 397 sc->b2b_mw = sc->mw_count - 1; 398 399 /* Use half of the window for B2B, but no less then 1MB. */ 400 mw = &sc->mw_info[sc->b2b_mw]; 401 if (mw->mw_size >= 2 * 1024 * 1024) 402 sc->b2b_off = mw->mw_size / 2; 403 else 404 sc->b2b_off = 0; 405 } 406 407 snprintf(buf, sizeof(buf), "hint.%s.%d.split", device_get_name(dev), 408 device_get_unit(dev)); 409 TUNABLE_INT_FETCH(buf, &sc->split); 410 if (sc->split > 7) { 411 device_printf(dev, "Split value is too high (%u)\n", sc->split); 412 sc->split = 0; 413 } else if (sc->split > 0 && sc->alut == 0) { 414 device_printf(dev, "Can't split with disabled A-LUT\n"); 415 sc->split = 0; 416 } else if (sc->split > 0 && (sc->mw_count == 0 || sc->mw_info[0].mw_bar != 2)) { 417 device_printf(dev, "Can't split disabled BAR2\n"); 418 sc->split = 0; 419 } else if (sc->split > 0 && (sc->b2b_mw == 0 && sc->b2b_off == 0)) { 420 device_printf(dev, "Can't split BAR2 consumed by B2B\n"); 421 sc->split = 0; 422 } else if (sc->split > 0) { 423 device_printf(dev, "Splitting BAR2 into %d memory windows\n", 424 1 << sc->split); 425 } 426 427 /* 428 * Use Physical Layer User Test Pattern as additional scratchpad. 429 * Make sure they are present and enabled by writing to them. 430 * XXX: Its a hack, but standard 8 registers are not enough. 431 */ 432 sc->spad_offp1 = sc->spad_off1 = PLX_NTX_OUR_BASE(sc) + 0xc6c; 433 sc->spad_offp2 = sc->spad_off2 = PLX_PORT_BASE(sc->ntx * 8) + 0x20c; 434 if (sc->b2b_mw >= 0) { 435 /* In NTB-to-NTB mode each side has own scratchpads. */ 436 sc->spad_count1 = PLX_NUM_SPAD; 437 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678); 438 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678) 439 sc->spad_count2 = PLX_NUM_SPAD_PATT; 440 } else { 441 /* Otherwise we have share scratchpads with the peer. */ 442 if (sc->link) { 443 sc->spad_off1 += PLX_NUM_SPAD / 2 * 4; 444 sc->spad_off2 += PLX_NUM_SPAD_PATT / 2 * 4; 445 } else { 446 sc->spad_offp1 += PLX_NUM_SPAD / 2 * 4; 447 sc->spad_offp2 += PLX_NUM_SPAD_PATT / 2 * 4; 448 } 449 sc->spad_count1 = PLX_NUM_SPAD / 2; 450 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678); 451 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678) 452 sc->spad_count2 = PLX_NUM_SPAD_PATT / 2; 453 } 454 455 /* Apply static part of NTB configuration. */ 456 ntb_plx_init(dev); 457 458 /* Allocate and setup interrupts. */ 459 error = ntb_plx_setup_intr(dev); 460 if (error) 461 goto out; 462 463 /* Attach children to this controller */ 464 error = ntb_register_device(dev); 465 466out: 467 if (error != 0) 468 ntb_plx_detach(dev); 469 return (error); 470} 471 472static int 473ntb_plx_detach(device_t dev) 474{ 475 struct ntb_plx_softc *sc = device_get_softc(dev); 476 struct ntb_plx_mw_info *mw; 477 int i; 478 479 /* Detach & delete all children */ 480 ntb_unregister_device(dev); 481 482 /* Disable and free interrupts. */ 483 ntb_plx_teardown_intr(dev); 484 485 /* Free memory resources. */ 486 for (i = 0; i < sc->mw_count; i++) { 487 mw = &sc->mw_info[i]; 488 bus_release_resource(dev, SYS_RES_MEMORY, mw->mw_rid, 489 mw->mw_res); 490 } 491 bus_release_resource(dev, SYS_RES_MEMORY, sc->conf_rid, sc->conf_res); 492 return (0); 493} 494 495 496static bool 497ntb_plx_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 498{ 499 uint16_t link; 500 501 link = pcie_read_config(dev, PCIER_LINK_STA, 2); 502 if (speed != NULL) 503 *speed = (link & PCIEM_LINK_STA_SPEED); 504 if (width != NULL) 505 *width = (link & PCIEM_LINK_STA_WIDTH) >> 4; 506 return ((link & PCIEM_LINK_STA_WIDTH) != 0); 507} 508 509static int 510ntb_plx_link_enable(device_t dev, enum ntb_speed speed __unused, 511 enum ntb_width width __unused) 512{ 513 struct ntb_plx_softc *sc = device_get_softc(dev); 514 uint32_t reg, val; 515 516 /* The fact that we see the Link Interface means link is enabled. */ 517 if (sc->link) { 518 ntb_link_event(dev); 519 return (0); 520 } 521 522 reg = PLX_PORT_CONTROL(sc); 523 val = bus_read_4(sc->conf_res, reg); 524 if ((val & (1 << (sc->port & 7))) == 0) { 525 /* If already enabled, generate fake link event and exit. */ 526 ntb_link_event(dev); 527 return (0); 528 } 529 val &= ~(1 << (sc->port & 7)); 530 bus_write_4(sc->conf_res, reg, val); 531 return (0); 532} 533 534static int 535ntb_plx_link_disable(device_t dev) 536{ 537 struct ntb_plx_softc *sc = device_get_softc(dev); 538 uint32_t reg, val; 539 540 /* Link disable for Link Interface would be suicidal. */ 541 if (sc->link) 542 return (0); 543 544 reg = PLX_PORT_CONTROL(sc); 545 val = bus_read_4(sc->conf_res, reg); 546 val |= (1 << (sc->port & 7)); 547 bus_write_4(sc->conf_res, reg, val); 548 return (0); 549} 550 551static bool 552ntb_plx_link_enabled(device_t dev) 553{ 554 struct ntb_plx_softc *sc = device_get_softc(dev); 555 uint32_t reg, val; 556 557 /* The fact that we see the Link Interface means link is enabled. */ 558 if (sc->link) 559 return (TRUE); 560 561 reg = PLX_PORT_CONTROL(sc); 562 val = bus_read_4(sc->conf_res, reg); 563 return ((val & (1 << (sc->port & 7))) == 0); 564} 565 566static uint8_t 567ntb_plx_mw_count(device_t dev) 568{ 569 struct ntb_plx_softc *sc = device_get_softc(dev); 570 uint8_t res; 571 572 res = sc->mw_count; 573 res += (1 << sc->split) - 1; 574 if (sc->b2b_mw >= 0 && sc->b2b_off == 0) 575 res--; /* B2B consumed whole window. */ 576 return (res); 577} 578 579static unsigned 580ntb_plx_user_mw_to_idx(struct ntb_plx_softc *sc, unsigned uidx, unsigned *sp) 581{ 582 unsigned t; 583 584 t = 1 << sc->split; 585 if (uidx < t) { 586 *sp = uidx; 587 return (0); 588 } 589 *sp = 0; 590 return (uidx - (t - 1)); 591} 592 593static int 594ntb_plx_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 595 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 596 bus_addr_t *plimit) 597{ 598 struct ntb_plx_softc *sc = device_get_softc(dev); 599 struct ntb_plx_mw_info *mw; 600 size_t off, ss; 601 unsigned sp, split; 602 603 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 604 if (mw_idx >= sc->mw_count) 605 return (EINVAL); 606 off = 0; 607 if (mw_idx == sc->b2b_mw) { 608 KASSERT(sc->b2b_off != 0, 609 ("user shouldn't get non-shared b2b mw")); 610 off = sc->b2b_off; 611 } 612 mw = &sc->mw_info[mw_idx]; 613 split = (mw->mw_bar == 2) ? sc->split : 0; 614 ss = (mw->mw_size - off) >> split; 615 616 /* Local to remote memory window parameters. */ 617 if (base != NULL) 618 *base = mw->mw_pbase + off + ss * sp; 619 if (vbase != NULL) 620 *vbase = mw->mw_vbase + off + ss * sp; 621 if (size != NULL) 622 *size = ss; 623 624 /* 625 * Remote to local memory window translation address alignment. 626 * Translation address has to be aligned to the BAR size, but A-LUT 627 * entries re-map addresses can be aligned to 1/128 or 1/256 of it. 628 * XXX: In B2B mode we can change BAR size (and so alignmet) live, 629 * but there is no way to report it here, so report safe value. 630 */ 631 if (align != NULL) { 632 if (sc->alut && mw->mw_bar == 2) 633 *align = (mw->mw_size - off) / 128 / sc->alut; 634 else 635 *align = mw->mw_size - off; 636 } 637 638 /* 639 * Remote to local memory window size alignment. 640 * The chip has no limit registers, but A-LUT, when available, allows 641 * access control with granularity of 1/128 or 1/256 of the BAR size. 642 * XXX: In B2B case we can change BAR size live, but there is no way 643 * to report it, so report half of the BAR size, that should be safe. 644 * In non-B2B case there is no control at all, so report the BAR size. 645 */ 646 if (align_size != NULL) { 647 if (sc->alut && mw->mw_bar == 2) 648 *align_size = (mw->mw_size - off) / 128 / sc->alut; 649 else if (sc->b2b_mw >= 0) 650 *align_size = (mw->mw_size - off) / 2; 651 else 652 *align_size = mw->mw_size - off; 653 } 654 655 /* Remote to local memory window translation address upper limit. */ 656 if (plimit != NULL) 657 *plimit = mw->mw_64bit ? BUS_SPACE_MAXADDR : 658 BUS_SPACE_MAXADDR_32BIT; 659 return (0); 660} 661 662static int 663ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx) 664{ 665 struct ntb_plx_softc *sc = device_get_softc(dev); 666 struct ntb_plx_mw_info *mw; 667 uint64_t addr, eaddr, off, size, bsize, esize, val64; 668 uint32_t val; 669 unsigned i, sp, split; 670 671 mw = &sc->mw_info[mw_idx]; 672 off = (mw_idx == sc->b2b_mw) ? sc->b2b_off : 0; 673 split = (mw->mw_bar == 2) ? sc->split : 0; 674 675 /* Get BAR size. In case of split or B2RP we can't change it. */ 676 if (split || sc->b2b_mw < 0) { 677 bsize = mw->mw_size - off; 678 } else { 679 bsize = mw->splits[0].mw_xlat_size; 680 if (!powerof2(bsize)) 681 bsize = 1LL << flsll(bsize); 682 if (bsize > 0 && bsize < 1024 * 1024) 683 bsize = 1024 * 1024; 684 } 685 686 /* 687 * While for B2B we can set any BAR size on a link side, for shared 688 * window we can't go above preconfigured size due to BAR address 689 * alignment requirements. 690 */ 691 if ((off & (bsize - 1)) != 0) 692 return (EINVAL); 693 694 /* In B2B mode set Link Interface BAR size/address. */ 695 if (sc->b2b_mw >= 0 && mw->mw_64bit) { 696 val64 = 0; 697 if (bsize > 0) 698 val64 = (~(bsize - 1) & ~0xfffff); 699 val64 |= 0xc; 700 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val64); 701 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4 + 4, val64 >> 32); 702 703 val64 = 0x2000000000000000 * mw->mw_bar + off; 704 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64); 705 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar) + 4, val64 >> 32); 706 } else if (sc->b2b_mw >= 0) { 707 val = 0; 708 if (bsize > 0) 709 val = (~(bsize - 1) & ~0xfffff); 710 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val); 711 712 val64 = 0x20000000 * mw->mw_bar + off; 713 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64); 714 } 715 716 /* Set BARs address translation */ 717 addr = split ? UINT64_MAX : mw->splits[0].mw_xlat_addr; 718 if (mw->mw_64bit) { 719 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr); 720 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, addr >> 32); 721 } else { 722 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr); 723 } 724 725 /* Configure and enable A-LUT if we need it. */ 726 size = split ? 0 : mw->splits[0].mw_xlat_size; 727 if (sc->alut && mw->mw_bar == 2 && (sc->split > 0 || 728 ((addr & (bsize - 1)) != 0 || size != bsize))) { 729 esize = bsize / (128 * sc->alut); 730 for (i = sp = 0; i < 128 * sc->alut; i++) { 731 if (i % (128 * sc->alut >> sc->split) == 0) { 732 eaddr = addr = mw->splits[sp].mw_xlat_addr; 733 size = mw->splits[sp++].mw_xlat_size; 734 } 735 val = sc->link ? 0 : 1; 736 if (sc->alut == 1) 737 val += 2 * sc->ntx; 738 val *= 0x1000 * sc->alut; 739 val += 0x38000 + i * 4 + (i >= 128 ? 0x0e00 : 0); 740 bus_write_4(sc->conf_res, val, eaddr); 741 bus_write_4(sc->conf_res, val + 0x400, eaddr >> 32); 742 bus_write_4(sc->conf_res, val + 0x800, 743 (eaddr < addr + size) ? 0x3 : 0); 744 eaddr += esize; 745 } 746 NTX_WRITE(sc, 0xc94, 0x10000000); 747 } else if (sc->alut && mw->mw_bar == 2) 748 NTX_WRITE(sc, 0xc94, 0); 749 750 return (0); 751} 752 753static int 754ntb_plx_mw_set_trans(device_t dev, unsigned mw_idx, bus_addr_t addr, size_t size) 755{ 756 struct ntb_plx_softc *sc = device_get_softc(dev); 757 struct ntb_plx_mw_info *mw; 758 unsigned sp; 759 760 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 761 if (mw_idx >= sc->mw_count) 762 return (EINVAL); 763 mw = &sc->mw_info[mw_idx]; 764 if (!mw->mw_64bit && 765 ((addr & UINT32_MAX) != addr || 766 ((addr + size) & UINT32_MAX) != (addr + size))) 767 return (ERANGE); 768 mw->splits[sp].mw_xlat_addr = addr; 769 mw->splits[sp].mw_xlat_size = size; 770 return (ntb_plx_mw_set_trans_internal(dev, mw_idx)); 771} 772 773static int 774ntb_plx_mw_clear_trans(device_t dev, unsigned mw_idx) 775{ 776 777 return (ntb_plx_mw_set_trans(dev, mw_idx, 0, 0)); 778} 779 780static int 781ntb_plx_mw_get_wc(device_t dev, unsigned mw_idx, vm_memattr_t *mode) 782{ 783 struct ntb_plx_softc *sc = device_get_softc(dev); 784 struct ntb_plx_mw_info *mw; 785 unsigned sp; 786 787 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 788 if (mw_idx >= sc->mw_count) 789 return (EINVAL); 790 mw = &sc->mw_info[mw_idx]; 791 *mode = mw->splits[sp].mw_map_mode; 792 return (0); 793} 794 795static int 796ntb_plx_mw_set_wc(device_t dev, unsigned mw_idx, vm_memattr_t mode) 797{ 798 struct ntb_plx_softc *sc = device_get_softc(dev); 799 struct ntb_plx_mw_info *mw; 800 uint64_t off, ss; 801 int rc; 802 unsigned sp, split; 803 804 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 805 if (mw_idx >= sc->mw_count) 806 return (EINVAL); 807 mw = &sc->mw_info[mw_idx]; 808 if (mw->splits[sp].mw_map_mode == mode) 809 return (0); 810 811 off = 0; 812 if (mw_idx == sc->b2b_mw) { 813 KASSERT(sc->b2b_off != 0, 814 ("user shouldn't get non-shared b2b mw")); 815 off = sc->b2b_off; 816 } 817 818 split = (mw->mw_bar == 2) ? sc->split : 0; 819 ss = (mw->mw_size - off) >> split; 820 rc = pmap_change_attr((vm_offset_t)mw->mw_vbase + off + ss * sp, 821 ss, mode); 822 if (rc == 0) 823 mw->splits[sp].mw_map_mode = mode; 824 return (rc); 825} 826 827static uint8_t 828ntb_plx_spad_count(device_t dev) 829{ 830 struct ntb_plx_softc *sc = device_get_softc(dev); 831 832 return (sc->spad_count1 + sc->spad_count2); 833} 834 835static int 836ntb_plx_spad_write(device_t dev, unsigned int idx, uint32_t val) 837{ 838 struct ntb_plx_softc *sc = device_get_softc(dev); 839 u_int off; 840 841 if (idx >= sc->spad_count1 + sc->spad_count2) 842 return (EINVAL); 843 844 if (idx < sc->spad_count1) 845 off = sc->spad_off1 + idx * 4; 846 else 847 off = sc->spad_off2 + (idx - sc->spad_count1) * 4; 848 bus_write_4(sc->conf_res, off, val); 849 return (0); 850} 851 852static void 853ntb_plx_spad_clear(device_t dev) 854{ 855 struct ntb_plx_softc *sc = device_get_softc(dev); 856 int i; 857 858 for (i = 0; i < sc->spad_count1 + sc->spad_count2; i++) 859 ntb_plx_spad_write(dev, i, 0); 860} 861 862static int 863ntb_plx_spad_read(device_t dev, unsigned int idx, uint32_t *val) 864{ 865 struct ntb_plx_softc *sc = device_get_softc(dev); 866 u_int off; 867 868 if (idx >= sc->spad_count1 + sc->spad_count2) 869 return (EINVAL); 870 871 if (idx < sc->spad_count1) 872 off = sc->spad_off1 + idx * 4; 873 else 874 off = sc->spad_off2 + (idx - sc->spad_count1) * 4; 875 *val = bus_read_4(sc->conf_res, off); 876 return (0); 877} 878 879static int 880ntb_plx_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 881{ 882 struct ntb_plx_softc *sc = device_get_softc(dev); 883 u_int off; 884 885 if (idx >= sc->spad_count1 + sc->spad_count2) 886 return (EINVAL); 887 888 if (idx < sc->spad_count1) 889 off = sc->spad_offp1 + idx * 4; 890 else 891 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4; 892 if (sc->b2b_mw >= 0) 893 bus_write_4(sc->mw_info[sc->b2b_mw].mw_res, off, val); 894 else 895 bus_write_4(sc->conf_res, off, val); 896 return (0); 897} 898 899static int 900ntb_plx_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 901{ 902 struct ntb_plx_softc *sc = device_get_softc(dev); 903 u_int off; 904 905 if (idx >= sc->spad_count1 + sc->spad_count2) 906 return (EINVAL); 907 908 if (idx < sc->spad_count1) 909 off = sc->spad_offp1 + idx * 4; 910 else 911 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4; 912 if (sc->b2b_mw >= 0) 913 *val = bus_read_4(sc->mw_info[sc->b2b_mw].mw_res, off); 914 else 915 *val = bus_read_4(sc->conf_res, off); 916 return (0); 917} 918 919static uint64_t 920ntb_plx_db_valid_mask(device_t dev) 921{ 922 923 return ((1LL << PLX_NUM_DB) - 1); 924} 925 926static int 927ntb_plx_db_vector_count(device_t dev) 928{ 929 930 return (1); 931} 932 933static uint64_t 934ntb_plx_db_vector_mask(device_t dev, uint32_t vector) 935{ 936 937 if (vector > 0) 938 return (0); 939 return ((1LL << PLX_NUM_DB) - 1); 940} 941 942static void 943ntb_plx_db_clear(device_t dev, uint64_t bits) 944{ 945 struct ntb_plx_softc *sc = device_get_softc(dev); 946 947 NTX_WRITE(sc, sc->link ? 0xc60 : 0xc50, bits); 948} 949 950static void 951ntb_plx_db_clear_mask(device_t dev, uint64_t bits) 952{ 953 struct ntb_plx_softc *sc = device_get_softc(dev); 954 955 NTX_WRITE(sc, sc->link ? 0xc68 : 0xc58, bits); 956} 957 958static uint64_t 959ntb_plx_db_read(device_t dev) 960{ 961 struct ntb_plx_softc *sc = device_get_softc(dev); 962 963 return (NTX_READ(sc, sc->link ? 0xc5c : 0xc4c)); 964} 965 966static void 967ntb_plx_db_set_mask(device_t dev, uint64_t bits) 968{ 969 struct ntb_plx_softc *sc = device_get_softc(dev); 970 971 NTX_WRITE(sc, sc->link ? 0xc64 : 0xc54, bits); 972} 973 974static int 975ntb_plx_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 976{ 977 struct ntb_plx_softc *sc = device_get_softc(dev); 978 struct ntb_plx_mw_info *mw; 979 980 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 981 982 if (sc->b2b_mw >= 0) { 983 mw = &sc->mw_info[sc->b2b_mw]; 984 *db_addr = (uint64_t)mw->mw_pbase + PLX_NTX_BASE(sc) + 0xc4c; 985 } else { 986 *db_addr = rman_get_start(sc->conf_res) + PLX_NTX_BASE(sc); 987 *db_addr += sc->link ? 0xc4c : 0xc5c; 988 } 989 *db_size = 4; 990 return (0); 991} 992 993static void 994ntb_plx_peer_db_set(device_t dev, uint64_t bit) 995{ 996 struct ntb_plx_softc *sc = device_get_softc(dev); 997 998 if (sc->b2b_mw >= 0) 999 BNTX_WRITE(sc, 0xc4c, bit); 1000 else 1001 NTX_WRITE(sc, sc->link ? 0xc4c : 0xc5c, bit); 1002} 1003 1004static device_method_t ntb_plx_methods[] = { 1005 /* Device interface */ 1006 DEVMETHOD(device_probe, ntb_plx_probe), 1007 DEVMETHOD(device_attach, ntb_plx_attach), 1008 DEVMETHOD(device_detach, ntb_plx_detach), 1009 /* Bus interface */ 1010 DEVMETHOD(bus_child_location_str, ntb_child_location_str), 1011 DEVMETHOD(bus_print_child, ntb_print_child), 1012 DEVMETHOD(bus_get_dma_tag, ntb_get_dma_tag), 1013 /* NTB interface */ 1014 DEVMETHOD(ntb_link_is_up, ntb_plx_link_is_up), 1015 DEVMETHOD(ntb_link_enable, ntb_plx_link_enable), 1016 DEVMETHOD(ntb_link_disable, ntb_plx_link_disable), 1017 DEVMETHOD(ntb_link_enabled, ntb_plx_link_enabled), 1018 DEVMETHOD(ntb_mw_count, ntb_plx_mw_count), 1019 DEVMETHOD(ntb_mw_get_range, ntb_plx_mw_get_range), 1020 DEVMETHOD(ntb_mw_set_trans, ntb_plx_mw_set_trans), 1021 DEVMETHOD(ntb_mw_clear_trans, ntb_plx_mw_clear_trans), 1022 DEVMETHOD(ntb_mw_get_wc, ntb_plx_mw_get_wc), 1023 DEVMETHOD(ntb_mw_set_wc, ntb_plx_mw_set_wc), 1024 DEVMETHOD(ntb_spad_count, ntb_plx_spad_count), 1025 DEVMETHOD(ntb_spad_clear, ntb_plx_spad_clear), 1026 DEVMETHOD(ntb_spad_write, ntb_plx_spad_write), 1027 DEVMETHOD(ntb_spad_read, ntb_plx_spad_read), 1028 DEVMETHOD(ntb_peer_spad_write, ntb_plx_peer_spad_write), 1029 DEVMETHOD(ntb_peer_spad_read, ntb_plx_peer_spad_read), 1030 DEVMETHOD(ntb_db_valid_mask, ntb_plx_db_valid_mask), 1031 DEVMETHOD(ntb_db_vector_count, ntb_plx_db_vector_count), 1032 DEVMETHOD(ntb_db_vector_mask, ntb_plx_db_vector_mask), 1033 DEVMETHOD(ntb_db_clear, ntb_plx_db_clear), 1034 DEVMETHOD(ntb_db_clear_mask, ntb_plx_db_clear_mask), 1035 DEVMETHOD(ntb_db_read, ntb_plx_db_read), 1036 DEVMETHOD(ntb_db_set_mask, ntb_plx_db_set_mask), 1037 DEVMETHOD(ntb_peer_db_addr, ntb_plx_peer_db_addr), 1038 DEVMETHOD(ntb_peer_db_set, ntb_plx_peer_db_set), 1039 DEVMETHOD_END 1040}; 1041 1042static DEFINE_CLASS_0(ntb_hw, ntb_plx_driver, ntb_plx_methods, 1043 sizeof(struct ntb_plx_softc)); 1044DRIVER_MODULE(ntb_hw_plx, pci, ntb_plx_driver, ntb_hw_devclass, NULL, NULL); 1045MODULE_DEPEND(ntb_hw_plx, ntb, 1, 1, 1); 1046MODULE_VERSION(ntb_hw_plx, 1); 1047