1// SPDX-License-Identifier: GPL-2.0 2/* 3 * PCI Express I/O Virtualization (IOV) support 4 * Single Root IOV 1.0 5 * Address Translation Service 1.0 6 * 7 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 8 */ 9 10#include <linux/pci.h> 11#include <linux/slab.h> 12#include <linux/export.h> 13#include <linux/string.h> 14#include <linux/delay.h> 15#include "pci.h" 16 17#define VIRTFN_ID_LEN 17 /* "virtfn%u\0" for 2^32 - 1 */ 18 19int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id) 20{ 21 if (!dev->is_physfn) 22 return -EINVAL; 23 return dev->bus->number + ((dev->devfn + dev->sriov->offset + 24 dev->sriov->stride * vf_id) >> 8); 25} 26 27int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id) 28{ 29 if (!dev->is_physfn) 30 return -EINVAL; 31 return (dev->devfn + dev->sriov->offset + 32 dev->sriov->stride * vf_id) & 0xff; 33} 34EXPORT_SYMBOL_GPL(pci_iov_virtfn_devfn); 35 36int pci_iov_vf_id(struct pci_dev *dev) 37{ 38 struct pci_dev *pf; 39 40 if (!dev->is_virtfn) 41 return -EINVAL; 42 43 pf = pci_physfn(dev); 44 return (pci_dev_id(dev) - (pci_dev_id(pf) + pf->sriov->offset)) / 45 pf->sriov->stride; 46} 47EXPORT_SYMBOL_GPL(pci_iov_vf_id); 48 49/** 50 * pci_iov_get_pf_drvdata - Return the drvdata of a PF 51 * @dev: VF pci_dev 52 * @pf_driver: Device driver required to own the PF 53 * 54 * This must be called from a context that ensures that a VF driver is attached. 55 * The value returned is invalid once the VF driver completes its remove() 56 * callback. 57 * 58 * Locking is achieved by the driver core. A VF driver cannot be probed until 59 * pci_enable_sriov() is called and pci_disable_sriov() does not return until 60 * all VF drivers have completed their remove(). 61 * 62 * The PF driver must call pci_disable_sriov() before it begins to destroy the 63 * drvdata. 64 */ 65void *pci_iov_get_pf_drvdata(struct pci_dev *dev, struct pci_driver *pf_driver) 66{ 67 struct pci_dev *pf_dev; 68 69 if (!dev->is_virtfn) 70 return ERR_PTR(-EINVAL); 71 pf_dev = dev->physfn; 72 if (pf_dev->driver != pf_driver) 73 return ERR_PTR(-EINVAL); 74 return pci_get_drvdata(pf_dev); 75} 76EXPORT_SYMBOL_GPL(pci_iov_get_pf_drvdata); 77 78/* 79 * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may 80 * change when NumVFs changes. 81 * 82 * Update iov->offset and iov->stride when NumVFs is written. 83 */ 84static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn) 85{ 86 struct pci_sriov *iov = dev->sriov; 87 88 pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn); 89 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset); 90 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride); 91} 92 93/* 94 * The PF consumes one bus number. NumVFs, First VF Offset, and VF Stride 95 * determine how many additional bus numbers will be consumed by VFs. 96 * 97 * Iterate over all valid NumVFs, validate offset and stride, and calculate 98 * the maximum number of bus numbers that could ever be required. 99 */ 100static int compute_max_vf_buses(struct pci_dev *dev) 101{ 102 struct pci_sriov *iov = dev->sriov; 103 int nr_virtfn, busnr, rc = 0; 104 105 for (nr_virtfn = iov->total_VFs; nr_virtfn; nr_virtfn--) { 106 pci_iov_set_numvfs(dev, nr_virtfn); 107 if (!iov->offset || (nr_virtfn > 1 && !iov->stride)) { 108 rc = -EIO; 109 goto out; 110 } 111 112 busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 113 if (busnr > iov->max_VF_buses) 114 iov->max_VF_buses = busnr; 115 } 116 117out: 118 pci_iov_set_numvfs(dev, 0); 119 return rc; 120} 121 122static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr) 123{ 124 struct pci_bus *child; 125 126 if (bus->number == busnr) 127 return bus; 128 129 child = pci_find_bus(pci_domain_nr(bus), busnr); 130 if (child) 131 return child; 132 133 child = pci_add_new_bus(bus, NULL, busnr); 134 if (!child) 135 return NULL; 136 137 pci_bus_insert_busn_res(child, busnr, busnr); 138 139 return child; 140} 141 142static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus) 143{ 144 if (physbus != virtbus && list_empty(&virtbus->devices)) 145 pci_remove_bus(virtbus); 146} 147 148resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno) 149{ 150 if (!dev->is_physfn) 151 return 0; 152 153 return dev->sriov->barsz[resno - PCI_IOV_RESOURCES]; 154} 155 156static void pci_read_vf_config_common(struct pci_dev *virtfn) 157{ 158 struct pci_dev *physfn = virtfn->physfn; 159 160 /* 161 * Some config registers are the same across all associated VFs. 162 * Read them once from VF0 so we can skip reading them from the 163 * other VFs. 164 * 165 * PCIe r4.0, sec 9.3.4.1, technically doesn't require all VFs to 166 * have the same Revision ID and Subsystem ID, but we assume they 167 * do. 168 */ 169 pci_read_config_dword(virtfn, PCI_CLASS_REVISION, 170 &physfn->sriov->class); 171 pci_read_config_byte(virtfn, PCI_HEADER_TYPE, 172 &physfn->sriov->hdr_type); 173 pci_read_config_word(virtfn, PCI_SUBSYSTEM_VENDOR_ID, 174 &physfn->sriov->subsystem_vendor); 175 pci_read_config_word(virtfn, PCI_SUBSYSTEM_ID, 176 &physfn->sriov->subsystem_device); 177} 178 179int pci_iov_sysfs_link(struct pci_dev *dev, 180 struct pci_dev *virtfn, int id) 181{ 182 char buf[VIRTFN_ID_LEN]; 183 int rc; 184 185 sprintf(buf, "virtfn%u", id); 186 rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf); 187 if (rc) 188 goto failed; 189 rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn"); 190 if (rc) 191 goto failed1; 192 193 kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE); 194 195 return 0; 196 197failed1: 198 sysfs_remove_link(&dev->dev.kobj, buf); 199failed: 200 return rc; 201} 202 203#ifdef CONFIG_PCI_MSI 204static ssize_t sriov_vf_total_msix_show(struct device *dev, 205 struct device_attribute *attr, 206 char *buf) 207{ 208 struct pci_dev *pdev = to_pci_dev(dev); 209 u32 vf_total_msix = 0; 210 211 device_lock(dev); 212 if (!pdev->driver || !pdev->driver->sriov_get_vf_total_msix) 213 goto unlock; 214 215 vf_total_msix = pdev->driver->sriov_get_vf_total_msix(pdev); 216unlock: 217 device_unlock(dev); 218 return sysfs_emit(buf, "%u\n", vf_total_msix); 219} 220static DEVICE_ATTR_RO(sriov_vf_total_msix); 221 222static ssize_t sriov_vf_msix_count_store(struct device *dev, 223 struct device_attribute *attr, 224 const char *buf, size_t count) 225{ 226 struct pci_dev *vf_dev = to_pci_dev(dev); 227 struct pci_dev *pdev = pci_physfn(vf_dev); 228 int val, ret = 0; 229 230 if (kstrtoint(buf, 0, &val) < 0) 231 return -EINVAL; 232 233 if (val < 0) 234 return -EINVAL; 235 236 device_lock(&pdev->dev); 237 if (!pdev->driver || !pdev->driver->sriov_set_msix_vec_count) { 238 ret = -EOPNOTSUPP; 239 goto err_pdev; 240 } 241 242 device_lock(&vf_dev->dev); 243 if (vf_dev->driver) { 244 /* 245 * A driver is already attached to this VF and has configured 246 * itself based on the current MSI-X vector count. Changing 247 * the vector size could mess up the driver, so block it. 248 */ 249 ret = -EBUSY; 250 goto err_dev; 251 } 252 253 ret = pdev->driver->sriov_set_msix_vec_count(vf_dev, val); 254 255err_dev: 256 device_unlock(&vf_dev->dev); 257err_pdev: 258 device_unlock(&pdev->dev); 259 return ret ? : count; 260} 261static DEVICE_ATTR_WO(sriov_vf_msix_count); 262#endif 263 264static struct attribute *sriov_vf_dev_attrs[] = { 265#ifdef CONFIG_PCI_MSI 266 &dev_attr_sriov_vf_msix_count.attr, 267#endif 268 NULL, 269}; 270 271static umode_t sriov_vf_attrs_are_visible(struct kobject *kobj, 272 struct attribute *a, int n) 273{ 274 struct device *dev = kobj_to_dev(kobj); 275 struct pci_dev *pdev = to_pci_dev(dev); 276 277 if (!pdev->is_virtfn) 278 return 0; 279 280 return a->mode; 281} 282 283const struct attribute_group sriov_vf_dev_attr_group = { 284 .attrs = sriov_vf_dev_attrs, 285 .is_visible = sriov_vf_attrs_are_visible, 286}; 287 288int pci_iov_add_virtfn(struct pci_dev *dev, int id) 289{ 290 int i; 291 int rc = -ENOMEM; 292 u64 size; 293 struct pci_dev *virtfn; 294 struct resource *res; 295 struct pci_sriov *iov = dev->sriov; 296 struct pci_bus *bus; 297 298 bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id)); 299 if (!bus) 300 goto failed; 301 302 virtfn = pci_alloc_dev(bus); 303 if (!virtfn) 304 goto failed0; 305 306 virtfn->devfn = pci_iov_virtfn_devfn(dev, id); 307 virtfn->vendor = dev->vendor; 308 virtfn->device = iov->vf_device; 309 virtfn->is_virtfn = 1; 310 virtfn->physfn = pci_dev_get(dev); 311 virtfn->no_command_memory = 1; 312 313 if (id == 0) 314 pci_read_vf_config_common(virtfn); 315 316 rc = pci_setup_device(virtfn); 317 if (rc) 318 goto failed1; 319 320 virtfn->dev.parent = dev->dev.parent; 321 virtfn->multifunction = 0; 322 323 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 324 res = &dev->resource[i + PCI_IOV_RESOURCES]; 325 if (!res->parent) 326 continue; 327 virtfn->resource[i].name = pci_name(virtfn); 328 virtfn->resource[i].flags = res->flags; 329 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES); 330 virtfn->resource[i].start = res->start + size * id; 331 virtfn->resource[i].end = virtfn->resource[i].start + size - 1; 332 rc = request_resource(res, &virtfn->resource[i]); 333 BUG_ON(rc); 334 } 335 336 pci_device_add(virtfn, virtfn->bus); 337 rc = pci_iov_sysfs_link(dev, virtfn, id); 338 if (rc) 339 goto failed1; 340 341 pci_bus_add_device(virtfn); 342 343 return 0; 344 345failed1: 346 pci_stop_and_remove_bus_device(virtfn); 347 pci_dev_put(dev); 348failed0: 349 virtfn_remove_bus(dev->bus, bus); 350failed: 351 352 return rc; 353} 354 355void pci_iov_remove_virtfn(struct pci_dev *dev, int id) 356{ 357 char buf[VIRTFN_ID_LEN]; 358 struct pci_dev *virtfn; 359 360 virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus), 361 pci_iov_virtfn_bus(dev, id), 362 pci_iov_virtfn_devfn(dev, id)); 363 if (!virtfn) 364 return; 365 366 sprintf(buf, "virtfn%u", id); 367 sysfs_remove_link(&dev->dev.kobj, buf); 368 /* 369 * pci_stop_dev() could have been called for this virtfn already, 370 * so the directory for the virtfn may have been removed before. 371 * Double check to avoid spurious sysfs warnings. 372 */ 373 if (virtfn->dev.kobj.sd) 374 sysfs_remove_link(&virtfn->dev.kobj, "physfn"); 375 376 pci_stop_and_remove_bus_device(virtfn); 377 virtfn_remove_bus(dev->bus, virtfn->bus); 378 379 /* balance pci_get_domain_bus_and_slot() */ 380 pci_dev_put(virtfn); 381 pci_dev_put(dev); 382} 383 384static ssize_t sriov_totalvfs_show(struct device *dev, 385 struct device_attribute *attr, 386 char *buf) 387{ 388 struct pci_dev *pdev = to_pci_dev(dev); 389 390 return sysfs_emit(buf, "%u\n", pci_sriov_get_totalvfs(pdev)); 391} 392 393static ssize_t sriov_numvfs_show(struct device *dev, 394 struct device_attribute *attr, 395 char *buf) 396{ 397 struct pci_dev *pdev = to_pci_dev(dev); 398 u16 num_vfs; 399 400 /* Serialize vs sriov_numvfs_store() so readers see valid num_VFs */ 401 device_lock(&pdev->dev); 402 num_vfs = pdev->sriov->num_VFs; 403 device_unlock(&pdev->dev); 404 405 return sysfs_emit(buf, "%u\n", num_vfs); 406} 407 408/* 409 * num_vfs > 0; number of VFs to enable 410 * num_vfs = 0; disable all VFs 411 * 412 * Note: SRIOV spec does not allow partial VF 413 * disable, so it's all or none. 414 */ 415static ssize_t sriov_numvfs_store(struct device *dev, 416 struct device_attribute *attr, 417 const char *buf, size_t count) 418{ 419 struct pci_dev *pdev = to_pci_dev(dev); 420 int ret = 0; 421 u16 num_vfs; 422 423 if (kstrtou16(buf, 0, &num_vfs) < 0) 424 return -EINVAL; 425 426 if (num_vfs > pci_sriov_get_totalvfs(pdev)) 427 return -ERANGE; 428 429 device_lock(&pdev->dev); 430 431 if (num_vfs == pdev->sriov->num_VFs) 432 goto exit; 433 434 /* is PF driver loaded */ 435 if (!pdev->driver) { 436 pci_info(pdev, "no driver bound to device; cannot configure SR-IOV\n"); 437 ret = -ENOENT; 438 goto exit; 439 } 440 441 /* is PF driver loaded w/callback */ 442 if (!pdev->driver->sriov_configure) { 443 pci_info(pdev, "driver does not support SR-IOV configuration via sysfs\n"); 444 ret = -ENOENT; 445 goto exit; 446 } 447 448 if (num_vfs == 0) { 449 /* disable VFs */ 450 ret = pdev->driver->sriov_configure(pdev, 0); 451 goto exit; 452 } 453 454 /* enable VFs */ 455 if (pdev->sriov->num_VFs) { 456 pci_warn(pdev, "%d VFs already enabled. Disable before enabling %d VFs\n", 457 pdev->sriov->num_VFs, num_vfs); 458 ret = -EBUSY; 459 goto exit; 460 } 461 462 ret = pdev->driver->sriov_configure(pdev, num_vfs); 463 if (ret < 0) 464 goto exit; 465 466 if (ret != num_vfs) 467 pci_warn(pdev, "%d VFs requested; only %d enabled\n", 468 num_vfs, ret); 469 470exit: 471 device_unlock(&pdev->dev); 472 473 if (ret < 0) 474 return ret; 475 476 return count; 477} 478 479static ssize_t sriov_offset_show(struct device *dev, 480 struct device_attribute *attr, 481 char *buf) 482{ 483 struct pci_dev *pdev = to_pci_dev(dev); 484 485 return sysfs_emit(buf, "%u\n", pdev->sriov->offset); 486} 487 488static ssize_t sriov_stride_show(struct device *dev, 489 struct device_attribute *attr, 490 char *buf) 491{ 492 struct pci_dev *pdev = to_pci_dev(dev); 493 494 return sysfs_emit(buf, "%u\n", pdev->sriov->stride); 495} 496 497static ssize_t sriov_vf_device_show(struct device *dev, 498 struct device_attribute *attr, 499 char *buf) 500{ 501 struct pci_dev *pdev = to_pci_dev(dev); 502 503 return sysfs_emit(buf, "%x\n", pdev->sriov->vf_device); 504} 505 506static ssize_t sriov_drivers_autoprobe_show(struct device *dev, 507 struct device_attribute *attr, 508 char *buf) 509{ 510 struct pci_dev *pdev = to_pci_dev(dev); 511 512 return sysfs_emit(buf, "%u\n", pdev->sriov->drivers_autoprobe); 513} 514 515static ssize_t sriov_drivers_autoprobe_store(struct device *dev, 516 struct device_attribute *attr, 517 const char *buf, size_t count) 518{ 519 struct pci_dev *pdev = to_pci_dev(dev); 520 bool drivers_autoprobe; 521 522 if (kstrtobool(buf, &drivers_autoprobe) < 0) 523 return -EINVAL; 524 525 pdev->sriov->drivers_autoprobe = drivers_autoprobe; 526 527 return count; 528} 529 530static DEVICE_ATTR_RO(sriov_totalvfs); 531static DEVICE_ATTR_RW(sriov_numvfs); 532static DEVICE_ATTR_RO(sriov_offset); 533static DEVICE_ATTR_RO(sriov_stride); 534static DEVICE_ATTR_RO(sriov_vf_device); 535static DEVICE_ATTR_RW(sriov_drivers_autoprobe); 536 537static struct attribute *sriov_pf_dev_attrs[] = { 538 &dev_attr_sriov_totalvfs.attr, 539 &dev_attr_sriov_numvfs.attr, 540 &dev_attr_sriov_offset.attr, 541 &dev_attr_sriov_stride.attr, 542 &dev_attr_sriov_vf_device.attr, 543 &dev_attr_sriov_drivers_autoprobe.attr, 544#ifdef CONFIG_PCI_MSI 545 &dev_attr_sriov_vf_total_msix.attr, 546#endif 547 NULL, 548}; 549 550static umode_t sriov_pf_attrs_are_visible(struct kobject *kobj, 551 struct attribute *a, int n) 552{ 553 struct device *dev = kobj_to_dev(kobj); 554 555 if (!dev_is_pf(dev)) 556 return 0; 557 558 return a->mode; 559} 560 561const struct attribute_group sriov_pf_dev_attr_group = { 562 .attrs = sriov_pf_dev_attrs, 563 .is_visible = sriov_pf_attrs_are_visible, 564}; 565 566int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) 567{ 568 return 0; 569} 570 571int __weak pcibios_sriov_disable(struct pci_dev *pdev) 572{ 573 return 0; 574} 575 576static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs) 577{ 578 unsigned int i; 579 int rc; 580 581 if (dev->no_vf_scan) 582 return 0; 583 584 for (i = 0; i < num_vfs; i++) { 585 rc = pci_iov_add_virtfn(dev, i); 586 if (rc) 587 goto failed; 588 } 589 return 0; 590failed: 591 while (i--) 592 pci_iov_remove_virtfn(dev, i); 593 594 return rc; 595} 596 597static int sriov_enable(struct pci_dev *dev, int nr_virtfn) 598{ 599 int rc; 600 int i; 601 int nres; 602 u16 initial; 603 struct resource *res; 604 struct pci_dev *pdev; 605 struct pci_sriov *iov = dev->sriov; 606 int bars = 0; 607 int bus; 608 609 if (!nr_virtfn) 610 return 0; 611 612 if (iov->num_VFs) 613 return -EINVAL; 614 615 pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial); 616 if (initial > iov->total_VFs || 617 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total_VFs))) 618 return -EIO; 619 620 if (nr_virtfn < 0 || nr_virtfn > iov->total_VFs || 621 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial))) 622 return -EINVAL; 623 624 nres = 0; 625 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 626 bars |= (1 << (i + PCI_IOV_RESOURCES)); 627 res = &dev->resource[i + PCI_IOV_RESOURCES]; 628 if (res->parent) 629 nres++; 630 } 631 if (nres != iov->nres) { 632 pci_err(dev, "not enough MMIO resources for SR-IOV\n"); 633 return -ENOMEM; 634 } 635 636 bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 637 if (bus > dev->bus->busn_res.end) { 638 pci_err(dev, "can't enable %d VFs (bus %02x out of range of %pR)\n", 639 nr_virtfn, bus, &dev->bus->busn_res); 640 return -ENOMEM; 641 } 642 643 if (pci_enable_resources(dev, bars)) { 644 pci_err(dev, "SR-IOV: IOV BARS not allocated\n"); 645 return -ENOMEM; 646 } 647 648 if (iov->link != dev->devfn) { 649 pdev = pci_get_slot(dev->bus, iov->link); 650 if (!pdev) 651 return -ENODEV; 652 653 if (!pdev->is_physfn) { 654 pci_dev_put(pdev); 655 return -ENOSYS; 656 } 657 658 rc = sysfs_create_link(&dev->dev.kobj, 659 &pdev->dev.kobj, "dep_link"); 660 pci_dev_put(pdev); 661 if (rc) 662 return rc; 663 } 664 665 iov->initial_VFs = initial; 666 if (nr_virtfn < initial) 667 initial = nr_virtfn; 668 669 rc = pcibios_sriov_enable(dev, initial); 670 if (rc) { 671 pci_err(dev, "failure %d from pcibios_sriov_enable()\n", rc); 672 goto err_pcibios; 673 } 674 675 pci_iov_set_numvfs(dev, nr_virtfn); 676 iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE; 677 pci_cfg_access_lock(dev); 678 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 679 msleep(100); 680 pci_cfg_access_unlock(dev); 681 682 rc = sriov_add_vfs(dev, initial); 683 if (rc) 684 goto err_pcibios; 685 686 kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE); 687 iov->num_VFs = nr_virtfn; 688 689 return 0; 690 691err_pcibios: 692 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 693 pci_cfg_access_lock(dev); 694 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 695 ssleep(1); 696 pci_cfg_access_unlock(dev); 697 698 pcibios_sriov_disable(dev); 699 700 if (iov->link != dev->devfn) 701 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 702 703 pci_iov_set_numvfs(dev, 0); 704 return rc; 705} 706 707static void sriov_del_vfs(struct pci_dev *dev) 708{ 709 struct pci_sriov *iov = dev->sriov; 710 int i; 711 712 for (i = 0; i < iov->num_VFs; i++) 713 pci_iov_remove_virtfn(dev, i); 714} 715 716static void sriov_disable(struct pci_dev *dev) 717{ 718 struct pci_sriov *iov = dev->sriov; 719 720 if (!iov->num_VFs) 721 return; 722 723 sriov_del_vfs(dev); 724 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 725 pci_cfg_access_lock(dev); 726 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 727 ssleep(1); 728 pci_cfg_access_unlock(dev); 729 730 pcibios_sriov_disable(dev); 731 732 if (iov->link != dev->devfn) 733 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 734 735 iov->num_VFs = 0; 736 pci_iov_set_numvfs(dev, 0); 737} 738 739static int sriov_init(struct pci_dev *dev, int pos) 740{ 741 int i, bar64; 742 int rc; 743 int nres; 744 u32 pgsz; 745 u16 ctrl, total; 746 struct pci_sriov *iov; 747 struct resource *res; 748 const char *res_name; 749 struct pci_dev *pdev; 750 751 pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl); 752 if (ctrl & PCI_SRIOV_CTRL_VFE) { 753 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0); 754 ssleep(1); 755 } 756 757 ctrl = 0; 758 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 759 if (pdev->is_physfn) 760 goto found; 761 762 pdev = NULL; 763 if (pci_ari_enabled(dev->bus)) 764 ctrl |= PCI_SRIOV_CTRL_ARI; 765 766found: 767 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl); 768 769 pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total); 770 if (!total) 771 return 0; 772 773 pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz); 774 i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0; 775 pgsz &= ~((1 << i) - 1); 776 if (!pgsz) 777 return -EIO; 778 779 pgsz &= ~(pgsz - 1); 780 pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz); 781 782 iov = kzalloc(sizeof(*iov), GFP_KERNEL); 783 if (!iov) 784 return -ENOMEM; 785 786 nres = 0; 787 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 788 res = &dev->resource[i + PCI_IOV_RESOURCES]; 789 res_name = pci_resource_name(dev, i + PCI_IOV_RESOURCES); 790 791 /* 792 * If it is already FIXED, don't change it, something 793 * (perhaps EA or header fixups) wants it this way. 794 */ 795 if (res->flags & IORESOURCE_PCI_FIXED) 796 bar64 = (res->flags & IORESOURCE_MEM_64) ? 1 : 0; 797 else 798 bar64 = __pci_read_base(dev, pci_bar_unknown, res, 799 pos + PCI_SRIOV_BAR + i * 4); 800 if (!res->flags) 801 continue; 802 if (resource_size(res) & (PAGE_SIZE - 1)) { 803 rc = -EIO; 804 goto failed; 805 } 806 iov->barsz[i] = resource_size(res); 807 res->end = res->start + resource_size(res) * total - 1; 808 pci_info(dev, "%s %pR: contains BAR %d for %d VFs\n", 809 res_name, res, i, total); 810 i += bar64; 811 nres++; 812 } 813 814 iov->pos = pos; 815 iov->nres = nres; 816 iov->ctrl = ctrl; 817 iov->total_VFs = total; 818 iov->driver_max_VFs = total; 819 pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_device); 820 iov->pgsz = pgsz; 821 iov->self = dev; 822 iov->drivers_autoprobe = true; 823 pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap); 824 pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link); 825 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) 826 iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link); 827 828 if (pdev) 829 iov->dev = pci_dev_get(pdev); 830 else 831 iov->dev = dev; 832 833 dev->sriov = iov; 834 dev->is_physfn = 1; 835 rc = compute_max_vf_buses(dev); 836 if (rc) 837 goto fail_max_buses; 838 839 return 0; 840 841fail_max_buses: 842 dev->sriov = NULL; 843 dev->is_physfn = 0; 844failed: 845 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 846 res = &dev->resource[i + PCI_IOV_RESOURCES]; 847 res->flags = 0; 848 } 849 850 kfree(iov); 851 return rc; 852} 853 854static void sriov_release(struct pci_dev *dev) 855{ 856 BUG_ON(dev->sriov->num_VFs); 857 858 if (dev != dev->sriov->dev) 859 pci_dev_put(dev->sriov->dev); 860 861 kfree(dev->sriov); 862 dev->sriov = NULL; 863} 864 865static void sriov_restore_state(struct pci_dev *dev) 866{ 867 int i; 868 u16 ctrl; 869 struct pci_sriov *iov = dev->sriov; 870 871 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl); 872 if (ctrl & PCI_SRIOV_CTRL_VFE) 873 return; 874 875 /* 876 * Restore PCI_SRIOV_CTRL_ARI before pci_iov_set_numvfs() because 877 * it reads offset & stride, which depend on PCI_SRIOV_CTRL_ARI. 878 */ 879 ctrl &= ~PCI_SRIOV_CTRL_ARI; 880 ctrl |= iov->ctrl & PCI_SRIOV_CTRL_ARI; 881 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl); 882 883 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) 884 pci_update_resource(dev, i + PCI_IOV_RESOURCES); 885 886 pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz); 887 pci_iov_set_numvfs(dev, iov->num_VFs); 888 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 889 if (iov->ctrl & PCI_SRIOV_CTRL_VFE) 890 msleep(100); 891} 892 893/** 894 * pci_iov_init - initialize the IOV capability 895 * @dev: the PCI device 896 * 897 * Returns 0 on success, or negative on failure. 898 */ 899int pci_iov_init(struct pci_dev *dev) 900{ 901 int pos; 902 903 if (!pci_is_pcie(dev)) 904 return -ENODEV; 905 906 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV); 907 if (pos) 908 return sriov_init(dev, pos); 909 910 return -ENODEV; 911} 912 913/** 914 * pci_iov_release - release resources used by the IOV capability 915 * @dev: the PCI device 916 */ 917void pci_iov_release(struct pci_dev *dev) 918{ 919 if (dev->is_physfn) 920 sriov_release(dev); 921} 922 923/** 924 * pci_iov_remove - clean up SR-IOV state after PF driver is detached 925 * @dev: the PCI device 926 */ 927void pci_iov_remove(struct pci_dev *dev) 928{ 929 struct pci_sriov *iov = dev->sriov; 930 931 if (!dev->is_physfn) 932 return; 933 934 iov->driver_max_VFs = iov->total_VFs; 935 if (iov->num_VFs) 936 pci_warn(dev, "driver left SR-IOV enabled after remove\n"); 937} 938 939/** 940 * pci_iov_update_resource - update a VF BAR 941 * @dev: the PCI device 942 * @resno: the resource number 943 * 944 * Update a VF BAR in the SR-IOV capability of a PF. 945 */ 946void pci_iov_update_resource(struct pci_dev *dev, int resno) 947{ 948 struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL; 949 struct resource *res = dev->resource + resno; 950 int vf_bar = resno - PCI_IOV_RESOURCES; 951 struct pci_bus_region region; 952 u16 cmd; 953 u32 new; 954 int reg; 955 956 /* 957 * The generic pci_restore_bars() path calls this for all devices, 958 * including VFs and non-SR-IOV devices. If this is not a PF, we 959 * have nothing to do. 960 */ 961 if (!iov) 962 return; 963 964 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd); 965 if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) { 966 dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n", 967 vf_bar, res); 968 return; 969 } 970 971 /* 972 * Ignore unimplemented BARs, unused resource slots for 64-bit 973 * BARs, and non-movable resources, e.g., those described via 974 * Enhanced Allocation. 975 */ 976 if (!res->flags) 977 return; 978 979 if (res->flags & IORESOURCE_UNSET) 980 return; 981 982 if (res->flags & IORESOURCE_PCI_FIXED) 983 return; 984 985 pcibios_resource_to_bus(dev->bus, ®ion, res); 986 new = region.start; 987 new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK; 988 989 reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar; 990 pci_write_config_dword(dev, reg, new); 991 if (res->flags & IORESOURCE_MEM_64) { 992 new = region.start >> 16 >> 16; 993 pci_write_config_dword(dev, reg + 4, new); 994 } 995} 996 997resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev, 998 int resno) 999{ 1000 return pci_iov_resource_size(dev, resno); 1001} 1002 1003/** 1004 * pci_sriov_resource_alignment - get resource alignment for VF BAR 1005 * @dev: the PCI device 1006 * @resno: the resource number 1007 * 1008 * Returns the alignment of the VF BAR found in the SR-IOV capability. 1009 * This is not the same as the resource size which is defined as 1010 * the VF BAR size multiplied by the number of VFs. The alignment 1011 * is just the VF BAR size. 1012 */ 1013resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno) 1014{ 1015 return pcibios_iov_resource_alignment(dev, resno); 1016} 1017 1018/** 1019 * pci_restore_iov_state - restore the state of the IOV capability 1020 * @dev: the PCI device 1021 */ 1022void pci_restore_iov_state(struct pci_dev *dev) 1023{ 1024 if (dev->is_physfn) 1025 sriov_restore_state(dev); 1026} 1027 1028/** 1029 * pci_vf_drivers_autoprobe - set PF property drivers_autoprobe for VFs 1030 * @dev: the PCI device 1031 * @auto_probe: set VF drivers auto probe flag 1032 */ 1033void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool auto_probe) 1034{ 1035 if (dev->is_physfn) 1036 dev->sriov->drivers_autoprobe = auto_probe; 1037} 1038 1039/** 1040 * pci_iov_bus_range - find bus range used by Virtual Function 1041 * @bus: the PCI bus 1042 * 1043 * Returns max number of buses (exclude current one) used by Virtual 1044 * Functions. 1045 */ 1046int pci_iov_bus_range(struct pci_bus *bus) 1047{ 1048 int max = 0; 1049 struct pci_dev *dev; 1050 1051 list_for_each_entry(dev, &bus->devices, bus_list) { 1052 if (!dev->is_physfn) 1053 continue; 1054 if (dev->sriov->max_VF_buses > max) 1055 max = dev->sriov->max_VF_buses; 1056 } 1057 1058 return max ? max - bus->number : 0; 1059} 1060 1061/** 1062 * pci_enable_sriov - enable the SR-IOV capability 1063 * @dev: the PCI device 1064 * @nr_virtfn: number of virtual functions to enable 1065 * 1066 * Returns 0 on success, or negative on failure. 1067 */ 1068int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) 1069{ 1070 might_sleep(); 1071 1072 if (!dev->is_physfn) 1073 return -ENOSYS; 1074 1075 return sriov_enable(dev, nr_virtfn); 1076} 1077EXPORT_SYMBOL_GPL(pci_enable_sriov); 1078 1079/** 1080 * pci_disable_sriov - disable the SR-IOV capability 1081 * @dev: the PCI device 1082 */ 1083void pci_disable_sriov(struct pci_dev *dev) 1084{ 1085 might_sleep(); 1086 1087 if (!dev->is_physfn) 1088 return; 1089 1090 sriov_disable(dev); 1091} 1092EXPORT_SYMBOL_GPL(pci_disable_sriov); 1093 1094/** 1095 * pci_num_vf - return number of VFs associated with a PF device_release_driver 1096 * @dev: the PCI device 1097 * 1098 * Returns number of VFs, or 0 if SR-IOV is not enabled. 1099 */ 1100int pci_num_vf(struct pci_dev *dev) 1101{ 1102 if (!dev->is_physfn) 1103 return 0; 1104 1105 return dev->sriov->num_VFs; 1106} 1107EXPORT_SYMBOL_GPL(pci_num_vf); 1108 1109/** 1110 * pci_vfs_assigned - returns number of VFs are assigned to a guest 1111 * @dev: the PCI device 1112 * 1113 * Returns number of VFs belonging to this device that are assigned to a guest. 1114 * If device is not a physical function returns 0. 1115 */ 1116int pci_vfs_assigned(struct pci_dev *dev) 1117{ 1118 struct pci_dev *vfdev; 1119 unsigned int vfs_assigned = 0; 1120 unsigned short dev_id; 1121 1122 /* only search if we are a PF */ 1123 if (!dev->is_physfn) 1124 return 0; 1125 1126 /* 1127 * determine the device ID for the VFs, the vendor ID will be the 1128 * same as the PF so there is no need to check for that one 1129 */ 1130 dev_id = dev->sriov->vf_device; 1131 1132 /* loop through all the VFs to see if we own any that are assigned */ 1133 vfdev = pci_get_device(dev->vendor, dev_id, NULL); 1134 while (vfdev) { 1135 /* 1136 * It is considered assigned if it is a virtual function with 1137 * our dev as the physical function and the assigned bit is set 1138 */ 1139 if (vfdev->is_virtfn && (vfdev->physfn == dev) && 1140 pci_is_dev_assigned(vfdev)) 1141 vfs_assigned++; 1142 1143 vfdev = pci_get_device(dev->vendor, dev_id, vfdev); 1144 } 1145 1146 return vfs_assigned; 1147} 1148EXPORT_SYMBOL_GPL(pci_vfs_assigned); 1149 1150/** 1151 * pci_sriov_set_totalvfs -- reduce the TotalVFs available 1152 * @dev: the PCI PF device 1153 * @numvfs: number that should be used for TotalVFs supported 1154 * 1155 * Should be called from PF driver's probe routine with 1156 * device's mutex held. 1157 * 1158 * Returns 0 if PF is an SRIOV-capable device and 1159 * value of numvfs valid. If not a PF return -ENOSYS; 1160 * if numvfs is invalid return -EINVAL; 1161 * if VFs already enabled, return -EBUSY. 1162 */ 1163int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs) 1164{ 1165 if (!dev->is_physfn) 1166 return -ENOSYS; 1167 1168 if (numvfs > dev->sriov->total_VFs) 1169 return -EINVAL; 1170 1171 /* Shouldn't change if VFs already enabled */ 1172 if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE) 1173 return -EBUSY; 1174 1175 dev->sriov->driver_max_VFs = numvfs; 1176 return 0; 1177} 1178EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs); 1179 1180/** 1181 * pci_sriov_get_totalvfs -- get total VFs supported on this device 1182 * @dev: the PCI PF device 1183 * 1184 * For a PCIe device with SRIOV support, return the PCIe 1185 * SRIOV capability value of TotalVFs or the value of driver_max_VFs 1186 * if the driver reduced it. Otherwise 0. 1187 */ 1188int pci_sriov_get_totalvfs(struct pci_dev *dev) 1189{ 1190 if (!dev->is_physfn) 1191 return 0; 1192 1193 return dev->sriov->driver_max_VFs; 1194} 1195EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs); 1196 1197/** 1198 * pci_sriov_configure_simple - helper to configure SR-IOV 1199 * @dev: the PCI device 1200 * @nr_virtfn: number of virtual functions to enable, 0 to disable 1201 * 1202 * Enable or disable SR-IOV for devices that don't require any PF setup 1203 * before enabling SR-IOV. Return value is negative on error, or number of 1204 * VFs allocated on success. 1205 */ 1206int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn) 1207{ 1208 int rc; 1209 1210 might_sleep(); 1211 1212 if (!dev->is_physfn) 1213 return -ENODEV; 1214 1215 if (pci_vfs_assigned(dev)) { 1216 pci_warn(dev, "Cannot modify SR-IOV while VFs are assigned\n"); 1217 return -EPERM; 1218 } 1219 1220 if (nr_virtfn == 0) { 1221 sriov_disable(dev); 1222 return 0; 1223 } 1224 1225 rc = sriov_enable(dev, nr_virtfn); 1226 if (rc < 0) 1227 return rc; 1228 1229 return nr_virtfn; 1230} 1231EXPORT_SYMBOL_GPL(pci_sriov_configure_simple); 1232