1// SPDX-License-Identifier: GPL-2.0-only 2/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ 3#include <linux/platform_device.h> 4#include <linux/memregion.h> 5#include <linux/workqueue.h> 6#include <linux/einj-cxl.h> 7#include <linux/debugfs.h> 8#include <linux/device.h> 9#include <linux/module.h> 10#include <linux/pci.h> 11#include <linux/slab.h> 12#include <linux/idr.h> 13#include <linux/node.h> 14#include <cxlmem.h> 15#include <cxlpci.h> 16#include <cxl.h> 17#include "core.h" 18 19/** 20 * DOC: cxl core 21 * 22 * The CXL core provides a set of interfaces that can be consumed by CXL aware 23 * drivers. The interfaces allow for creation, modification, and destruction of 24 * regions, memory devices, ports, and decoders. CXL aware drivers must register 25 * with the CXL core via these interfaces in order to be able to participate in 26 * cross-device interleave coordination. The CXL core also establishes and 27 * maintains the bridge to the nvdimm subsystem. 28 * 29 * CXL core introduces sysfs hierarchy to control the devices that are 30 * instantiated by the core. 31 */ 32 33/* 34 * All changes to the interleave configuration occur with this lock held 35 * for write. 36 */ 37DECLARE_RWSEM(cxl_region_rwsem); 38 39static DEFINE_IDA(cxl_port_ida); 40static DEFINE_XARRAY(cxl_root_buses); 41 42int cxl_num_decoders_committed(struct cxl_port *port) 43{ 44 lockdep_assert_held(&cxl_region_rwsem); 45 46 return port->commit_end + 1; 47} 48 49static ssize_t devtype_show(struct device *dev, struct device_attribute *attr, 50 char *buf) 51{ 52 return sysfs_emit(buf, "%s\n", dev->type->name); 53} 54static DEVICE_ATTR_RO(devtype); 55 56static int cxl_device_id(const struct device *dev) 57{ 58 if (dev->type == &cxl_nvdimm_bridge_type) 59 return CXL_DEVICE_NVDIMM_BRIDGE; 60 if (dev->type == &cxl_nvdimm_type) 61 return CXL_DEVICE_NVDIMM; 62 if (dev->type == CXL_PMEM_REGION_TYPE()) 63 return CXL_DEVICE_PMEM_REGION; 64 if (dev->type == CXL_DAX_REGION_TYPE()) 65 return CXL_DEVICE_DAX_REGION; 66 if (is_cxl_port(dev)) { 67 if (is_cxl_root(to_cxl_port(dev))) 68 return CXL_DEVICE_ROOT; 69 return CXL_DEVICE_PORT; 70 } 71 if (is_cxl_memdev(dev)) 72 return CXL_DEVICE_MEMORY_EXPANDER; 73 if (dev->type == CXL_REGION_TYPE()) 74 return CXL_DEVICE_REGION; 75 if (dev->type == &cxl_pmu_type) 76 return CXL_DEVICE_PMU; 77 return 0; 78} 79 80static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, 81 char *buf) 82{ 83 return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev)); 84} 85static DEVICE_ATTR_RO(modalias); 86 87static struct attribute *cxl_base_attributes[] = { 88 &dev_attr_devtype.attr, 89 &dev_attr_modalias.attr, 90 NULL, 91}; 92 93struct attribute_group cxl_base_attribute_group = { 94 .attrs = cxl_base_attributes, 95}; 96 97static ssize_t start_show(struct device *dev, struct device_attribute *attr, 98 char *buf) 99{ 100 struct cxl_decoder *cxld = to_cxl_decoder(dev); 101 102 return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start); 103} 104static DEVICE_ATTR_ADMIN_RO(start); 105 106static ssize_t size_show(struct device *dev, struct device_attribute *attr, 107 char *buf) 108{ 109 struct cxl_decoder *cxld = to_cxl_decoder(dev); 110 111 return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range)); 112} 113static DEVICE_ATTR_RO(size); 114 115#define CXL_DECODER_FLAG_ATTR(name, flag) \ 116static ssize_t name##_show(struct device *dev, \ 117 struct device_attribute *attr, char *buf) \ 118{ \ 119 struct cxl_decoder *cxld = to_cxl_decoder(dev); \ 120 \ 121 return sysfs_emit(buf, "%s\n", \ 122 (cxld->flags & (flag)) ? "1" : "0"); \ 123} \ 124static DEVICE_ATTR_RO(name) 125 126CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM); 127CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM); 128CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2); 129CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3); 130CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK); 131 132static ssize_t target_type_show(struct device *dev, 133 struct device_attribute *attr, char *buf) 134{ 135 struct cxl_decoder *cxld = to_cxl_decoder(dev); 136 137 switch (cxld->target_type) { 138 case CXL_DECODER_DEVMEM: 139 return sysfs_emit(buf, "accelerator\n"); 140 case CXL_DECODER_HOSTONLYMEM: 141 return sysfs_emit(buf, "expander\n"); 142 } 143 return -ENXIO; 144} 145static DEVICE_ATTR_RO(target_type); 146 147static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf) 148{ 149 struct cxl_decoder *cxld = &cxlsd->cxld; 150 ssize_t offset = 0; 151 int i, rc = 0; 152 153 for (i = 0; i < cxld->interleave_ways; i++) { 154 struct cxl_dport *dport = cxlsd->target[i]; 155 struct cxl_dport *next = NULL; 156 157 if (!dport) 158 break; 159 160 if (i + 1 < cxld->interleave_ways) 161 next = cxlsd->target[i + 1]; 162 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id, 163 next ? "," : ""); 164 if (rc < 0) 165 return rc; 166 offset += rc; 167 } 168 169 return offset; 170} 171 172static ssize_t target_list_show(struct device *dev, 173 struct device_attribute *attr, char *buf) 174{ 175 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev); 176 ssize_t offset; 177 int rc; 178 179 guard(rwsem_read)(&cxl_region_rwsem); 180 rc = emit_target_list(cxlsd, buf); 181 if (rc < 0) 182 return rc; 183 offset = rc; 184 185 rc = sysfs_emit_at(buf, offset, "\n"); 186 if (rc < 0) 187 return rc; 188 189 return offset + rc; 190} 191static DEVICE_ATTR_RO(target_list); 192 193static ssize_t mode_show(struct device *dev, struct device_attribute *attr, 194 char *buf) 195{ 196 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev); 197 198 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxled->mode)); 199} 200 201static ssize_t mode_store(struct device *dev, struct device_attribute *attr, 202 const char *buf, size_t len) 203{ 204 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev); 205 enum cxl_decoder_mode mode; 206 ssize_t rc; 207 208 if (sysfs_streq(buf, "pmem")) 209 mode = CXL_DECODER_PMEM; 210 else if (sysfs_streq(buf, "ram")) 211 mode = CXL_DECODER_RAM; 212 else 213 return -EINVAL; 214 215 rc = cxl_dpa_set_mode(cxled, mode); 216 if (rc) 217 return rc; 218 219 return len; 220} 221static DEVICE_ATTR_RW(mode); 222 223static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr, 224 char *buf) 225{ 226 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev); 227 228 guard(rwsem_read)(&cxl_dpa_rwsem); 229 return sysfs_emit(buf, "%#llx\n", (u64)cxl_dpa_resource_start(cxled)); 230} 231static DEVICE_ATTR_RO(dpa_resource); 232 233static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr, 234 char *buf) 235{ 236 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev); 237 resource_size_t size = cxl_dpa_size(cxled); 238 239 return sysfs_emit(buf, "%pa\n", &size); 240} 241 242static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr, 243 const char *buf, size_t len) 244{ 245 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev); 246 unsigned long long size; 247 ssize_t rc; 248 249 rc = kstrtoull(buf, 0, &size); 250 if (rc) 251 return rc; 252 253 if (!IS_ALIGNED(size, SZ_256M)) 254 return -EINVAL; 255 256 rc = cxl_dpa_free(cxled); 257 if (rc) 258 return rc; 259 260 if (size == 0) 261 return len; 262 263 rc = cxl_dpa_alloc(cxled, size); 264 if (rc) 265 return rc; 266 267 return len; 268} 269static DEVICE_ATTR_RW(dpa_size); 270 271static ssize_t interleave_granularity_show(struct device *dev, 272 struct device_attribute *attr, 273 char *buf) 274{ 275 struct cxl_decoder *cxld = to_cxl_decoder(dev); 276 277 return sysfs_emit(buf, "%d\n", cxld->interleave_granularity); 278} 279 280static DEVICE_ATTR_RO(interleave_granularity); 281 282static ssize_t interleave_ways_show(struct device *dev, 283 struct device_attribute *attr, char *buf) 284{ 285 struct cxl_decoder *cxld = to_cxl_decoder(dev); 286 287 return sysfs_emit(buf, "%d\n", cxld->interleave_ways); 288} 289 290static DEVICE_ATTR_RO(interleave_ways); 291 292static ssize_t qos_class_show(struct device *dev, 293 struct device_attribute *attr, char *buf) 294{ 295 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 296 297 return sysfs_emit(buf, "%d\n", cxlrd->qos_class); 298} 299static DEVICE_ATTR_RO(qos_class); 300 301static struct attribute *cxl_decoder_base_attrs[] = { 302 &dev_attr_start.attr, 303 &dev_attr_size.attr, 304 &dev_attr_locked.attr, 305 &dev_attr_interleave_granularity.attr, 306 &dev_attr_interleave_ways.attr, 307 NULL, 308}; 309 310static struct attribute_group cxl_decoder_base_attribute_group = { 311 .attrs = cxl_decoder_base_attrs, 312}; 313 314static struct attribute *cxl_decoder_root_attrs[] = { 315 &dev_attr_cap_pmem.attr, 316 &dev_attr_cap_ram.attr, 317 &dev_attr_cap_type2.attr, 318 &dev_attr_cap_type3.attr, 319 &dev_attr_target_list.attr, 320 &dev_attr_qos_class.attr, 321 SET_CXL_REGION_ATTR(create_pmem_region) 322 SET_CXL_REGION_ATTR(create_ram_region) 323 SET_CXL_REGION_ATTR(delete_region) 324 NULL, 325}; 326 327static bool can_create_pmem(struct cxl_root_decoder *cxlrd) 328{ 329 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM; 330 331 return (cxlrd->cxlsd.cxld.flags & flags) == flags; 332} 333 334static bool can_create_ram(struct cxl_root_decoder *cxlrd) 335{ 336 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM; 337 338 return (cxlrd->cxlsd.cxld.flags & flags) == flags; 339} 340 341static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n) 342{ 343 struct device *dev = kobj_to_dev(kobj); 344 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 345 346 if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd)) 347 return 0; 348 349 if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd)) 350 return 0; 351 352 if (a == CXL_REGION_ATTR(delete_region) && 353 !(can_create_pmem(cxlrd) || can_create_ram(cxlrd))) 354 return 0; 355 356 return a->mode; 357} 358 359static struct attribute_group cxl_decoder_root_attribute_group = { 360 .attrs = cxl_decoder_root_attrs, 361 .is_visible = cxl_root_decoder_visible, 362}; 363 364static const struct attribute_group *cxl_decoder_root_attribute_groups[] = { 365 &cxl_decoder_root_attribute_group, 366 &cxl_decoder_base_attribute_group, 367 &cxl_base_attribute_group, 368 NULL, 369}; 370 371static struct attribute *cxl_decoder_switch_attrs[] = { 372 &dev_attr_target_type.attr, 373 &dev_attr_target_list.attr, 374 SET_CXL_REGION_ATTR(region) 375 NULL, 376}; 377 378static struct attribute_group cxl_decoder_switch_attribute_group = { 379 .attrs = cxl_decoder_switch_attrs, 380}; 381 382static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = { 383 &cxl_decoder_switch_attribute_group, 384 &cxl_decoder_base_attribute_group, 385 &cxl_base_attribute_group, 386 NULL, 387}; 388 389static struct attribute *cxl_decoder_endpoint_attrs[] = { 390 &dev_attr_target_type.attr, 391 &dev_attr_mode.attr, 392 &dev_attr_dpa_size.attr, 393 &dev_attr_dpa_resource.attr, 394 SET_CXL_REGION_ATTR(region) 395 NULL, 396}; 397 398static struct attribute_group cxl_decoder_endpoint_attribute_group = { 399 .attrs = cxl_decoder_endpoint_attrs, 400}; 401 402static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = { 403 &cxl_decoder_base_attribute_group, 404 &cxl_decoder_endpoint_attribute_group, 405 &cxl_base_attribute_group, 406 NULL, 407}; 408 409static void __cxl_decoder_release(struct cxl_decoder *cxld) 410{ 411 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 412 413 ida_free(&port->decoder_ida, cxld->id); 414 put_device(&port->dev); 415} 416 417static void cxl_endpoint_decoder_release(struct device *dev) 418{ 419 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev); 420 421 __cxl_decoder_release(&cxled->cxld); 422 kfree(cxled); 423} 424 425static void cxl_switch_decoder_release(struct device *dev) 426{ 427 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev); 428 429 __cxl_decoder_release(&cxlsd->cxld); 430 kfree(cxlsd); 431} 432 433struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev) 434{ 435 if (dev_WARN_ONCE(dev, !is_root_decoder(dev), 436 "not a cxl_root_decoder device\n")) 437 return NULL; 438 return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev); 439} 440EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, CXL); 441 442static void cxl_root_decoder_release(struct device *dev) 443{ 444 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 445 446 if (atomic_read(&cxlrd->region_id) >= 0) 447 memregion_free(atomic_read(&cxlrd->region_id)); 448 __cxl_decoder_release(&cxlrd->cxlsd.cxld); 449 kfree(cxlrd); 450} 451 452static const struct device_type cxl_decoder_endpoint_type = { 453 .name = "cxl_decoder_endpoint", 454 .release = cxl_endpoint_decoder_release, 455 .groups = cxl_decoder_endpoint_attribute_groups, 456}; 457 458static const struct device_type cxl_decoder_switch_type = { 459 .name = "cxl_decoder_switch", 460 .release = cxl_switch_decoder_release, 461 .groups = cxl_decoder_switch_attribute_groups, 462}; 463 464static const struct device_type cxl_decoder_root_type = { 465 .name = "cxl_decoder_root", 466 .release = cxl_root_decoder_release, 467 .groups = cxl_decoder_root_attribute_groups, 468}; 469 470bool is_endpoint_decoder(struct device *dev) 471{ 472 return dev->type == &cxl_decoder_endpoint_type; 473} 474EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, CXL); 475 476bool is_root_decoder(struct device *dev) 477{ 478 return dev->type == &cxl_decoder_root_type; 479} 480EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL); 481 482bool is_switch_decoder(struct device *dev) 483{ 484 return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type; 485} 486EXPORT_SYMBOL_NS_GPL(is_switch_decoder, CXL); 487 488struct cxl_decoder *to_cxl_decoder(struct device *dev) 489{ 490 if (dev_WARN_ONCE(dev, 491 !is_switch_decoder(dev) && !is_endpoint_decoder(dev), 492 "not a cxl_decoder device\n")) 493 return NULL; 494 return container_of(dev, struct cxl_decoder, dev); 495} 496EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL); 497 498struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev) 499{ 500 if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev), 501 "not a cxl_endpoint_decoder device\n")) 502 return NULL; 503 return container_of(dev, struct cxl_endpoint_decoder, cxld.dev); 504} 505EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, CXL); 506 507struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev) 508{ 509 if (dev_WARN_ONCE(dev, !is_switch_decoder(dev), 510 "not a cxl_switch_decoder device\n")) 511 return NULL; 512 return container_of(dev, struct cxl_switch_decoder, cxld.dev); 513} 514EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, CXL); 515 516static void cxl_ep_release(struct cxl_ep *ep) 517{ 518 put_device(ep->ep); 519 kfree(ep); 520} 521 522static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep) 523{ 524 if (!ep) 525 return; 526 xa_erase(&port->endpoints, (unsigned long) ep->ep); 527 cxl_ep_release(ep); 528} 529 530static void cxl_port_release(struct device *dev) 531{ 532 struct cxl_port *port = to_cxl_port(dev); 533 unsigned long index; 534 struct cxl_ep *ep; 535 536 xa_for_each(&port->endpoints, index, ep) 537 cxl_ep_remove(port, ep); 538 xa_destroy(&port->endpoints); 539 xa_destroy(&port->dports); 540 xa_destroy(&port->regions); 541 ida_free(&cxl_port_ida, port->id); 542 if (is_cxl_root(port)) 543 kfree(to_cxl_root(port)); 544 else 545 kfree(port); 546} 547 548static ssize_t decoders_committed_show(struct device *dev, 549 struct device_attribute *attr, char *buf) 550{ 551 struct cxl_port *port = to_cxl_port(dev); 552 int rc; 553 554 down_read(&cxl_region_rwsem); 555 rc = sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port)); 556 up_read(&cxl_region_rwsem); 557 558 return rc; 559} 560 561static DEVICE_ATTR_RO(decoders_committed); 562 563static struct attribute *cxl_port_attrs[] = { 564 &dev_attr_decoders_committed.attr, 565 NULL, 566}; 567 568static struct attribute_group cxl_port_attribute_group = { 569 .attrs = cxl_port_attrs, 570}; 571 572static const struct attribute_group *cxl_port_attribute_groups[] = { 573 &cxl_base_attribute_group, 574 &cxl_port_attribute_group, 575 NULL, 576}; 577 578static const struct device_type cxl_port_type = { 579 .name = "cxl_port", 580 .release = cxl_port_release, 581 .groups = cxl_port_attribute_groups, 582}; 583 584bool is_cxl_port(const struct device *dev) 585{ 586 return dev->type == &cxl_port_type; 587} 588EXPORT_SYMBOL_NS_GPL(is_cxl_port, CXL); 589 590struct cxl_port *to_cxl_port(const struct device *dev) 591{ 592 if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type, 593 "not a cxl_port device\n")) 594 return NULL; 595 return container_of(dev, struct cxl_port, dev); 596} 597EXPORT_SYMBOL_NS_GPL(to_cxl_port, CXL); 598 599static void unregister_port(void *_port) 600{ 601 struct cxl_port *port = _port; 602 struct cxl_port *parent; 603 struct device *lock_dev; 604 605 if (is_cxl_root(port)) 606 parent = NULL; 607 else 608 parent = to_cxl_port(port->dev.parent); 609 610 /* 611 * CXL root port's and the first level of ports are unregistered 612 * under the platform firmware device lock, all other ports are 613 * unregistered while holding their parent port lock. 614 */ 615 if (!parent) 616 lock_dev = port->uport_dev; 617 else if (is_cxl_root(parent)) 618 lock_dev = parent->uport_dev; 619 else 620 lock_dev = &parent->dev; 621 622 device_lock_assert(lock_dev); 623 port->dead = true; 624 device_unregister(&port->dev); 625} 626 627static void cxl_unlink_uport(void *_port) 628{ 629 struct cxl_port *port = _port; 630 631 sysfs_remove_link(&port->dev.kobj, "uport"); 632} 633 634static int devm_cxl_link_uport(struct device *host, struct cxl_port *port) 635{ 636 int rc; 637 638 rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj, 639 "uport"); 640 if (rc) 641 return rc; 642 return devm_add_action_or_reset(host, cxl_unlink_uport, port); 643} 644 645static void cxl_unlink_parent_dport(void *_port) 646{ 647 struct cxl_port *port = _port; 648 649 sysfs_remove_link(&port->dev.kobj, "parent_dport"); 650} 651 652static int devm_cxl_link_parent_dport(struct device *host, 653 struct cxl_port *port, 654 struct cxl_dport *parent_dport) 655{ 656 int rc; 657 658 if (!parent_dport) 659 return 0; 660 661 rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj, 662 "parent_dport"); 663 if (rc) 664 return rc; 665 return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port); 666} 667 668static struct lock_class_key cxl_port_key; 669 670static struct cxl_port *cxl_port_alloc(struct device *uport_dev, 671 struct cxl_dport *parent_dport) 672{ 673 struct cxl_root *cxl_root __free(kfree) = NULL; 674 struct cxl_port *port, *_port __free(kfree) = NULL; 675 struct device *dev; 676 int rc; 677 678 /* No parent_dport, root cxl_port */ 679 if (!parent_dport) { 680 cxl_root = kzalloc(sizeof(*cxl_root), GFP_KERNEL); 681 if (!cxl_root) 682 return ERR_PTR(-ENOMEM); 683 } else { 684 _port = kzalloc(sizeof(*port), GFP_KERNEL); 685 if (!_port) 686 return ERR_PTR(-ENOMEM); 687 } 688 689 rc = ida_alloc(&cxl_port_ida, GFP_KERNEL); 690 if (rc < 0) 691 return ERR_PTR(rc); 692 693 if (cxl_root) 694 port = &no_free_ptr(cxl_root)->port; 695 else 696 port = no_free_ptr(_port); 697 698 port->id = rc; 699 port->uport_dev = uport_dev; 700 701 /* 702 * The top-level cxl_port "cxl_root" does not have a cxl_port as 703 * its parent and it does not have any corresponding component 704 * registers as its decode is described by a fixed platform 705 * description. 706 */ 707 dev = &port->dev; 708 if (parent_dport) { 709 struct cxl_port *parent_port = parent_dport->port; 710 struct cxl_port *iter; 711 712 dev->parent = &parent_port->dev; 713 port->depth = parent_port->depth + 1; 714 port->parent_dport = parent_dport; 715 716 /* 717 * walk to the host bridge, or the first ancestor that knows 718 * the host bridge 719 */ 720 iter = port; 721 while (!iter->host_bridge && 722 !is_cxl_root(to_cxl_port(iter->dev.parent))) 723 iter = to_cxl_port(iter->dev.parent); 724 if (iter->host_bridge) 725 port->host_bridge = iter->host_bridge; 726 else if (parent_dport->rch) 727 port->host_bridge = parent_dport->dport_dev; 728 else 729 port->host_bridge = iter->uport_dev; 730 dev_dbg(uport_dev, "host-bridge: %s\n", 731 dev_name(port->host_bridge)); 732 } else 733 dev->parent = uport_dev; 734 735 ida_init(&port->decoder_ida); 736 port->hdm_end = -1; 737 port->commit_end = -1; 738 xa_init(&port->dports); 739 xa_init(&port->endpoints); 740 xa_init(&port->regions); 741 742 device_initialize(dev); 743 lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth); 744 device_set_pm_not_required(dev); 745 dev->bus = &cxl_bus_type; 746 dev->type = &cxl_port_type; 747 748 return port; 749} 750 751static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map, 752 resource_size_t component_reg_phys) 753{ 754 *map = (struct cxl_register_map) { 755 .host = host, 756 .reg_type = CXL_REGLOC_RBI_EMPTY, 757 .resource = component_reg_phys, 758 }; 759 760 if (component_reg_phys == CXL_RESOURCE_NONE) 761 return 0; 762 763 map->reg_type = CXL_REGLOC_RBI_COMPONENT; 764 map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE; 765 766 return cxl_setup_regs(map); 767} 768 769static int cxl_port_setup_regs(struct cxl_port *port, 770 resource_size_t component_reg_phys) 771{ 772 if (dev_is_platform(port->uport_dev)) 773 return 0; 774 return cxl_setup_comp_regs(&port->dev, &port->reg_map, 775 component_reg_phys); 776} 777 778static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport, 779 resource_size_t component_reg_phys) 780{ 781 int rc; 782 783 if (dev_is_platform(dport->dport_dev)) 784 return 0; 785 786 /* 787 * use @dport->dport_dev for the context for error messages during 788 * register probing, and fixup @host after the fact, since @host may be 789 * NULL. 790 */ 791 rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map, 792 component_reg_phys); 793 dport->reg_map.host = host; 794 return rc; 795} 796 797DEFINE_SHOW_ATTRIBUTE(einj_cxl_available_error_type); 798 799static int cxl_einj_inject(void *data, u64 type) 800{ 801 struct cxl_dport *dport = data; 802 803 if (dport->rch) 804 return einj_cxl_inject_rch_error(dport->rcrb.base, type); 805 806 return einj_cxl_inject_error(to_pci_dev(dport->dport_dev), type); 807} 808DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject, 809 "0x%llx\n"); 810 811static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) 812{ 813 struct dentry *dir; 814 815 if (!einj_cxl_is_initialized()) 816 return; 817 818 /* 819 * dport_dev needs to be a PCIe port for CXL 2.0+ ports because 820 * EINJ expects a dport SBDF to be specified for 2.0 error injection. 821 */ 822 if (!dport->rch && !dev_is_pci(dport->dport_dev)) 823 return; 824 825 dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev)); 826 827 debugfs_create_file("einj_inject", 0200, dir, dport, 828 &cxl_einj_inject_fops); 829} 830 831static struct cxl_port *__devm_cxl_add_port(struct device *host, 832 struct device *uport_dev, 833 resource_size_t component_reg_phys, 834 struct cxl_dport *parent_dport) 835{ 836 struct cxl_port *port; 837 struct device *dev; 838 int rc; 839 840 port = cxl_port_alloc(uport_dev, parent_dport); 841 if (IS_ERR(port)) 842 return port; 843 844 dev = &port->dev; 845 if (is_cxl_memdev(uport_dev)) { 846 struct cxl_memdev *cxlmd = to_cxl_memdev(uport_dev); 847 struct cxl_dev_state *cxlds = cxlmd->cxlds; 848 849 rc = dev_set_name(dev, "endpoint%d", port->id); 850 if (rc) 851 goto err; 852 853 /* 854 * The endpoint driver already enumerated the component and RAS 855 * registers. Reuse that enumeration while prepping them to be 856 * mapped by the cxl_port driver. 857 */ 858 port->reg_map = cxlds->reg_map; 859 port->reg_map.host = &port->dev; 860 cxlmd->endpoint = port; 861 } else if (parent_dport) { 862 rc = dev_set_name(dev, "port%d", port->id); 863 if (rc) 864 goto err; 865 866 rc = cxl_port_setup_regs(port, component_reg_phys); 867 if (rc) 868 goto err; 869 } else 870 rc = dev_set_name(dev, "root%d", port->id); 871 if (rc) 872 goto err; 873 874 rc = device_add(dev); 875 if (rc) 876 goto err; 877 878 rc = devm_add_action_or_reset(host, unregister_port, port); 879 if (rc) 880 return ERR_PTR(rc); 881 882 rc = devm_cxl_link_uport(host, port); 883 if (rc) 884 return ERR_PTR(rc); 885 886 rc = devm_cxl_link_parent_dport(host, port, parent_dport); 887 if (rc) 888 return ERR_PTR(rc); 889 890 if (parent_dport && dev_is_pci(uport_dev)) 891 port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev)); 892 893 return port; 894 895err: 896 put_device(dev); 897 return ERR_PTR(rc); 898} 899 900/** 901 * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy 902 * @host: host device for devm operations 903 * @uport_dev: "physical" device implementing this upstream port 904 * @component_reg_phys: (optional) for configurable cxl_port instances 905 * @parent_dport: next hop up in the CXL memory decode hierarchy 906 */ 907struct cxl_port *devm_cxl_add_port(struct device *host, 908 struct device *uport_dev, 909 resource_size_t component_reg_phys, 910 struct cxl_dport *parent_dport) 911{ 912 struct cxl_port *port, *parent_port; 913 914 port = __devm_cxl_add_port(host, uport_dev, component_reg_phys, 915 parent_dport); 916 917 parent_port = parent_dport ? parent_dport->port : NULL; 918 if (IS_ERR(port)) { 919 dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n", 920 parent_port ? " port to " : "", 921 parent_port ? dev_name(&parent_port->dev) : "", 922 parent_port ? "" : " root port", 923 PTR_ERR(port)); 924 } else { 925 dev_dbg(uport_dev, "%s added%s%s%s\n", 926 dev_name(&port->dev), 927 parent_port ? " to " : "", 928 parent_port ? dev_name(&parent_port->dev) : "", 929 parent_port ? "" : " (root port)"); 930 } 931 932 return port; 933} 934EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL); 935 936struct cxl_root *devm_cxl_add_root(struct device *host, 937 const struct cxl_root_ops *ops) 938{ 939 struct cxl_root *cxl_root; 940 struct cxl_port *port; 941 942 port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL); 943 if (IS_ERR(port)) 944 return (struct cxl_root *)port; 945 946 cxl_root = to_cxl_root(port); 947 cxl_root->ops = ops; 948 return cxl_root; 949} 950EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, CXL); 951 952struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port) 953{ 954 /* There is no pci_bus associated with a CXL platform-root port */ 955 if (is_cxl_root(port)) 956 return NULL; 957 958 if (dev_is_pci(port->uport_dev)) { 959 struct pci_dev *pdev = to_pci_dev(port->uport_dev); 960 961 return pdev->subordinate; 962 } 963 964 return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev); 965} 966EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL); 967 968static void unregister_pci_bus(void *uport_dev) 969{ 970 xa_erase(&cxl_root_buses, (unsigned long)uport_dev); 971} 972 973int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev, 974 struct pci_bus *bus) 975{ 976 int rc; 977 978 if (dev_is_pci(uport_dev)) 979 return -EINVAL; 980 981 rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus, 982 GFP_KERNEL); 983 if (rc) 984 return rc; 985 return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev); 986} 987EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL); 988 989static bool dev_is_cxl_root_child(struct device *dev) 990{ 991 struct cxl_port *port, *parent; 992 993 if (!is_cxl_port(dev)) 994 return false; 995 996 port = to_cxl_port(dev); 997 if (is_cxl_root(port)) 998 return false; 999 1000 parent = to_cxl_port(port->dev.parent); 1001 if (is_cxl_root(parent)) 1002 return true; 1003 1004 return false; 1005} 1006 1007struct cxl_root *find_cxl_root(struct cxl_port *port) 1008{ 1009 struct cxl_port *iter = port; 1010 1011 while (iter && !is_cxl_root(iter)) 1012 iter = to_cxl_port(iter->dev.parent); 1013 1014 if (!iter) 1015 return NULL; 1016 get_device(&iter->dev); 1017 return to_cxl_root(iter); 1018} 1019EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL); 1020 1021void put_cxl_root(struct cxl_root *cxl_root) 1022{ 1023 if (!cxl_root) 1024 return; 1025 1026 put_device(&cxl_root->port.dev); 1027} 1028EXPORT_SYMBOL_NS_GPL(put_cxl_root, CXL); 1029 1030static struct cxl_dport *find_dport(struct cxl_port *port, int id) 1031{ 1032 struct cxl_dport *dport; 1033 unsigned long index; 1034 1035 device_lock_assert(&port->dev); 1036 xa_for_each(&port->dports, index, dport) 1037 if (dport->port_id == id) 1038 return dport; 1039 return NULL; 1040} 1041 1042static int add_dport(struct cxl_port *port, struct cxl_dport *dport) 1043{ 1044 struct cxl_dport *dup; 1045 int rc; 1046 1047 device_lock_assert(&port->dev); 1048 dup = find_dport(port, dport->port_id); 1049 if (dup) { 1050 dev_err(&port->dev, 1051 "unable to add dport%d-%s non-unique port id (%s)\n", 1052 dport->port_id, dev_name(dport->dport_dev), 1053 dev_name(dup->dport_dev)); 1054 return -EBUSY; 1055 } 1056 1057 rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport, 1058 GFP_KERNEL); 1059 if (rc) 1060 return rc; 1061 1062 port->nr_dports++; 1063 return 0; 1064} 1065 1066/* 1067 * Since root-level CXL dports cannot be enumerated by PCI they are not 1068 * enumerated by the common port driver that acquires the port lock over 1069 * dport add/remove. Instead, root dports are manually added by a 1070 * platform driver and cond_cxl_root_lock() is used to take the missing 1071 * port lock in that case. 1072 */ 1073static void cond_cxl_root_lock(struct cxl_port *port) 1074{ 1075 if (is_cxl_root(port)) 1076 device_lock(&port->dev); 1077} 1078 1079static void cond_cxl_root_unlock(struct cxl_port *port) 1080{ 1081 if (is_cxl_root(port)) 1082 device_unlock(&port->dev); 1083} 1084 1085static void cxl_dport_remove(void *data) 1086{ 1087 struct cxl_dport *dport = data; 1088 struct cxl_port *port = dport->port; 1089 1090 xa_erase(&port->dports, (unsigned long) dport->dport_dev); 1091 put_device(dport->dport_dev); 1092} 1093 1094static void cxl_dport_unlink(void *data) 1095{ 1096 struct cxl_dport *dport = data; 1097 struct cxl_port *port = dport->port; 1098 char link_name[CXL_TARGET_STRLEN]; 1099 1100 sprintf(link_name, "dport%d", dport->port_id); 1101 sysfs_remove_link(&port->dev.kobj, link_name); 1102} 1103 1104static struct cxl_dport * 1105__devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, 1106 int port_id, resource_size_t component_reg_phys, 1107 resource_size_t rcrb) 1108{ 1109 char link_name[CXL_TARGET_STRLEN]; 1110 struct cxl_dport *dport; 1111 struct device *host; 1112 int rc; 1113 1114 if (is_cxl_root(port)) 1115 host = port->uport_dev; 1116 else 1117 host = &port->dev; 1118 1119 if (!host->driver) { 1120 dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n", 1121 dev_name(dport_dev)); 1122 return ERR_PTR(-ENXIO); 1123 } 1124 1125 if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >= 1126 CXL_TARGET_STRLEN) 1127 return ERR_PTR(-EINVAL); 1128 1129 dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL); 1130 if (!dport) 1131 return ERR_PTR(-ENOMEM); 1132 1133 dport->dport_dev = dport_dev; 1134 dport->port_id = port_id; 1135 dport->port = port; 1136 1137 if (rcrb == CXL_RESOURCE_NONE) { 1138 rc = cxl_dport_setup_regs(&port->dev, dport, 1139 component_reg_phys); 1140 if (rc) 1141 return ERR_PTR(rc); 1142 } else { 1143 dport->rcrb.base = rcrb; 1144 component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb, 1145 CXL_RCRB_DOWNSTREAM); 1146 if (component_reg_phys == CXL_RESOURCE_NONE) { 1147 dev_warn(dport_dev, "Invalid Component Registers in RCRB"); 1148 return ERR_PTR(-ENXIO); 1149 } 1150 1151 /* 1152 * RCH @dport is not ready to map until associated with its 1153 * memdev 1154 */ 1155 rc = cxl_dport_setup_regs(NULL, dport, component_reg_phys); 1156 if (rc) 1157 return ERR_PTR(rc); 1158 1159 dport->rch = true; 1160 } 1161 1162 if (component_reg_phys != CXL_RESOURCE_NONE) 1163 dev_dbg(dport_dev, "Component Registers found for dport: %pa\n", 1164 &component_reg_phys); 1165 1166 cond_cxl_root_lock(port); 1167 rc = add_dport(port, dport); 1168 cond_cxl_root_unlock(port); 1169 if (rc) 1170 return ERR_PTR(rc); 1171 1172 get_device(dport_dev); 1173 rc = devm_add_action_or_reset(host, cxl_dport_remove, dport); 1174 if (rc) 1175 return ERR_PTR(rc); 1176 1177 rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name); 1178 if (rc) 1179 return ERR_PTR(rc); 1180 1181 rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport); 1182 if (rc) 1183 return ERR_PTR(rc); 1184 1185 if (dev_is_pci(dport_dev)) 1186 dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev)); 1187 1188 cxl_debugfs_create_dport_dir(dport); 1189 1190 return dport; 1191} 1192 1193/** 1194 * devm_cxl_add_dport - append VH downstream port data to a cxl_port 1195 * @port: the cxl_port that references this dport 1196 * @dport_dev: firmware or PCI device representing the dport 1197 * @port_id: identifier for this dport in a decoder's target list 1198 * @component_reg_phys: optional location of CXL component registers 1199 * 1200 * Note that dports are appended to the devm release action's of the 1201 * either the port's host (for root ports), or the port itself (for 1202 * switch ports) 1203 */ 1204struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port, 1205 struct device *dport_dev, int port_id, 1206 resource_size_t component_reg_phys) 1207{ 1208 struct cxl_dport *dport; 1209 1210 dport = __devm_cxl_add_dport(port, dport_dev, port_id, 1211 component_reg_phys, CXL_RESOURCE_NONE); 1212 if (IS_ERR(dport)) { 1213 dev_dbg(dport_dev, "failed to add dport to %s: %ld\n", 1214 dev_name(&port->dev), PTR_ERR(dport)); 1215 } else { 1216 dev_dbg(dport_dev, "dport added to %s\n", 1217 dev_name(&port->dev)); 1218 } 1219 1220 return dport; 1221} 1222EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL); 1223 1224/** 1225 * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port 1226 * @port: the cxl_port that references this dport 1227 * @dport_dev: firmware or PCI device representing the dport 1228 * @port_id: identifier for this dport in a decoder's target list 1229 * @rcrb: mandatory location of a Root Complex Register Block 1230 * 1231 * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH 1232 */ 1233struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, 1234 struct device *dport_dev, int port_id, 1235 resource_size_t rcrb) 1236{ 1237 struct cxl_dport *dport; 1238 1239 if (rcrb == CXL_RESOURCE_NONE) { 1240 dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n"); 1241 return ERR_PTR(-EINVAL); 1242 } 1243 1244 dport = __devm_cxl_add_dport(port, dport_dev, port_id, 1245 CXL_RESOURCE_NONE, rcrb); 1246 if (IS_ERR(dport)) { 1247 dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n", 1248 dev_name(&port->dev), PTR_ERR(dport)); 1249 } else { 1250 dev_dbg(dport_dev, "RCH dport added to %s\n", 1251 dev_name(&port->dev)); 1252 } 1253 1254 return dport; 1255} 1256EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, CXL); 1257 1258static int add_ep(struct cxl_ep *new) 1259{ 1260 struct cxl_port *port = new->dport->port; 1261 int rc; 1262 1263 device_lock(&port->dev); 1264 if (port->dead) { 1265 device_unlock(&port->dev); 1266 return -ENXIO; 1267 } 1268 rc = xa_insert(&port->endpoints, (unsigned long)new->ep, new, 1269 GFP_KERNEL); 1270 device_unlock(&port->dev); 1271 1272 return rc; 1273} 1274 1275/** 1276 * cxl_add_ep - register an endpoint's interest in a port 1277 * @dport: the dport that routes to @ep_dev 1278 * @ep_dev: device representing the endpoint 1279 * 1280 * Intermediate CXL ports are scanned based on the arrival of endpoints. 1281 * When those endpoints depart the port can be destroyed once all 1282 * endpoints that care about that port have been removed. 1283 */ 1284static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev) 1285{ 1286 struct cxl_ep *ep; 1287 int rc; 1288 1289 ep = kzalloc(sizeof(*ep), GFP_KERNEL); 1290 if (!ep) 1291 return -ENOMEM; 1292 1293 ep->ep = get_device(ep_dev); 1294 ep->dport = dport; 1295 1296 rc = add_ep(ep); 1297 if (rc) 1298 cxl_ep_release(ep); 1299 return rc; 1300} 1301 1302struct cxl_find_port_ctx { 1303 const struct device *dport_dev; 1304 const struct cxl_port *parent_port; 1305 struct cxl_dport **dport; 1306}; 1307 1308static int match_port_by_dport(struct device *dev, const void *data) 1309{ 1310 const struct cxl_find_port_ctx *ctx = data; 1311 struct cxl_dport *dport; 1312 struct cxl_port *port; 1313 1314 if (!is_cxl_port(dev)) 1315 return 0; 1316 if (ctx->parent_port && dev->parent != &ctx->parent_port->dev) 1317 return 0; 1318 1319 port = to_cxl_port(dev); 1320 dport = cxl_find_dport_by_dev(port, ctx->dport_dev); 1321 if (ctx->dport) 1322 *ctx->dport = dport; 1323 return dport != NULL; 1324} 1325 1326static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx) 1327{ 1328 struct device *dev; 1329 1330 if (!ctx->dport_dev) 1331 return NULL; 1332 1333 dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport); 1334 if (dev) 1335 return to_cxl_port(dev); 1336 return NULL; 1337} 1338 1339static struct cxl_port *find_cxl_port(struct device *dport_dev, 1340 struct cxl_dport **dport) 1341{ 1342 struct cxl_find_port_ctx ctx = { 1343 .dport_dev = dport_dev, 1344 .dport = dport, 1345 }; 1346 struct cxl_port *port; 1347 1348 port = __find_cxl_port(&ctx); 1349 return port; 1350} 1351 1352static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port, 1353 struct device *dport_dev, 1354 struct cxl_dport **dport) 1355{ 1356 struct cxl_find_port_ctx ctx = { 1357 .dport_dev = dport_dev, 1358 .parent_port = parent_port, 1359 .dport = dport, 1360 }; 1361 struct cxl_port *port; 1362 1363 port = __find_cxl_port(&ctx); 1364 return port; 1365} 1366 1367/* 1368 * All users of grandparent() are using it to walk PCIe-like switch port 1369 * hierarchy. A PCIe switch is comprised of a bridge device representing the 1370 * upstream switch port and N bridges representing downstream switch ports. When 1371 * bridges stack the grand-parent of a downstream switch port is another 1372 * downstream switch port in the immediate ancestor switch. 1373 */ 1374static struct device *grandparent(struct device *dev) 1375{ 1376 if (dev && dev->parent) 1377 return dev->parent->parent; 1378 return NULL; 1379} 1380 1381static struct device *endpoint_host(struct cxl_port *endpoint) 1382{ 1383 struct cxl_port *port = to_cxl_port(endpoint->dev.parent); 1384 1385 if (is_cxl_root(port)) 1386 return port->uport_dev; 1387 return &port->dev; 1388} 1389 1390static void delete_endpoint(void *data) 1391{ 1392 struct cxl_memdev *cxlmd = data; 1393 struct cxl_port *endpoint = cxlmd->endpoint; 1394 struct device *host = endpoint_host(endpoint); 1395 1396 device_lock(host); 1397 if (host->driver && !endpoint->dead) { 1398 devm_release_action(host, cxl_unlink_parent_dport, endpoint); 1399 devm_release_action(host, cxl_unlink_uport, endpoint); 1400 devm_release_action(host, unregister_port, endpoint); 1401 } 1402 cxlmd->endpoint = NULL; 1403 device_unlock(host); 1404 put_device(&endpoint->dev); 1405 put_device(host); 1406} 1407 1408int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint) 1409{ 1410 struct device *host = endpoint_host(endpoint); 1411 struct device *dev = &cxlmd->dev; 1412 1413 get_device(host); 1414 get_device(&endpoint->dev); 1415 cxlmd->depth = endpoint->depth; 1416 return devm_add_action_or_reset(dev, delete_endpoint, cxlmd); 1417} 1418EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL); 1419 1420/* 1421 * The natural end of life of a non-root 'cxl_port' is when its parent port goes 1422 * through a ->remove() event ("top-down" unregistration). The unnatural trigger 1423 * for a port to be unregistered is when all memdevs beneath that port have gone 1424 * through ->remove(). This "bottom-up" removal selectively removes individual 1425 * child ports manually. This depends on devm_cxl_add_port() to not change is 1426 * devm action registration order, and for dports to have already been 1427 * destroyed by reap_dports(). 1428 */ 1429static void delete_switch_port(struct cxl_port *port) 1430{ 1431 devm_release_action(port->dev.parent, cxl_unlink_parent_dport, port); 1432 devm_release_action(port->dev.parent, cxl_unlink_uport, port); 1433 devm_release_action(port->dev.parent, unregister_port, port); 1434} 1435 1436static void reap_dports(struct cxl_port *port) 1437{ 1438 struct cxl_dport *dport; 1439 unsigned long index; 1440 1441 device_lock_assert(&port->dev); 1442 1443 xa_for_each(&port->dports, index, dport) { 1444 devm_release_action(&port->dev, cxl_dport_unlink, dport); 1445 devm_release_action(&port->dev, cxl_dport_remove, dport); 1446 devm_kfree(&port->dev, dport); 1447 } 1448} 1449 1450struct detach_ctx { 1451 struct cxl_memdev *cxlmd; 1452 int depth; 1453}; 1454 1455static int port_has_memdev(struct device *dev, const void *data) 1456{ 1457 const struct detach_ctx *ctx = data; 1458 struct cxl_port *port; 1459 1460 if (!is_cxl_port(dev)) 1461 return 0; 1462 1463 port = to_cxl_port(dev); 1464 if (port->depth != ctx->depth) 1465 return 0; 1466 1467 return !!cxl_ep_load(port, ctx->cxlmd); 1468} 1469 1470static void cxl_detach_ep(void *data) 1471{ 1472 struct cxl_memdev *cxlmd = data; 1473 1474 for (int i = cxlmd->depth - 1; i >= 1; i--) { 1475 struct cxl_port *port, *parent_port; 1476 struct detach_ctx ctx = { 1477 .cxlmd = cxlmd, 1478 .depth = i, 1479 }; 1480 struct device *dev; 1481 struct cxl_ep *ep; 1482 bool died = false; 1483 1484 dev = bus_find_device(&cxl_bus_type, NULL, &ctx, 1485 port_has_memdev); 1486 if (!dev) 1487 continue; 1488 port = to_cxl_port(dev); 1489 1490 parent_port = to_cxl_port(port->dev.parent); 1491 device_lock(&parent_port->dev); 1492 device_lock(&port->dev); 1493 ep = cxl_ep_load(port, cxlmd); 1494 dev_dbg(&cxlmd->dev, "disconnect %s from %s\n", 1495 ep ? dev_name(ep->ep) : "", dev_name(&port->dev)); 1496 cxl_ep_remove(port, ep); 1497 if (ep && !port->dead && xa_empty(&port->endpoints) && 1498 !is_cxl_root(parent_port) && parent_port->dev.driver) { 1499 /* 1500 * This was the last ep attached to a dynamically 1501 * enumerated port. Block new cxl_add_ep() and garbage 1502 * collect the port. 1503 */ 1504 died = true; 1505 port->dead = true; 1506 reap_dports(port); 1507 } 1508 device_unlock(&port->dev); 1509 1510 if (died) { 1511 dev_dbg(&cxlmd->dev, "delete %s\n", 1512 dev_name(&port->dev)); 1513 delete_switch_port(port); 1514 } 1515 put_device(&port->dev); 1516 device_unlock(&parent_port->dev); 1517 } 1518} 1519 1520static resource_size_t find_component_registers(struct device *dev) 1521{ 1522 struct cxl_register_map map; 1523 struct pci_dev *pdev; 1524 1525 /* 1526 * Theoretically, CXL component registers can be hosted on a 1527 * non-PCI device, in practice, only cxl_test hits this case. 1528 */ 1529 if (!dev_is_pci(dev)) 1530 return CXL_RESOURCE_NONE; 1531 1532 pdev = to_pci_dev(dev); 1533 1534 cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map); 1535 return map.resource; 1536} 1537 1538static int add_port_attach_ep(struct cxl_memdev *cxlmd, 1539 struct device *uport_dev, 1540 struct device *dport_dev) 1541{ 1542 struct device *dparent = grandparent(dport_dev); 1543 struct cxl_port *port, *parent_port = NULL; 1544 struct cxl_dport *dport, *parent_dport; 1545 resource_size_t component_reg_phys; 1546 int rc; 1547 1548 if (!dparent) { 1549 /* 1550 * The iteration reached the topology root without finding the 1551 * CXL-root 'cxl_port' on a previous iteration, fail for now to 1552 * be re-probed after platform driver attaches. 1553 */ 1554 dev_dbg(&cxlmd->dev, "%s is a root dport\n", 1555 dev_name(dport_dev)); 1556 return -ENXIO; 1557 } 1558 1559 parent_port = find_cxl_port(dparent, &parent_dport); 1560 if (!parent_port) { 1561 /* iterate to create this parent_port */ 1562 return -EAGAIN; 1563 } 1564 1565 device_lock(&parent_port->dev); 1566 if (!parent_port->dev.driver) { 1567 dev_warn(&cxlmd->dev, 1568 "port %s:%s disabled, failed to enumerate CXL.mem\n", 1569 dev_name(&parent_port->dev), dev_name(uport_dev)); 1570 port = ERR_PTR(-ENXIO); 1571 goto out; 1572 } 1573 1574 port = find_cxl_port_at(parent_port, dport_dev, &dport); 1575 if (!port) { 1576 component_reg_phys = find_component_registers(uport_dev); 1577 port = devm_cxl_add_port(&parent_port->dev, uport_dev, 1578 component_reg_phys, parent_dport); 1579 /* retry find to pick up the new dport information */ 1580 if (!IS_ERR(port)) 1581 port = find_cxl_port_at(parent_port, dport_dev, &dport); 1582 } 1583out: 1584 device_unlock(&parent_port->dev); 1585 1586 if (IS_ERR(port)) 1587 rc = PTR_ERR(port); 1588 else { 1589 dev_dbg(&cxlmd->dev, "add to new port %s:%s\n", 1590 dev_name(&port->dev), dev_name(port->uport_dev)); 1591 rc = cxl_add_ep(dport, &cxlmd->dev); 1592 if (rc == -EBUSY) { 1593 /* 1594 * "can't" happen, but this error code means 1595 * something to the caller, so translate it. 1596 */ 1597 rc = -ENXIO; 1598 } 1599 put_device(&port->dev); 1600 } 1601 1602 put_device(&parent_port->dev); 1603 return rc; 1604} 1605 1606int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd) 1607{ 1608 struct device *dev = &cxlmd->dev; 1609 struct device *iter; 1610 int rc; 1611 1612 /* 1613 * Skip intermediate port enumeration in the RCH case, there 1614 * are no ports in between a host bridge and an endpoint. 1615 */ 1616 if (cxlmd->cxlds->rcd) 1617 return 0; 1618 1619 rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd); 1620 if (rc) 1621 return rc; 1622 1623 /* 1624 * Scan for and add all cxl_ports in this device's ancestry. 1625 * Repeat until no more ports are added. Abort if a port add 1626 * attempt fails. 1627 */ 1628retry: 1629 for (iter = dev; iter; iter = grandparent(iter)) { 1630 struct device *dport_dev = grandparent(iter); 1631 struct device *uport_dev; 1632 struct cxl_dport *dport; 1633 struct cxl_port *port; 1634 1635 /* 1636 * The terminal "grandparent" in PCI is NULL and @platform_bus 1637 * for platform devices 1638 */ 1639 if (!dport_dev || dport_dev == &platform_bus) 1640 return 0; 1641 1642 uport_dev = dport_dev->parent; 1643 if (!uport_dev) { 1644 dev_warn(dev, "at %s no parent for dport: %s\n", 1645 dev_name(iter), dev_name(dport_dev)); 1646 return -ENXIO; 1647 } 1648 1649 dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n", 1650 dev_name(iter), dev_name(dport_dev), 1651 dev_name(uport_dev)); 1652 port = find_cxl_port(dport_dev, &dport); 1653 if (port) { 1654 dev_dbg(&cxlmd->dev, 1655 "found already registered port %s:%s\n", 1656 dev_name(&port->dev), 1657 dev_name(port->uport_dev)); 1658 rc = cxl_add_ep(dport, &cxlmd->dev); 1659 1660 /* 1661 * If the endpoint already exists in the port's list, 1662 * that's ok, it was added on a previous pass. 1663 * Otherwise, retry in add_port_attach_ep() after taking 1664 * the parent_port lock as the current port may be being 1665 * reaped. 1666 */ 1667 if (rc && rc != -EBUSY) { 1668 put_device(&port->dev); 1669 return rc; 1670 } 1671 1672 /* Any more ports to add between this one and the root? */ 1673 if (!dev_is_cxl_root_child(&port->dev)) { 1674 put_device(&port->dev); 1675 continue; 1676 } 1677 1678 put_device(&port->dev); 1679 return 0; 1680 } 1681 1682 rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev); 1683 /* port missing, try to add parent */ 1684 if (rc == -EAGAIN) 1685 continue; 1686 /* failed to add ep or port */ 1687 if (rc) 1688 return rc; 1689 /* port added, new descendants possible, start over */ 1690 goto retry; 1691 } 1692 1693 return 0; 1694} 1695EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL); 1696 1697struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev, 1698 struct cxl_dport **dport) 1699{ 1700 return find_cxl_port(pdev->dev.parent, dport); 1701} 1702EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, CXL); 1703 1704struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd, 1705 struct cxl_dport **dport) 1706{ 1707 return find_cxl_port(grandparent(&cxlmd->dev), dport); 1708} 1709EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL); 1710 1711static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd, 1712 struct cxl_port *port, int *target_map) 1713{ 1714 int i; 1715 1716 if (!target_map) 1717 return 0; 1718 1719 device_lock_assert(&port->dev); 1720 1721 if (xa_empty(&port->dports)) 1722 return -EINVAL; 1723 1724 guard(rwsem_write)(&cxl_region_rwsem); 1725 for (i = 0; i < cxlsd->cxld.interleave_ways; i++) { 1726 struct cxl_dport *dport = find_dport(port, target_map[i]); 1727 1728 if (!dport) 1729 return -ENXIO; 1730 cxlsd->target[i] = dport; 1731 } 1732 1733 return 0; 1734} 1735 1736struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos) 1737{ 1738 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd; 1739 struct cxl_decoder *cxld = &cxlsd->cxld; 1740 int iw; 1741 1742 iw = cxld->interleave_ways; 1743 if (dev_WARN_ONCE(&cxld->dev, iw != cxlsd->nr_targets, 1744 "misconfigured root decoder\n")) 1745 return NULL; 1746 1747 return cxlrd->cxlsd.target[pos % iw]; 1748} 1749EXPORT_SYMBOL_NS_GPL(cxl_hb_modulo, CXL); 1750 1751static struct lock_class_key cxl_decoder_key; 1752 1753/** 1754 * cxl_decoder_init - Common decoder setup / initialization 1755 * @port: owning port of this decoder 1756 * @cxld: common decoder properties to initialize 1757 * 1758 * A port may contain one or more decoders. Each of those decoders 1759 * enable some address space for CXL.mem utilization. A decoder is 1760 * expected to be configured by the caller before registering via 1761 * cxl_decoder_add() 1762 */ 1763static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld) 1764{ 1765 struct device *dev; 1766 int rc; 1767 1768 rc = ida_alloc(&port->decoder_ida, GFP_KERNEL); 1769 if (rc < 0) 1770 return rc; 1771 1772 /* need parent to stick around to release the id */ 1773 get_device(&port->dev); 1774 cxld->id = rc; 1775 1776 dev = &cxld->dev; 1777 device_initialize(dev); 1778 lockdep_set_class(&dev->mutex, &cxl_decoder_key); 1779 device_set_pm_not_required(dev); 1780 dev->parent = &port->dev; 1781 dev->bus = &cxl_bus_type; 1782 1783 /* Pre initialize an "empty" decoder */ 1784 cxld->interleave_ways = 1; 1785 cxld->interleave_granularity = PAGE_SIZE; 1786 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 1787 cxld->hpa_range = (struct range) { 1788 .start = 0, 1789 .end = -1, 1790 }; 1791 1792 return 0; 1793} 1794 1795static int cxl_switch_decoder_init(struct cxl_port *port, 1796 struct cxl_switch_decoder *cxlsd, 1797 int nr_targets) 1798{ 1799 if (nr_targets > CXL_DECODER_MAX_INTERLEAVE) 1800 return -EINVAL; 1801 1802 cxlsd->nr_targets = nr_targets; 1803 return cxl_decoder_init(port, &cxlsd->cxld); 1804} 1805 1806/** 1807 * cxl_root_decoder_alloc - Allocate a root level decoder 1808 * @port: owning CXL root of this decoder 1809 * @nr_targets: static number of downstream targets 1810 * @calc_hb: which host bridge covers the n'th position by granularity 1811 * 1812 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A 1813 * 'CXL root' decoder is one that decodes from a top-level / static platform 1814 * firmware description of CXL resources into a CXL standard decode 1815 * topology. 1816 */ 1817struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port, 1818 unsigned int nr_targets, 1819 cxl_calc_hb_fn calc_hb) 1820{ 1821 struct cxl_root_decoder *cxlrd; 1822 struct cxl_switch_decoder *cxlsd; 1823 struct cxl_decoder *cxld; 1824 int rc; 1825 1826 if (!is_cxl_root(port)) 1827 return ERR_PTR(-EINVAL); 1828 1829 cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets), 1830 GFP_KERNEL); 1831 if (!cxlrd) 1832 return ERR_PTR(-ENOMEM); 1833 1834 cxlsd = &cxlrd->cxlsd; 1835 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets); 1836 if (rc) { 1837 kfree(cxlrd); 1838 return ERR_PTR(rc); 1839 } 1840 1841 cxlrd->calc_hb = calc_hb; 1842 mutex_init(&cxlrd->range_lock); 1843 1844 cxld = &cxlsd->cxld; 1845 cxld->dev.type = &cxl_decoder_root_type; 1846 /* 1847 * cxl_root_decoder_release() special cases negative ids to 1848 * detect memregion_alloc() failures. 1849 */ 1850 atomic_set(&cxlrd->region_id, -1); 1851 rc = memregion_alloc(GFP_KERNEL); 1852 if (rc < 0) { 1853 put_device(&cxld->dev); 1854 return ERR_PTR(rc); 1855 } 1856 1857 atomic_set(&cxlrd->region_id, rc); 1858 cxlrd->qos_class = CXL_QOS_CLASS_INVALID; 1859 return cxlrd; 1860} 1861EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL); 1862 1863/** 1864 * cxl_switch_decoder_alloc - Allocate a switch level decoder 1865 * @port: owning CXL switch port of this decoder 1866 * @nr_targets: max number of dynamically addressable downstream targets 1867 * 1868 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A 1869 * 'switch' decoder is any decoder that can be enumerated by PCIe 1870 * topology and the HDM Decoder Capability. This includes the decoders 1871 * that sit between Switch Upstream Ports / Switch Downstream Ports and 1872 * Host Bridges / Root Ports. 1873 */ 1874struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port, 1875 unsigned int nr_targets) 1876{ 1877 struct cxl_switch_decoder *cxlsd; 1878 struct cxl_decoder *cxld; 1879 int rc; 1880 1881 if (is_cxl_root(port) || is_cxl_endpoint(port)) 1882 return ERR_PTR(-EINVAL); 1883 1884 cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL); 1885 if (!cxlsd) 1886 return ERR_PTR(-ENOMEM); 1887 1888 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets); 1889 if (rc) { 1890 kfree(cxlsd); 1891 return ERR_PTR(rc); 1892 } 1893 1894 cxld = &cxlsd->cxld; 1895 cxld->dev.type = &cxl_decoder_switch_type; 1896 return cxlsd; 1897} 1898EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL); 1899 1900/** 1901 * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder 1902 * @port: owning port of this decoder 1903 * 1904 * Return: A new cxl decoder to be registered by cxl_decoder_add() 1905 */ 1906struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port) 1907{ 1908 struct cxl_endpoint_decoder *cxled; 1909 struct cxl_decoder *cxld; 1910 int rc; 1911 1912 if (!is_cxl_endpoint(port)) 1913 return ERR_PTR(-EINVAL); 1914 1915 cxled = kzalloc(sizeof(*cxled), GFP_KERNEL); 1916 if (!cxled) 1917 return ERR_PTR(-ENOMEM); 1918 1919 cxled->pos = -1; 1920 cxld = &cxled->cxld; 1921 rc = cxl_decoder_init(port, cxld); 1922 if (rc) { 1923 kfree(cxled); 1924 return ERR_PTR(rc); 1925 } 1926 1927 cxld->dev.type = &cxl_decoder_endpoint_type; 1928 return cxled; 1929} 1930EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL); 1931 1932/** 1933 * cxl_decoder_add_locked - Add a decoder with targets 1934 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc() 1935 * @target_map: A list of downstream ports that this decoder can direct memory 1936 * traffic to. These numbers should correspond with the port number 1937 * in the PCIe Link Capabilities structure. 1938 * 1939 * Certain types of decoders may not have any targets. The main example of this 1940 * is an endpoint device. A more awkward example is a hostbridge whose root 1941 * ports get hot added (technically possible, though unlikely). 1942 * 1943 * This is the locked variant of cxl_decoder_add(). 1944 * 1945 * Context: Process context. Expects the device lock of the port that owns the 1946 * @cxld to be held. 1947 * 1948 * Return: Negative error code if the decoder wasn't properly configured; else 1949 * returns 0. 1950 */ 1951int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map) 1952{ 1953 struct cxl_port *port; 1954 struct device *dev; 1955 int rc; 1956 1957 if (WARN_ON_ONCE(!cxld)) 1958 return -EINVAL; 1959 1960 if (WARN_ON_ONCE(IS_ERR(cxld))) 1961 return PTR_ERR(cxld); 1962 1963 if (cxld->interleave_ways < 1) 1964 return -EINVAL; 1965 1966 dev = &cxld->dev; 1967 1968 port = to_cxl_port(cxld->dev.parent); 1969 if (!is_endpoint_decoder(dev)) { 1970 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev); 1971 1972 rc = decoder_populate_targets(cxlsd, port, target_map); 1973 if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) { 1974 dev_err(&port->dev, 1975 "Failed to populate active decoder targets\n"); 1976 return rc; 1977 } 1978 } 1979 1980 rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id); 1981 if (rc) 1982 return rc; 1983 1984 return device_add(dev); 1985} 1986EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL); 1987 1988/** 1989 * cxl_decoder_add - Add a decoder with targets 1990 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc() 1991 * @target_map: A list of downstream ports that this decoder can direct memory 1992 * traffic to. These numbers should correspond with the port number 1993 * in the PCIe Link Capabilities structure. 1994 * 1995 * This is the unlocked variant of cxl_decoder_add_locked(). 1996 * See cxl_decoder_add_locked(). 1997 * 1998 * Context: Process context. Takes and releases the device lock of the port that 1999 * owns the @cxld. 2000 */ 2001int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map) 2002{ 2003 struct cxl_port *port; 2004 int rc; 2005 2006 if (WARN_ON_ONCE(!cxld)) 2007 return -EINVAL; 2008 2009 if (WARN_ON_ONCE(IS_ERR(cxld))) 2010 return PTR_ERR(cxld); 2011 2012 port = to_cxl_port(cxld->dev.parent); 2013 2014 device_lock(&port->dev); 2015 rc = cxl_decoder_add_locked(cxld, target_map); 2016 device_unlock(&port->dev); 2017 2018 return rc; 2019} 2020EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL); 2021 2022static void cxld_unregister(void *dev) 2023{ 2024 struct cxl_endpoint_decoder *cxled; 2025 2026 if (is_endpoint_decoder(dev)) { 2027 cxled = to_cxl_endpoint_decoder(dev); 2028 cxl_decoder_kill_region(cxled); 2029 } 2030 2031 device_unregister(dev); 2032} 2033 2034int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld) 2035{ 2036 return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev); 2037} 2038EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL); 2039 2040/** 2041 * __cxl_driver_register - register a driver for the cxl bus 2042 * @cxl_drv: cxl driver structure to attach 2043 * @owner: owning module/driver 2044 * @modname: KBUILD_MODNAME for parent driver 2045 */ 2046int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner, 2047 const char *modname) 2048{ 2049 if (!cxl_drv->probe) { 2050 pr_debug("%s ->probe() must be specified\n", modname); 2051 return -EINVAL; 2052 } 2053 2054 if (!cxl_drv->name) { 2055 pr_debug("%s ->name must be specified\n", modname); 2056 return -EINVAL; 2057 } 2058 2059 if (!cxl_drv->id) { 2060 pr_debug("%s ->id must be specified\n", modname); 2061 return -EINVAL; 2062 } 2063 2064 cxl_drv->drv.bus = &cxl_bus_type; 2065 cxl_drv->drv.owner = owner; 2066 cxl_drv->drv.mod_name = modname; 2067 cxl_drv->drv.name = cxl_drv->name; 2068 2069 return driver_register(&cxl_drv->drv); 2070} 2071EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, CXL); 2072 2073void cxl_driver_unregister(struct cxl_driver *cxl_drv) 2074{ 2075 driver_unregister(&cxl_drv->drv); 2076} 2077EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, CXL); 2078 2079static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env) 2080{ 2081 return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT, 2082 cxl_device_id(dev)); 2083} 2084 2085static int cxl_bus_match(struct device *dev, struct device_driver *drv) 2086{ 2087 return cxl_device_id(dev) == to_cxl_drv(drv)->id; 2088} 2089 2090static int cxl_bus_probe(struct device *dev) 2091{ 2092 int rc; 2093 2094 rc = to_cxl_drv(dev->driver)->probe(dev); 2095 dev_dbg(dev, "probe: %d\n", rc); 2096 return rc; 2097} 2098 2099static void cxl_bus_remove(struct device *dev) 2100{ 2101 struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver); 2102 2103 if (cxl_drv->remove) 2104 cxl_drv->remove(dev); 2105} 2106 2107static struct workqueue_struct *cxl_bus_wq; 2108 2109static void cxl_bus_rescan_queue(struct work_struct *w) 2110{ 2111 int rc = bus_rescan_devices(&cxl_bus_type); 2112 2113 pr_debug("CXL bus rescan result: %d\n", rc); 2114} 2115 2116void cxl_bus_rescan(void) 2117{ 2118 static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue); 2119 2120 queue_work(cxl_bus_wq, &rescan_work); 2121} 2122EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, CXL); 2123 2124void cxl_bus_drain(void) 2125{ 2126 drain_workqueue(cxl_bus_wq); 2127} 2128EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, CXL); 2129 2130bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd) 2131{ 2132 return queue_work(cxl_bus_wq, &cxlmd->detach_work); 2133} 2134EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL); 2135 2136static void add_latency(struct access_coordinate *c, long latency) 2137{ 2138 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { 2139 c[i].write_latency += latency; 2140 c[i].read_latency += latency; 2141 } 2142} 2143 2144static bool coordinates_valid(struct access_coordinate *c) 2145{ 2146 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { 2147 if (c[i].read_bandwidth && c[i].write_bandwidth && 2148 c[i].read_latency && c[i].write_latency) 2149 continue; 2150 return false; 2151 } 2152 2153 return true; 2154} 2155 2156static void set_min_bandwidth(struct access_coordinate *c, unsigned int bw) 2157{ 2158 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { 2159 c[i].write_bandwidth = min(c[i].write_bandwidth, bw); 2160 c[i].read_bandwidth = min(c[i].read_bandwidth, bw); 2161 } 2162} 2163 2164static void set_access_coordinates(struct access_coordinate *out, 2165 struct access_coordinate *in) 2166{ 2167 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) 2168 out[i] = in[i]; 2169} 2170 2171static bool parent_port_is_cxl_root(struct cxl_port *port) 2172{ 2173 return is_cxl_root(to_cxl_port(port->dev.parent)); 2174} 2175 2176/** 2177 * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports 2178 * of CXL path 2179 * @port: endpoint cxl_port 2180 * @coord: output performance data 2181 * 2182 * Return: errno on failure, 0 on success. 2183 */ 2184int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, 2185 struct access_coordinate *coord) 2186{ 2187 struct access_coordinate c[] = { 2188 { 2189 .read_bandwidth = UINT_MAX, 2190 .write_bandwidth = UINT_MAX, 2191 }, 2192 { 2193 .read_bandwidth = UINT_MAX, 2194 .write_bandwidth = UINT_MAX, 2195 }, 2196 }; 2197 struct cxl_port *iter = port; 2198 struct cxl_dport *dport; 2199 struct pci_dev *pdev; 2200 unsigned int bw; 2201 bool is_cxl_root; 2202 2203 if (!is_cxl_endpoint(port)) 2204 return -EINVAL; 2205 2206 /* 2207 * Exit the loop when the parent port of the current iter port is cxl 2208 * root. The iterative loop starts at the endpoint and gathers the 2209 * latency of the CXL link from the current device/port to the connected 2210 * downstream port each iteration. 2211 */ 2212 do { 2213 dport = iter->parent_dport; 2214 iter = to_cxl_port(iter->dev.parent); 2215 is_cxl_root = parent_port_is_cxl_root(iter); 2216 2217 /* 2218 * There's no valid access_coordinate for a root port since RPs do not 2219 * have CDAT and therefore needs to be skipped. 2220 */ 2221 if (!is_cxl_root) { 2222 if (!coordinates_valid(dport->coord)) 2223 return -EINVAL; 2224 cxl_coordinates_combine(c, c, dport->coord); 2225 } 2226 add_latency(c, dport->link_latency); 2227 } while (!is_cxl_root); 2228 2229 dport = iter->parent_dport; 2230 /* Retrieve HB coords */ 2231 if (!coordinates_valid(dport->coord)) 2232 return -EINVAL; 2233 cxl_coordinates_combine(c, c, dport->coord); 2234 2235 /* Get the calculated PCI paths bandwidth */ 2236 pdev = to_pci_dev(port->uport_dev->parent); 2237 bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL); 2238 if (bw == 0) 2239 return -ENXIO; 2240 bw /= BITS_PER_BYTE; 2241 2242 set_min_bandwidth(c, bw); 2243 set_access_coordinates(coord, c); 2244 2245 return 0; 2246} 2247EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, CXL); 2248 2249/* for user tooling to ensure port disable work has completed */ 2250static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count) 2251{ 2252 if (sysfs_streq(buf, "1")) { 2253 flush_workqueue(cxl_bus_wq); 2254 return count; 2255 } 2256 2257 return -EINVAL; 2258} 2259 2260static BUS_ATTR_WO(flush); 2261 2262static struct attribute *cxl_bus_attributes[] = { 2263 &bus_attr_flush.attr, 2264 NULL, 2265}; 2266 2267static struct attribute_group cxl_bus_attribute_group = { 2268 .attrs = cxl_bus_attributes, 2269}; 2270 2271static const struct attribute_group *cxl_bus_attribute_groups[] = { 2272 &cxl_bus_attribute_group, 2273 NULL, 2274}; 2275 2276struct bus_type cxl_bus_type = { 2277 .name = "cxl", 2278 .uevent = cxl_bus_uevent, 2279 .match = cxl_bus_match, 2280 .probe = cxl_bus_probe, 2281 .remove = cxl_bus_remove, 2282 .bus_groups = cxl_bus_attribute_groups, 2283}; 2284EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL); 2285 2286static struct dentry *cxl_debugfs; 2287 2288struct dentry *cxl_debugfs_create_dir(const char *dir) 2289{ 2290 return debugfs_create_dir(dir, cxl_debugfs); 2291} 2292EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, CXL); 2293 2294static __init int cxl_core_init(void) 2295{ 2296 int rc; 2297 2298 cxl_debugfs = debugfs_create_dir("cxl", NULL); 2299 2300 if (einj_cxl_is_initialized()) 2301 debugfs_create_file("einj_types", 0400, cxl_debugfs, NULL, 2302 &einj_cxl_available_error_type_fops); 2303 2304 cxl_mbox_init(); 2305 2306 rc = cxl_memdev_init(); 2307 if (rc) 2308 return rc; 2309 2310 cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0); 2311 if (!cxl_bus_wq) { 2312 rc = -ENOMEM; 2313 goto err_wq; 2314 } 2315 2316 rc = bus_register(&cxl_bus_type); 2317 if (rc) 2318 goto err_bus; 2319 2320 rc = cxl_region_init(); 2321 if (rc) 2322 goto err_region; 2323 2324 return 0; 2325 2326err_region: 2327 bus_unregister(&cxl_bus_type); 2328err_bus: 2329 destroy_workqueue(cxl_bus_wq); 2330err_wq: 2331 cxl_memdev_exit(); 2332 return rc; 2333} 2334 2335static void cxl_core_exit(void) 2336{ 2337 cxl_region_exit(); 2338 bus_unregister(&cxl_bus_type); 2339 destroy_workqueue(cxl_bus_wq); 2340 cxl_memdev_exit(); 2341 debugfs_remove_recursive(cxl_debugfs); 2342} 2343 2344subsys_initcall(cxl_core_init); 2345module_exit(cxl_core_exit); 2346MODULE_LICENSE("GPL v2"); 2347MODULE_IMPORT_NS(CXL); 2348