1// SPDX-License-Identifier: GPL-2.0 OR MIT 2/* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24#include <linux/types.h> 25#include <linux/kernel.h> 26#include <linux/pci.h> 27#include <linux/errno.h> 28#include <linux/acpi.h> 29#include <linux/hash.h> 30#include <linux/cpufreq.h> 31#include <linux/log2.h> 32#include <linux/dmi.h> 33#include <linux/atomic.h> 34 35#include "kfd_priv.h" 36#include "kfd_crat.h" 37#include "kfd_topology.h" 38#include "kfd_device_queue_manager.h" 39#include "kfd_svm.h" 40#include "kfd_debug.h" 41#include "amdgpu_amdkfd.h" 42#include "amdgpu_ras.h" 43#include "amdgpu.h" 44 45/* topology_device_list - Master list of all topology devices */ 46static struct list_head topology_device_list; 47static struct kfd_system_properties sys_props; 48 49static DECLARE_RWSEM(topology_lock); 50static uint32_t topology_crat_proximity_domain; 51 52struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock( 53 uint32_t proximity_domain) 54{ 55 struct kfd_topology_device *top_dev; 56 struct kfd_topology_device *device = NULL; 57 58 list_for_each_entry(top_dev, &topology_device_list, list) 59 if (top_dev->proximity_domain == proximity_domain) { 60 device = top_dev; 61 break; 62 } 63 64 return device; 65} 66 67struct kfd_topology_device *kfd_topology_device_by_proximity_domain( 68 uint32_t proximity_domain) 69{ 70 struct kfd_topology_device *device = NULL; 71 72 down_read(&topology_lock); 73 74 device = kfd_topology_device_by_proximity_domain_no_lock( 75 proximity_domain); 76 up_read(&topology_lock); 77 78 return device; 79} 80 81struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id) 82{ 83 struct kfd_topology_device *top_dev = NULL; 84 struct kfd_topology_device *ret = NULL; 85 86 down_read(&topology_lock); 87 88 list_for_each_entry(top_dev, &topology_device_list, list) 89 if (top_dev->gpu_id == gpu_id) { 90 ret = top_dev; 91 break; 92 } 93 94 up_read(&topology_lock); 95 96 return ret; 97} 98 99struct kfd_node *kfd_device_by_id(uint32_t gpu_id) 100{ 101 struct kfd_topology_device *top_dev; 102 103 top_dev = kfd_topology_device_by_id(gpu_id); 104 if (!top_dev) 105 return NULL; 106 107 return top_dev->gpu; 108} 109 110struct kfd_node *kfd_device_by_pci_dev(const struct pci_dev *pdev) 111{ 112 struct kfd_topology_device *top_dev; 113 struct kfd_node *device = NULL; 114 115 down_read(&topology_lock); 116 117 list_for_each_entry(top_dev, &topology_device_list, list) 118 if (top_dev->gpu && top_dev->gpu->adev->pdev == pdev) { 119 device = top_dev->gpu; 120 break; 121 } 122 123 up_read(&topology_lock); 124 125 return device; 126} 127 128/* Called with write topology_lock acquired */ 129static void kfd_release_topology_device(struct kfd_topology_device *dev) 130{ 131 struct kfd_mem_properties *mem; 132 struct kfd_cache_properties *cache; 133 struct kfd_iolink_properties *iolink; 134 struct kfd_iolink_properties *p2plink; 135 struct kfd_perf_properties *perf; 136 137 list_del(&dev->list); 138 139 while (dev->mem_props.next != &dev->mem_props) { 140 mem = container_of(dev->mem_props.next, 141 struct kfd_mem_properties, list); 142 list_del(&mem->list); 143 kfree(mem); 144 } 145 146 while (dev->cache_props.next != &dev->cache_props) { 147 cache = container_of(dev->cache_props.next, 148 struct kfd_cache_properties, list); 149 list_del(&cache->list); 150 kfree(cache); 151 } 152 153 while (dev->io_link_props.next != &dev->io_link_props) { 154 iolink = container_of(dev->io_link_props.next, 155 struct kfd_iolink_properties, list); 156 list_del(&iolink->list); 157 kfree(iolink); 158 } 159 160 while (dev->p2p_link_props.next != &dev->p2p_link_props) { 161 p2plink = container_of(dev->p2p_link_props.next, 162 struct kfd_iolink_properties, list); 163 list_del(&p2plink->list); 164 kfree(p2plink); 165 } 166 167 while (dev->perf_props.next != &dev->perf_props) { 168 perf = container_of(dev->perf_props.next, 169 struct kfd_perf_properties, list); 170 list_del(&perf->list); 171 kfree(perf); 172 } 173 174 kfree(dev); 175} 176 177void kfd_release_topology_device_list(struct list_head *device_list) 178{ 179 struct kfd_topology_device *dev; 180 181 while (!list_empty(device_list)) { 182 dev = list_first_entry(device_list, 183 struct kfd_topology_device, list); 184 kfd_release_topology_device(dev); 185 } 186} 187 188static void kfd_release_live_view(void) 189{ 190 kfd_release_topology_device_list(&topology_device_list); 191 memset(&sys_props, 0, sizeof(sys_props)); 192} 193 194struct kfd_topology_device *kfd_create_topology_device( 195 struct list_head *device_list) 196{ 197 struct kfd_topology_device *dev; 198 199 dev = kfd_alloc_struct(dev); 200 if (!dev) { 201 pr_err("No memory to allocate a topology device"); 202 return NULL; 203 } 204 205 INIT_LIST_HEAD(&dev->mem_props); 206 INIT_LIST_HEAD(&dev->cache_props); 207 INIT_LIST_HEAD(&dev->io_link_props); 208 INIT_LIST_HEAD(&dev->p2p_link_props); 209 INIT_LIST_HEAD(&dev->perf_props); 210 211 list_add_tail(&dev->list, device_list); 212 213 return dev; 214} 215 216 217#define sysfs_show_gen_prop(buffer, offs, fmt, ...) \ 218 (offs += snprintf(buffer+offs, PAGE_SIZE-offs, \ 219 fmt, __VA_ARGS__)) 220#define sysfs_show_32bit_prop(buffer, offs, name, value) \ 221 sysfs_show_gen_prop(buffer, offs, "%s %u\n", name, value) 222#define sysfs_show_64bit_prop(buffer, offs, name, value) \ 223 sysfs_show_gen_prop(buffer, offs, "%s %llu\n", name, value) 224#define sysfs_show_32bit_val(buffer, offs, value) \ 225 sysfs_show_gen_prop(buffer, offs, "%u\n", value) 226#define sysfs_show_str_val(buffer, offs, value) \ 227 sysfs_show_gen_prop(buffer, offs, "%s\n", value) 228 229static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr, 230 char *buffer) 231{ 232 int offs = 0; 233 234 /* Making sure that the buffer is an empty string */ 235 buffer[0] = 0; 236 237 if (attr == &sys_props.attr_genid) { 238 sysfs_show_32bit_val(buffer, offs, 239 sys_props.generation_count); 240 } else if (attr == &sys_props.attr_props) { 241 sysfs_show_64bit_prop(buffer, offs, "platform_oem", 242 sys_props.platform_oem); 243 sysfs_show_64bit_prop(buffer, offs, "platform_id", 244 sys_props.platform_id); 245 sysfs_show_64bit_prop(buffer, offs, "platform_rev", 246 sys_props.platform_rev); 247 } else { 248 offs = -EINVAL; 249 } 250 251 return offs; 252} 253 254static void kfd_topology_kobj_release(struct kobject *kobj) 255{ 256 kfree(kobj); 257} 258 259static const struct sysfs_ops sysprops_ops = { 260 .show = sysprops_show, 261}; 262 263static const struct kobj_type sysprops_type = { 264 .release = kfd_topology_kobj_release, 265 .sysfs_ops = &sysprops_ops, 266}; 267 268static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr, 269 char *buffer) 270{ 271 int offs = 0; 272 struct kfd_iolink_properties *iolink; 273 274 /* Making sure that the buffer is an empty string */ 275 buffer[0] = 0; 276 277 iolink = container_of(attr, struct kfd_iolink_properties, attr); 278 if (iolink->gpu && kfd_devcgroup_check_permission(iolink->gpu)) 279 return -EPERM; 280 sysfs_show_32bit_prop(buffer, offs, "type", iolink->iolink_type); 281 sysfs_show_32bit_prop(buffer, offs, "version_major", iolink->ver_maj); 282 sysfs_show_32bit_prop(buffer, offs, "version_minor", iolink->ver_min); 283 sysfs_show_32bit_prop(buffer, offs, "node_from", iolink->node_from); 284 sysfs_show_32bit_prop(buffer, offs, "node_to", iolink->node_to); 285 sysfs_show_32bit_prop(buffer, offs, "weight", iolink->weight); 286 sysfs_show_32bit_prop(buffer, offs, "min_latency", iolink->min_latency); 287 sysfs_show_32bit_prop(buffer, offs, "max_latency", iolink->max_latency); 288 sysfs_show_32bit_prop(buffer, offs, "min_bandwidth", 289 iolink->min_bandwidth); 290 sysfs_show_32bit_prop(buffer, offs, "max_bandwidth", 291 iolink->max_bandwidth); 292 sysfs_show_32bit_prop(buffer, offs, "recommended_transfer_size", 293 iolink->rec_transfer_size); 294 sysfs_show_32bit_prop(buffer, offs, "flags", iolink->flags); 295 296 return offs; 297} 298 299static const struct sysfs_ops iolink_ops = { 300 .show = iolink_show, 301}; 302 303static const struct kobj_type iolink_type = { 304 .release = kfd_topology_kobj_release, 305 .sysfs_ops = &iolink_ops, 306}; 307 308static ssize_t mem_show(struct kobject *kobj, struct attribute *attr, 309 char *buffer) 310{ 311 int offs = 0; 312 struct kfd_mem_properties *mem; 313 314 /* Making sure that the buffer is an empty string */ 315 buffer[0] = 0; 316 317 mem = container_of(attr, struct kfd_mem_properties, attr); 318 if (mem->gpu && kfd_devcgroup_check_permission(mem->gpu)) 319 return -EPERM; 320 sysfs_show_32bit_prop(buffer, offs, "heap_type", mem->heap_type); 321 sysfs_show_64bit_prop(buffer, offs, "size_in_bytes", 322 mem->size_in_bytes); 323 sysfs_show_32bit_prop(buffer, offs, "flags", mem->flags); 324 sysfs_show_32bit_prop(buffer, offs, "width", mem->width); 325 sysfs_show_32bit_prop(buffer, offs, "mem_clk_max", 326 mem->mem_clk_max); 327 328 return offs; 329} 330 331static const struct sysfs_ops mem_ops = { 332 .show = mem_show, 333}; 334 335static const struct kobj_type mem_type = { 336 .release = kfd_topology_kobj_release, 337 .sysfs_ops = &mem_ops, 338}; 339 340static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr, 341 char *buffer) 342{ 343 int offs = 0; 344 uint32_t i, j; 345 struct kfd_cache_properties *cache; 346 347 /* Making sure that the buffer is an empty string */ 348 buffer[0] = 0; 349 cache = container_of(attr, struct kfd_cache_properties, attr); 350 if (cache->gpu && kfd_devcgroup_check_permission(cache->gpu)) 351 return -EPERM; 352 sysfs_show_32bit_prop(buffer, offs, "processor_id_low", 353 cache->processor_id_low); 354 sysfs_show_32bit_prop(buffer, offs, "level", cache->cache_level); 355 sysfs_show_32bit_prop(buffer, offs, "size", cache->cache_size); 356 sysfs_show_32bit_prop(buffer, offs, "cache_line_size", 357 cache->cacheline_size); 358 sysfs_show_32bit_prop(buffer, offs, "cache_lines_per_tag", 359 cache->cachelines_per_tag); 360 sysfs_show_32bit_prop(buffer, offs, "association", cache->cache_assoc); 361 sysfs_show_32bit_prop(buffer, offs, "latency", cache->cache_latency); 362 sysfs_show_32bit_prop(buffer, offs, "type", cache->cache_type); 363 364 offs += snprintf(buffer+offs, PAGE_SIZE-offs, "sibling_map "); 365 for (i = 0; i < cache->sibling_map_size; i++) 366 for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++) 367 /* Check each bit */ 368 offs += snprintf(buffer+offs, PAGE_SIZE-offs, "%d,", 369 (cache->sibling_map[i] >> j) & 1); 370 371 /* Replace the last "," with end of line */ 372 buffer[offs-1] = '\n'; 373 return offs; 374} 375 376static const struct sysfs_ops cache_ops = { 377 .show = kfd_cache_show, 378}; 379 380static const struct kobj_type cache_type = { 381 .release = kfd_topology_kobj_release, 382 .sysfs_ops = &cache_ops, 383}; 384 385/****** Sysfs of Performance Counters ******/ 386 387struct kfd_perf_attr { 388 struct kobj_attribute attr; 389 uint32_t data; 390}; 391 392static ssize_t perf_show(struct kobject *kobj, struct kobj_attribute *attrs, 393 char *buf) 394{ 395 int offs = 0; 396 struct kfd_perf_attr *attr; 397 398 buf[0] = 0; 399 attr = container_of(attrs, struct kfd_perf_attr, attr); 400 if (!attr->data) /* invalid data for PMC */ 401 return 0; 402 else 403 return sysfs_show_32bit_val(buf, offs, attr->data); 404} 405 406#define KFD_PERF_DESC(_name, _data) \ 407{ \ 408 .attr = __ATTR(_name, 0444, perf_show, NULL), \ 409 .data = _data, \ 410} 411 412static struct kfd_perf_attr perf_attr_iommu[] = { 413 KFD_PERF_DESC(max_concurrent, 0), 414 KFD_PERF_DESC(num_counters, 0), 415 KFD_PERF_DESC(counter_ids, 0), 416}; 417/****************************************/ 418 419static ssize_t node_show(struct kobject *kobj, struct attribute *attr, 420 char *buffer) 421{ 422 int offs = 0; 423 struct kfd_topology_device *dev; 424 uint32_t log_max_watch_addr; 425 426 /* Making sure that the buffer is an empty string */ 427 buffer[0] = 0; 428 429 if (strcmp(attr->name, "gpu_id") == 0) { 430 dev = container_of(attr, struct kfd_topology_device, 431 attr_gpuid); 432 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 433 return -EPERM; 434 return sysfs_show_32bit_val(buffer, offs, dev->gpu_id); 435 } 436 437 if (strcmp(attr->name, "name") == 0) { 438 dev = container_of(attr, struct kfd_topology_device, 439 attr_name); 440 441 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 442 return -EPERM; 443 return sysfs_show_str_val(buffer, offs, dev->node_props.name); 444 } 445 446 dev = container_of(attr, struct kfd_topology_device, 447 attr_props); 448 if (dev->gpu && kfd_devcgroup_check_permission(dev->gpu)) 449 return -EPERM; 450 sysfs_show_32bit_prop(buffer, offs, "cpu_cores_count", 451 dev->node_props.cpu_cores_count); 452 sysfs_show_32bit_prop(buffer, offs, "simd_count", 453 dev->gpu ? dev->node_props.simd_count : 0); 454 sysfs_show_32bit_prop(buffer, offs, "mem_banks_count", 455 dev->node_props.mem_banks_count); 456 sysfs_show_32bit_prop(buffer, offs, "caches_count", 457 dev->node_props.caches_count); 458 sysfs_show_32bit_prop(buffer, offs, "io_links_count", 459 dev->node_props.io_links_count); 460 sysfs_show_32bit_prop(buffer, offs, "p2p_links_count", 461 dev->node_props.p2p_links_count); 462 sysfs_show_32bit_prop(buffer, offs, "cpu_core_id_base", 463 dev->node_props.cpu_core_id_base); 464 sysfs_show_32bit_prop(buffer, offs, "simd_id_base", 465 dev->node_props.simd_id_base); 466 sysfs_show_32bit_prop(buffer, offs, "max_waves_per_simd", 467 dev->node_props.max_waves_per_simd); 468 sysfs_show_32bit_prop(buffer, offs, "lds_size_in_kb", 469 dev->node_props.lds_size_in_kb); 470 sysfs_show_32bit_prop(buffer, offs, "gds_size_in_kb", 471 dev->node_props.gds_size_in_kb); 472 sysfs_show_32bit_prop(buffer, offs, "num_gws", 473 dev->node_props.num_gws); 474 sysfs_show_32bit_prop(buffer, offs, "wave_front_size", 475 dev->node_props.wave_front_size); 476 sysfs_show_32bit_prop(buffer, offs, "array_count", 477 dev->gpu ? (dev->node_props.array_count * 478 NUM_XCC(dev->gpu->xcc_mask)) : 0); 479 sysfs_show_32bit_prop(buffer, offs, "simd_arrays_per_engine", 480 dev->node_props.simd_arrays_per_engine); 481 sysfs_show_32bit_prop(buffer, offs, "cu_per_simd_array", 482 dev->node_props.cu_per_simd_array); 483 sysfs_show_32bit_prop(buffer, offs, "simd_per_cu", 484 dev->node_props.simd_per_cu); 485 sysfs_show_32bit_prop(buffer, offs, "max_slots_scratch_cu", 486 dev->node_props.max_slots_scratch_cu); 487 sysfs_show_32bit_prop(buffer, offs, "gfx_target_version", 488 dev->node_props.gfx_target_version); 489 sysfs_show_32bit_prop(buffer, offs, "vendor_id", 490 dev->node_props.vendor_id); 491 sysfs_show_32bit_prop(buffer, offs, "device_id", 492 dev->node_props.device_id); 493 sysfs_show_32bit_prop(buffer, offs, "location_id", 494 dev->node_props.location_id); 495 sysfs_show_32bit_prop(buffer, offs, "domain", 496 dev->node_props.domain); 497 sysfs_show_32bit_prop(buffer, offs, "drm_render_minor", 498 dev->node_props.drm_render_minor); 499 sysfs_show_64bit_prop(buffer, offs, "hive_id", 500 dev->node_props.hive_id); 501 sysfs_show_32bit_prop(buffer, offs, "num_sdma_engines", 502 dev->node_props.num_sdma_engines); 503 sysfs_show_32bit_prop(buffer, offs, "num_sdma_xgmi_engines", 504 dev->node_props.num_sdma_xgmi_engines); 505 sysfs_show_32bit_prop(buffer, offs, "num_sdma_queues_per_engine", 506 dev->node_props.num_sdma_queues_per_engine); 507 sysfs_show_32bit_prop(buffer, offs, "num_cp_queues", 508 dev->node_props.num_cp_queues); 509 510 if (dev->gpu) { 511 log_max_watch_addr = 512 __ilog2_u32(dev->gpu->kfd->device_info.num_of_watch_points); 513 514 if (log_max_watch_addr) { 515 dev->node_props.capability |= 516 HSA_CAP_WATCH_POINTS_SUPPORTED; 517 518 dev->node_props.capability |= 519 ((log_max_watch_addr << 520 HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT) & 521 HSA_CAP_WATCH_POINTS_TOTALBITS_MASK); 522 } 523 524 if (dev->gpu->adev->asic_type == CHIP_TONGA) 525 dev->node_props.capability |= 526 HSA_CAP_AQL_QUEUE_DOUBLE_MAP; 527 528 sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_fcompute", 529 dev->node_props.max_engine_clk_fcompute); 530 531 sysfs_show_64bit_prop(buffer, offs, "local_mem_size", 0ULL); 532 533 sysfs_show_32bit_prop(buffer, offs, "fw_version", 534 dev->gpu->kfd->mec_fw_version); 535 sysfs_show_32bit_prop(buffer, offs, "capability", 536 dev->node_props.capability); 537 sysfs_show_64bit_prop(buffer, offs, "debug_prop", 538 dev->node_props.debug_prop); 539 sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version", 540 dev->gpu->kfd->sdma_fw_version); 541 sysfs_show_64bit_prop(buffer, offs, "unique_id", 542 dev->gpu->adev->unique_id); 543 sysfs_show_32bit_prop(buffer, offs, "num_xcc", 544 NUM_XCC(dev->gpu->xcc_mask)); 545 } 546 547 return sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_ccompute", 548 cpufreq_quick_get_max(0)/1000); 549} 550 551static const struct sysfs_ops node_ops = { 552 .show = node_show, 553}; 554 555static const struct kobj_type node_type = { 556 .release = kfd_topology_kobj_release, 557 .sysfs_ops = &node_ops, 558}; 559 560static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr) 561{ 562 sysfs_remove_file(kobj, attr); 563 kobject_del(kobj); 564 kobject_put(kobj); 565} 566 567static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev) 568{ 569 struct kfd_iolink_properties *p2plink; 570 struct kfd_iolink_properties *iolink; 571 struct kfd_cache_properties *cache; 572 struct kfd_mem_properties *mem; 573 struct kfd_perf_properties *perf; 574 575 if (dev->kobj_iolink) { 576 list_for_each_entry(iolink, &dev->io_link_props, list) 577 if (iolink->kobj) { 578 kfd_remove_sysfs_file(iolink->kobj, 579 &iolink->attr); 580 iolink->kobj = NULL; 581 } 582 kobject_del(dev->kobj_iolink); 583 kobject_put(dev->kobj_iolink); 584 dev->kobj_iolink = NULL; 585 } 586 587 if (dev->kobj_p2plink) { 588 list_for_each_entry(p2plink, &dev->p2p_link_props, list) 589 if (p2plink->kobj) { 590 kfd_remove_sysfs_file(p2plink->kobj, 591 &p2plink->attr); 592 p2plink->kobj = NULL; 593 } 594 kobject_del(dev->kobj_p2plink); 595 kobject_put(dev->kobj_p2plink); 596 dev->kobj_p2plink = NULL; 597 } 598 599 if (dev->kobj_cache) { 600 list_for_each_entry(cache, &dev->cache_props, list) 601 if (cache->kobj) { 602 kfd_remove_sysfs_file(cache->kobj, 603 &cache->attr); 604 cache->kobj = NULL; 605 } 606 kobject_del(dev->kobj_cache); 607 kobject_put(dev->kobj_cache); 608 dev->kobj_cache = NULL; 609 } 610 611 if (dev->kobj_mem) { 612 list_for_each_entry(mem, &dev->mem_props, list) 613 if (mem->kobj) { 614 kfd_remove_sysfs_file(mem->kobj, &mem->attr); 615 mem->kobj = NULL; 616 } 617 kobject_del(dev->kobj_mem); 618 kobject_put(dev->kobj_mem); 619 dev->kobj_mem = NULL; 620 } 621 622 if (dev->kobj_perf) { 623 list_for_each_entry(perf, &dev->perf_props, list) { 624 kfree(perf->attr_group); 625 perf->attr_group = NULL; 626 } 627 kobject_del(dev->kobj_perf); 628 kobject_put(dev->kobj_perf); 629 dev->kobj_perf = NULL; 630 } 631 632 if (dev->kobj_node) { 633 sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid); 634 sysfs_remove_file(dev->kobj_node, &dev->attr_name); 635 sysfs_remove_file(dev->kobj_node, &dev->attr_props); 636 kobject_del(dev->kobj_node); 637 kobject_put(dev->kobj_node); 638 dev->kobj_node = NULL; 639 } 640} 641 642static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev, 643 uint32_t id) 644{ 645 struct kfd_iolink_properties *p2plink; 646 struct kfd_iolink_properties *iolink; 647 struct kfd_cache_properties *cache; 648 struct kfd_mem_properties *mem; 649 struct kfd_perf_properties *perf; 650 int ret; 651 uint32_t i, num_attrs; 652 struct attribute **attrs; 653 654 if (WARN_ON(dev->kobj_node)) 655 return -EEXIST; 656 657 /* 658 * Creating the sysfs folders 659 */ 660 dev->kobj_node = kfd_alloc_struct(dev->kobj_node); 661 if (!dev->kobj_node) 662 return -ENOMEM; 663 664 ret = kobject_init_and_add(dev->kobj_node, &node_type, 665 sys_props.kobj_nodes, "%d", id); 666 if (ret < 0) { 667 kobject_put(dev->kobj_node); 668 return ret; 669 } 670 671 dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node); 672 if (!dev->kobj_mem) 673 return -ENOMEM; 674 675 dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node); 676 if (!dev->kobj_cache) 677 return -ENOMEM; 678 679 dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node); 680 if (!dev->kobj_iolink) 681 return -ENOMEM; 682 683 dev->kobj_p2plink = kobject_create_and_add("p2p_links", dev->kobj_node); 684 if (!dev->kobj_p2plink) 685 return -ENOMEM; 686 687 dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node); 688 if (!dev->kobj_perf) 689 return -ENOMEM; 690 691 /* 692 * Creating sysfs files for node properties 693 */ 694 dev->attr_gpuid.name = "gpu_id"; 695 dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE; 696 sysfs_attr_init(&dev->attr_gpuid); 697 dev->attr_name.name = "name"; 698 dev->attr_name.mode = KFD_SYSFS_FILE_MODE; 699 sysfs_attr_init(&dev->attr_name); 700 dev->attr_props.name = "properties"; 701 dev->attr_props.mode = KFD_SYSFS_FILE_MODE; 702 sysfs_attr_init(&dev->attr_props); 703 ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid); 704 if (ret < 0) 705 return ret; 706 ret = sysfs_create_file(dev->kobj_node, &dev->attr_name); 707 if (ret < 0) 708 return ret; 709 ret = sysfs_create_file(dev->kobj_node, &dev->attr_props); 710 if (ret < 0) 711 return ret; 712 713 i = 0; 714 list_for_each_entry(mem, &dev->mem_props, list) { 715 mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 716 if (!mem->kobj) 717 return -ENOMEM; 718 ret = kobject_init_and_add(mem->kobj, &mem_type, 719 dev->kobj_mem, "%d", i); 720 if (ret < 0) { 721 kobject_put(mem->kobj); 722 return ret; 723 } 724 725 mem->attr.name = "properties"; 726 mem->attr.mode = KFD_SYSFS_FILE_MODE; 727 sysfs_attr_init(&mem->attr); 728 ret = sysfs_create_file(mem->kobj, &mem->attr); 729 if (ret < 0) 730 return ret; 731 i++; 732 } 733 734 i = 0; 735 list_for_each_entry(cache, &dev->cache_props, list) { 736 cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 737 if (!cache->kobj) 738 return -ENOMEM; 739 ret = kobject_init_and_add(cache->kobj, &cache_type, 740 dev->kobj_cache, "%d", i); 741 if (ret < 0) { 742 kobject_put(cache->kobj); 743 return ret; 744 } 745 746 cache->attr.name = "properties"; 747 cache->attr.mode = KFD_SYSFS_FILE_MODE; 748 sysfs_attr_init(&cache->attr); 749 ret = sysfs_create_file(cache->kobj, &cache->attr); 750 if (ret < 0) 751 return ret; 752 i++; 753 } 754 755 i = 0; 756 list_for_each_entry(iolink, &dev->io_link_props, list) { 757 iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 758 if (!iolink->kobj) 759 return -ENOMEM; 760 ret = kobject_init_and_add(iolink->kobj, &iolink_type, 761 dev->kobj_iolink, "%d", i); 762 if (ret < 0) { 763 kobject_put(iolink->kobj); 764 return ret; 765 } 766 767 iolink->attr.name = "properties"; 768 iolink->attr.mode = KFD_SYSFS_FILE_MODE; 769 sysfs_attr_init(&iolink->attr); 770 ret = sysfs_create_file(iolink->kobj, &iolink->attr); 771 if (ret < 0) 772 return ret; 773 i++; 774 } 775 776 i = 0; 777 list_for_each_entry(p2plink, &dev->p2p_link_props, list) { 778 p2plink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 779 if (!p2plink->kobj) 780 return -ENOMEM; 781 ret = kobject_init_and_add(p2plink->kobj, &iolink_type, 782 dev->kobj_p2plink, "%d", i); 783 if (ret < 0) { 784 kobject_put(p2plink->kobj); 785 return ret; 786 } 787 788 p2plink->attr.name = "properties"; 789 p2plink->attr.mode = KFD_SYSFS_FILE_MODE; 790 sysfs_attr_init(&p2plink->attr); 791 ret = sysfs_create_file(p2plink->kobj, &p2plink->attr); 792 if (ret < 0) 793 return ret; 794 i++; 795 } 796 797 /* All hardware blocks have the same number of attributes. */ 798 num_attrs = ARRAY_SIZE(perf_attr_iommu); 799 list_for_each_entry(perf, &dev->perf_props, list) { 800 perf->attr_group = kzalloc(sizeof(struct kfd_perf_attr) 801 * num_attrs + sizeof(struct attribute_group), 802 GFP_KERNEL); 803 if (!perf->attr_group) 804 return -ENOMEM; 805 806 attrs = (struct attribute **)(perf->attr_group + 1); 807 if (!strcmp(perf->block_name, "iommu")) { 808 /* Information of IOMMU's num_counters and counter_ids is shown 809 * under /sys/bus/event_source/devices/amd_iommu. We don't 810 * duplicate here. 811 */ 812 perf_attr_iommu[0].data = perf->max_concurrent; 813 for (i = 0; i < num_attrs; i++) 814 attrs[i] = &perf_attr_iommu[i].attr.attr; 815 } 816 perf->attr_group->name = perf->block_name; 817 perf->attr_group->attrs = attrs; 818 ret = sysfs_create_group(dev->kobj_perf, perf->attr_group); 819 if (ret < 0) 820 return ret; 821 } 822 823 return 0; 824} 825 826/* Called with write topology lock acquired */ 827static int kfd_build_sysfs_node_tree(void) 828{ 829 struct kfd_topology_device *dev; 830 int ret; 831 uint32_t i = 0; 832 833 list_for_each_entry(dev, &topology_device_list, list) { 834 ret = kfd_build_sysfs_node_entry(dev, i); 835 if (ret < 0) 836 return ret; 837 i++; 838 } 839 840 return 0; 841} 842 843/* Called with write topology lock acquired */ 844static void kfd_remove_sysfs_node_tree(void) 845{ 846 struct kfd_topology_device *dev; 847 848 list_for_each_entry(dev, &topology_device_list, list) 849 kfd_remove_sysfs_node_entry(dev); 850} 851 852static int kfd_topology_update_sysfs(void) 853{ 854 int ret; 855 856 if (!sys_props.kobj_topology) { 857 sys_props.kobj_topology = 858 kfd_alloc_struct(sys_props.kobj_topology); 859 if (!sys_props.kobj_topology) 860 return -ENOMEM; 861 862 ret = kobject_init_and_add(sys_props.kobj_topology, 863 &sysprops_type, &kfd_device->kobj, 864 "topology"); 865 if (ret < 0) { 866 kobject_put(sys_props.kobj_topology); 867 return ret; 868 } 869 870 sys_props.kobj_nodes = kobject_create_and_add("nodes", 871 sys_props.kobj_topology); 872 if (!sys_props.kobj_nodes) 873 return -ENOMEM; 874 875 sys_props.attr_genid.name = "generation_id"; 876 sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE; 877 sysfs_attr_init(&sys_props.attr_genid); 878 ret = sysfs_create_file(sys_props.kobj_topology, 879 &sys_props.attr_genid); 880 if (ret < 0) 881 return ret; 882 883 sys_props.attr_props.name = "system_properties"; 884 sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE; 885 sysfs_attr_init(&sys_props.attr_props); 886 ret = sysfs_create_file(sys_props.kobj_topology, 887 &sys_props.attr_props); 888 if (ret < 0) 889 return ret; 890 } 891 892 kfd_remove_sysfs_node_tree(); 893 894 return kfd_build_sysfs_node_tree(); 895} 896 897static void kfd_topology_release_sysfs(void) 898{ 899 kfd_remove_sysfs_node_tree(); 900 if (sys_props.kobj_topology) { 901 sysfs_remove_file(sys_props.kobj_topology, 902 &sys_props.attr_genid); 903 sysfs_remove_file(sys_props.kobj_topology, 904 &sys_props.attr_props); 905 if (sys_props.kobj_nodes) { 906 kobject_del(sys_props.kobj_nodes); 907 kobject_put(sys_props.kobj_nodes); 908 sys_props.kobj_nodes = NULL; 909 } 910 kobject_del(sys_props.kobj_topology); 911 kobject_put(sys_props.kobj_topology); 912 sys_props.kobj_topology = NULL; 913 } 914} 915 916/* Called with write topology_lock acquired */ 917static void kfd_topology_update_device_list(struct list_head *temp_list, 918 struct list_head *master_list) 919{ 920 while (!list_empty(temp_list)) { 921 list_move_tail(temp_list->next, master_list); 922 sys_props.num_devices++; 923 } 924} 925 926static void kfd_debug_print_topology(void) 927{ 928 struct kfd_topology_device *dev; 929 930 down_read(&topology_lock); 931 932 dev = list_last_entry(&topology_device_list, 933 struct kfd_topology_device, list); 934 if (dev) { 935 if (dev->node_props.cpu_cores_count && 936 dev->node_props.simd_count) { 937 pr_info("Topology: Add APU node [0x%0x:0x%0x]\n", 938 dev->node_props.device_id, 939 dev->node_props.vendor_id); 940 } else if (dev->node_props.cpu_cores_count) 941 pr_info("Topology: Add CPU node\n"); 942 else if (dev->node_props.simd_count) 943 pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n", 944 dev->node_props.device_id, 945 dev->node_props.vendor_id); 946 } 947 up_read(&topology_lock); 948} 949 950/* Helper function for intializing platform_xx members of 951 * kfd_system_properties. Uses OEM info from the last CPU/APU node. 952 */ 953static void kfd_update_system_properties(void) 954{ 955 struct kfd_topology_device *dev; 956 957 down_read(&topology_lock); 958 dev = list_last_entry(&topology_device_list, 959 struct kfd_topology_device, list); 960 if (dev) { 961 sys_props.platform_id = 962 (*((uint64_t *)dev->oem_id)) & CRAT_OEMID_64BIT_MASK; 963 sys_props.platform_oem = *((uint64_t *)dev->oem_table_id); 964 sys_props.platform_rev = dev->oem_revision; 965 } 966 up_read(&topology_lock); 967} 968 969static void find_system_memory(const struct dmi_header *dm, 970 void *private) 971{ 972 struct kfd_mem_properties *mem; 973 u16 mem_width, mem_clock; 974 struct kfd_topology_device *kdev = 975 (struct kfd_topology_device *)private; 976 const u8 *dmi_data = (const u8 *)(dm + 1); 977 978 if (dm->type == DMI_ENTRY_MEM_DEVICE && dm->length >= 0x15) { 979 mem_width = (u16)(*(const u16 *)(dmi_data + 0x6)); 980 mem_clock = (u16)(*(const u16 *)(dmi_data + 0x11)); 981 list_for_each_entry(mem, &kdev->mem_props, list) { 982 if (mem_width != 0xFFFF && mem_width != 0) 983 mem->width = mem_width; 984 if (mem_clock != 0) 985 mem->mem_clk_max = mem_clock; 986 } 987 } 988} 989 990/* kfd_add_non_crat_information - Add information that is not currently 991 * defined in CRAT but is necessary for KFD topology 992 * @dev - topology device to which addition info is added 993 */ 994static void kfd_add_non_crat_information(struct kfd_topology_device *kdev) 995{ 996 /* Check if CPU only node. */ 997 if (!kdev->gpu) { 998 /* Add system memory information */ 999 dmi_walk(find_system_memory, kdev); 1000 } 1001 /* TODO: For GPU node, rearrange code from kfd_topology_add_device */ 1002} 1003 1004int kfd_topology_init(void) 1005{ 1006 void *crat_image = NULL; 1007 size_t image_size = 0; 1008 int ret; 1009 struct list_head temp_topology_device_list; 1010 int cpu_only_node = 0; 1011 struct kfd_topology_device *kdev; 1012 int proximity_domain; 1013 1014 /* topology_device_list - Master list of all topology devices 1015 * temp_topology_device_list - temporary list created while parsing CRAT 1016 * or VCRAT. Once parsing is complete the contents of list is moved to 1017 * topology_device_list 1018 */ 1019 1020 /* Initialize the head for the both the lists */ 1021 INIT_LIST_HEAD(&topology_device_list); 1022 INIT_LIST_HEAD(&temp_topology_device_list); 1023 init_rwsem(&topology_lock); 1024 1025 memset(&sys_props, 0, sizeof(sys_props)); 1026 1027 /* Proximity domains in ACPI CRAT tables start counting at 1028 * 0. The same should be true for virtual CRAT tables created 1029 * at this stage. GPUs added later in kfd_topology_add_device 1030 * use a counter. 1031 */ 1032 proximity_domain = 0; 1033 1034 ret = kfd_create_crat_image_virtual(&crat_image, &image_size, 1035 COMPUTE_UNIT_CPU, NULL, 1036 proximity_domain); 1037 cpu_only_node = 1; 1038 if (ret) { 1039 pr_err("Error creating VCRAT table for CPU\n"); 1040 return ret; 1041 } 1042 1043 ret = kfd_parse_crat_table(crat_image, 1044 &temp_topology_device_list, 1045 proximity_domain); 1046 if (ret) { 1047 pr_err("Error parsing VCRAT table for CPU\n"); 1048 goto err; 1049 } 1050 1051 kdev = list_first_entry(&temp_topology_device_list, 1052 struct kfd_topology_device, list); 1053 1054 down_write(&topology_lock); 1055 kfd_topology_update_device_list(&temp_topology_device_list, 1056 &topology_device_list); 1057 topology_crat_proximity_domain = sys_props.num_devices-1; 1058 ret = kfd_topology_update_sysfs(); 1059 up_write(&topology_lock); 1060 1061 if (!ret) { 1062 sys_props.generation_count++; 1063 kfd_update_system_properties(); 1064 kfd_debug_print_topology(); 1065 } else 1066 pr_err("Failed to update topology in sysfs ret=%d\n", ret); 1067 1068 /* For nodes with GPU, this information gets added 1069 * when GPU is detected (kfd_topology_add_device). 1070 */ 1071 if (cpu_only_node) { 1072 /* Add additional information to CPU only node created above */ 1073 down_write(&topology_lock); 1074 kdev = list_first_entry(&topology_device_list, 1075 struct kfd_topology_device, list); 1076 up_write(&topology_lock); 1077 kfd_add_non_crat_information(kdev); 1078 } 1079 1080err: 1081 kfd_destroy_crat_image(crat_image); 1082 return ret; 1083} 1084 1085void kfd_topology_shutdown(void) 1086{ 1087 down_write(&topology_lock); 1088 kfd_topology_release_sysfs(); 1089 kfd_release_live_view(); 1090 up_write(&topology_lock); 1091} 1092 1093static uint32_t kfd_generate_gpu_id(struct kfd_node *gpu) 1094{ 1095 uint32_t hashout; 1096 uint32_t buf[8]; 1097 uint64_t local_mem_size; 1098 int i; 1099 1100 if (!gpu) 1101 return 0; 1102 1103 local_mem_size = gpu->local_mem_info.local_mem_size_private + 1104 gpu->local_mem_info.local_mem_size_public; 1105 buf[0] = gpu->adev->pdev->devfn; 1106 buf[1] = gpu->adev->pdev->subsystem_vendor | 1107 (gpu->adev->pdev->subsystem_device << 16); 1108 buf[2] = pci_domain_nr(gpu->adev->pdev->bus); 1109 buf[3] = gpu->adev->pdev->device; 1110 buf[4] = gpu->adev->pdev->bus->number; 1111 buf[5] = lower_32_bits(local_mem_size); 1112 buf[6] = upper_32_bits(local_mem_size); 1113 buf[7] = (ffs(gpu->xcc_mask) - 1) | (NUM_XCC(gpu->xcc_mask) << 16); 1114 1115 for (i = 0, hashout = 0; i < 8; i++) 1116 hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH); 1117 1118 return hashout; 1119} 1120/* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If 1121 * the GPU device is not already present in the topology device 1122 * list then return NULL. This means a new topology device has to 1123 * be created for this GPU. 1124 */ 1125static struct kfd_topology_device *kfd_assign_gpu(struct kfd_node *gpu) 1126{ 1127 struct kfd_topology_device *dev; 1128 struct kfd_topology_device *out_dev = NULL; 1129 struct kfd_mem_properties *mem; 1130 struct kfd_cache_properties *cache; 1131 struct kfd_iolink_properties *iolink; 1132 struct kfd_iolink_properties *p2plink; 1133 1134 list_for_each_entry(dev, &topology_device_list, list) { 1135 /* Discrete GPUs need their own topology device list 1136 * entries. Don't assign them to CPU/APU nodes. 1137 */ 1138 if (dev->node_props.cpu_cores_count) 1139 continue; 1140 1141 if (!dev->gpu && (dev->node_props.simd_count > 0)) { 1142 dev->gpu = gpu; 1143 out_dev = dev; 1144 1145 list_for_each_entry(mem, &dev->mem_props, list) 1146 mem->gpu = dev->gpu; 1147 list_for_each_entry(cache, &dev->cache_props, list) 1148 cache->gpu = dev->gpu; 1149 list_for_each_entry(iolink, &dev->io_link_props, list) 1150 iolink->gpu = dev->gpu; 1151 list_for_each_entry(p2plink, &dev->p2p_link_props, list) 1152 p2plink->gpu = dev->gpu; 1153 break; 1154 } 1155 } 1156 return out_dev; 1157} 1158 1159static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival) 1160{ 1161 /* 1162 * TODO: Generate an event for thunk about the arrival/removal 1163 * of the GPU 1164 */ 1165} 1166 1167/* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info, 1168 * patch this after CRAT parsing. 1169 */ 1170static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev) 1171{ 1172 struct kfd_mem_properties *mem; 1173 struct kfd_local_mem_info local_mem_info; 1174 1175 if (!dev) 1176 return; 1177 1178 /* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with 1179 * single bank of VRAM local memory. 1180 * for dGPUs - VCRAT reports only one bank of Local Memory 1181 * for APUs - If CRAT from ACPI reports more than one bank, then 1182 * all the banks will report the same mem_clk_max information 1183 */ 1184 amdgpu_amdkfd_get_local_mem_info(dev->gpu->adev, &local_mem_info, 1185 dev->gpu->xcp); 1186 1187 list_for_each_entry(mem, &dev->mem_props, list) 1188 mem->mem_clk_max = local_mem_info.mem_clk_max; 1189} 1190 1191static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev, 1192 struct kfd_topology_device *target_gpu_dev, 1193 struct kfd_iolink_properties *link) 1194{ 1195 /* xgmi always supports atomics between links. */ 1196 if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI) 1197 return; 1198 1199 /* check pcie support to set cpu(dev) flags for target_gpu_dev link. */ 1200 if (target_gpu_dev) { 1201 uint32_t cap; 1202 1203 pcie_capability_read_dword(target_gpu_dev->gpu->adev->pdev, 1204 PCI_EXP_DEVCAP2, &cap); 1205 1206 if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 1207 PCI_EXP_DEVCAP2_ATOMIC_COMP64))) 1208 link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT | 1209 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT; 1210 /* set gpu (dev) flags. */ 1211 } else { 1212 if (!dev->gpu->kfd->pci_atomic_requested || 1213 dev->gpu->adev->asic_type == CHIP_HAWAII) 1214 link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT | 1215 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT; 1216 } 1217} 1218 1219static void kfd_set_iolink_non_coherent(struct kfd_topology_device *to_dev, 1220 struct kfd_iolink_properties *outbound_link, 1221 struct kfd_iolink_properties *inbound_link) 1222{ 1223 /* CPU -> GPU with PCIe */ 1224 if (!to_dev->gpu && 1225 inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS) 1226 inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1227 1228 if (to_dev->gpu) { 1229 /* GPU <-> GPU with PCIe and 1230 * Vega20 with XGMI 1231 */ 1232 if (inbound_link->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS || 1233 (inbound_link->iolink_type == CRAT_IOLINK_TYPE_XGMI && 1234 KFD_GC_VERSION(to_dev->gpu) == IP_VERSION(9, 4, 0))) { 1235 outbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1236 inbound_link->flags |= CRAT_IOLINK_FLAGS_NON_COHERENT; 1237 } 1238 } 1239} 1240 1241static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev) 1242{ 1243 struct kfd_iolink_properties *link, *inbound_link; 1244 struct kfd_topology_device *peer_dev; 1245 1246 if (!dev || !dev->gpu) 1247 return; 1248 1249 /* GPU only creates direct links so apply flags setting to all */ 1250 list_for_each_entry(link, &dev->io_link_props, list) { 1251 link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1252 kfd_set_iolink_no_atomics(dev, NULL, link); 1253 peer_dev = kfd_topology_device_by_proximity_domain( 1254 link->node_to); 1255 1256 if (!peer_dev) 1257 continue; 1258 1259 /* Include the CPU peer in GPU hive if connected over xGMI. */ 1260 if (!peer_dev->gpu && 1261 link->iolink_type == CRAT_IOLINK_TYPE_XGMI) { 1262 /* 1263 * If the GPU is not part of a GPU hive, use its pci 1264 * device location as the hive ID to bind with the CPU. 1265 */ 1266 if (!dev->node_props.hive_id) 1267 dev->node_props.hive_id = pci_dev_id(dev->gpu->adev->pdev); 1268 peer_dev->node_props.hive_id = dev->node_props.hive_id; 1269 } 1270 1271 list_for_each_entry(inbound_link, &peer_dev->io_link_props, 1272 list) { 1273 if (inbound_link->node_to != link->node_from) 1274 continue; 1275 1276 inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1277 kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link); 1278 kfd_set_iolink_non_coherent(peer_dev, link, inbound_link); 1279 } 1280 } 1281 1282 /* Create indirect links so apply flags setting to all */ 1283 list_for_each_entry(link, &dev->p2p_link_props, list) { 1284 link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1285 kfd_set_iolink_no_atomics(dev, NULL, link); 1286 peer_dev = kfd_topology_device_by_proximity_domain( 1287 link->node_to); 1288 1289 if (!peer_dev) 1290 continue; 1291 1292 list_for_each_entry(inbound_link, &peer_dev->p2p_link_props, 1293 list) { 1294 if (inbound_link->node_to != link->node_from) 1295 continue; 1296 1297 inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED; 1298 kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link); 1299 kfd_set_iolink_non_coherent(peer_dev, link, inbound_link); 1300 } 1301 } 1302} 1303 1304static int kfd_build_p2p_node_entry(struct kfd_topology_device *dev, 1305 struct kfd_iolink_properties *p2plink) 1306{ 1307 int ret; 1308 1309 p2plink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); 1310 if (!p2plink->kobj) 1311 return -ENOMEM; 1312 1313 ret = kobject_init_and_add(p2plink->kobj, &iolink_type, 1314 dev->kobj_p2plink, "%d", dev->node_props.p2p_links_count - 1); 1315 if (ret < 0) { 1316 kobject_put(p2plink->kobj); 1317 return ret; 1318 } 1319 1320 p2plink->attr.name = "properties"; 1321 p2plink->attr.mode = KFD_SYSFS_FILE_MODE; 1322 sysfs_attr_init(&p2plink->attr); 1323 ret = sysfs_create_file(p2plink->kobj, &p2plink->attr); 1324 if (ret < 0) 1325 return ret; 1326 1327 return 0; 1328} 1329 1330static int kfd_create_indirect_link_prop(struct kfd_topology_device *kdev, int gpu_node) 1331{ 1332 struct kfd_iolink_properties *gpu_link, *tmp_link, *cpu_link; 1333 struct kfd_iolink_properties *props = NULL, *props2 = NULL; 1334 struct kfd_topology_device *cpu_dev; 1335 int ret = 0; 1336 int i, num_cpu; 1337 1338 num_cpu = 0; 1339 list_for_each_entry(cpu_dev, &topology_device_list, list) { 1340 if (cpu_dev->gpu) 1341 break; 1342 num_cpu++; 1343 } 1344 1345 if (list_empty(&kdev->io_link_props)) 1346 return -ENODATA; 1347 1348 gpu_link = list_first_entry(&kdev->io_link_props, 1349 struct kfd_iolink_properties, list); 1350 1351 for (i = 0; i < num_cpu; i++) { 1352 /* CPU <--> GPU */ 1353 if (gpu_link->node_to == i) 1354 continue; 1355 1356 /* find CPU <--> CPU links */ 1357 cpu_link = NULL; 1358 cpu_dev = kfd_topology_device_by_proximity_domain(i); 1359 if (cpu_dev) { 1360 list_for_each_entry(tmp_link, 1361 &cpu_dev->io_link_props, list) { 1362 if (tmp_link->node_to == gpu_link->node_to) { 1363 cpu_link = tmp_link; 1364 break; 1365 } 1366 } 1367 } 1368 1369 if (!cpu_link) 1370 return -ENOMEM; 1371 1372 /* CPU <--> CPU <--> GPU, GPU node*/ 1373 props = kfd_alloc_struct(props); 1374 if (!props) 1375 return -ENOMEM; 1376 1377 memcpy(props, gpu_link, sizeof(struct kfd_iolink_properties)); 1378 props->weight = gpu_link->weight + cpu_link->weight; 1379 props->min_latency = gpu_link->min_latency + cpu_link->min_latency; 1380 props->max_latency = gpu_link->max_latency + cpu_link->max_latency; 1381 props->min_bandwidth = min(gpu_link->min_bandwidth, cpu_link->min_bandwidth); 1382 props->max_bandwidth = min(gpu_link->max_bandwidth, cpu_link->max_bandwidth); 1383 1384 props->node_from = gpu_node; 1385 props->node_to = i; 1386 kdev->node_props.p2p_links_count++; 1387 list_add_tail(&props->list, &kdev->p2p_link_props); 1388 ret = kfd_build_p2p_node_entry(kdev, props); 1389 if (ret < 0) 1390 return ret; 1391 1392 /* for small Bar, no CPU --> GPU in-direct links */ 1393 if (kfd_dev_is_large_bar(kdev->gpu)) { 1394 /* CPU <--> CPU <--> GPU, CPU node*/ 1395 props2 = kfd_alloc_struct(props2); 1396 if (!props2) 1397 return -ENOMEM; 1398 1399 memcpy(props2, props, sizeof(struct kfd_iolink_properties)); 1400 props2->node_from = i; 1401 props2->node_to = gpu_node; 1402 props2->kobj = NULL; 1403 cpu_dev->node_props.p2p_links_count++; 1404 list_add_tail(&props2->list, &cpu_dev->p2p_link_props); 1405 ret = kfd_build_p2p_node_entry(cpu_dev, props2); 1406 if (ret < 0) 1407 return ret; 1408 } 1409 } 1410 return ret; 1411} 1412 1413#if defined(CONFIG_HSA_AMD_P2P) 1414static int kfd_add_peer_prop(struct kfd_topology_device *kdev, 1415 struct kfd_topology_device *peer, int from, int to) 1416{ 1417 struct kfd_iolink_properties *props = NULL; 1418 struct kfd_iolink_properties *iolink1, *iolink2, *iolink3; 1419 struct kfd_topology_device *cpu_dev; 1420 int ret = 0; 1421 1422 if (!amdgpu_device_is_peer_accessible( 1423 kdev->gpu->adev, 1424 peer->gpu->adev)) 1425 return ret; 1426 1427 if (list_empty(&kdev->io_link_props)) 1428 return -ENODATA; 1429 1430 iolink1 = list_first_entry(&kdev->io_link_props, 1431 struct kfd_iolink_properties, list); 1432 1433 if (list_empty(&peer->io_link_props)) 1434 return -ENODATA; 1435 1436 iolink2 = list_first_entry(&peer->io_link_props, 1437 struct kfd_iolink_properties, list); 1438 1439 props = kfd_alloc_struct(props); 1440 if (!props) 1441 return -ENOMEM; 1442 1443 memcpy(props, iolink1, sizeof(struct kfd_iolink_properties)); 1444 1445 props->weight = iolink1->weight + iolink2->weight; 1446 props->min_latency = iolink1->min_latency + iolink2->min_latency; 1447 props->max_latency = iolink1->max_latency + iolink2->max_latency; 1448 props->min_bandwidth = min(iolink1->min_bandwidth, iolink2->min_bandwidth); 1449 props->max_bandwidth = min(iolink2->max_bandwidth, iolink2->max_bandwidth); 1450 1451 if (iolink1->node_to != iolink2->node_to) { 1452 /* CPU->CPU link*/ 1453 cpu_dev = kfd_topology_device_by_proximity_domain(iolink1->node_to); 1454 if (cpu_dev) { 1455 list_for_each_entry(iolink3, &cpu_dev->io_link_props, list) { 1456 if (iolink3->node_to != iolink2->node_to) 1457 continue; 1458 1459 props->weight += iolink3->weight; 1460 props->min_latency += iolink3->min_latency; 1461 props->max_latency += iolink3->max_latency; 1462 props->min_bandwidth = min(props->min_bandwidth, 1463 iolink3->min_bandwidth); 1464 props->max_bandwidth = min(props->max_bandwidth, 1465 iolink3->max_bandwidth); 1466 break; 1467 } 1468 } else { 1469 WARN(1, "CPU node not found"); 1470 } 1471 } 1472 1473 props->node_from = from; 1474 props->node_to = to; 1475 peer->node_props.p2p_links_count++; 1476 list_add_tail(&props->list, &peer->p2p_link_props); 1477 ret = kfd_build_p2p_node_entry(peer, props); 1478 1479 return ret; 1480} 1481#endif 1482 1483static int kfd_dev_create_p2p_links(void) 1484{ 1485 struct kfd_topology_device *dev; 1486 struct kfd_topology_device *new_dev; 1487#if defined(CONFIG_HSA_AMD_P2P) 1488 uint32_t i; 1489#endif 1490 uint32_t k; 1491 int ret = 0; 1492 1493 k = 0; 1494 list_for_each_entry(dev, &topology_device_list, list) 1495 k++; 1496 if (k < 2) 1497 return 0; 1498 1499 new_dev = list_last_entry(&topology_device_list, struct kfd_topology_device, list); 1500 if (WARN_ON(!new_dev->gpu)) 1501 return 0; 1502 1503 k--; 1504 1505 /* create in-direct links */ 1506 ret = kfd_create_indirect_link_prop(new_dev, k); 1507 if (ret < 0) 1508 goto out; 1509 1510 /* create p2p links */ 1511#if defined(CONFIG_HSA_AMD_P2P) 1512 i = 0; 1513 list_for_each_entry(dev, &topology_device_list, list) { 1514 if (dev == new_dev) 1515 break; 1516 if (!dev->gpu || !dev->gpu->adev || 1517 (dev->gpu->kfd->hive_id && 1518 dev->gpu->kfd->hive_id == new_dev->gpu->kfd->hive_id)) 1519 goto next; 1520 1521 /* check if node(s) is/are peer accessible in one direction or bi-direction */ 1522 ret = kfd_add_peer_prop(new_dev, dev, i, k); 1523 if (ret < 0) 1524 goto out; 1525 1526 ret = kfd_add_peer_prop(dev, new_dev, k, i); 1527 if (ret < 0) 1528 goto out; 1529next: 1530 i++; 1531 } 1532#endif 1533 1534out: 1535 return ret; 1536} 1537 1538/* Helper function. See kfd_fill_gpu_cache_info for parameter description */ 1539static int fill_in_l1_pcache(struct kfd_cache_properties **props_ext, 1540 struct kfd_gpu_cache_info *pcache_info, 1541 int cu_bitmask, 1542 int cache_type, unsigned int cu_processor_id, 1543 int cu_block) 1544{ 1545 unsigned int cu_sibling_map_mask; 1546 int first_active_cu; 1547 struct kfd_cache_properties *pcache = NULL; 1548 1549 cu_sibling_map_mask = cu_bitmask; 1550 cu_sibling_map_mask >>= cu_block; 1551 cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1552 first_active_cu = ffs(cu_sibling_map_mask); 1553 1554 /* CU could be inactive. In case of shared cache find the first active 1555 * CU. and incase of non-shared cache check if the CU is inactive. If 1556 * inactive active skip it 1557 */ 1558 if (first_active_cu) { 1559 pcache = kfd_alloc_struct(pcache); 1560 if (!pcache) 1561 return -ENOMEM; 1562 1563 memset(pcache, 0, sizeof(struct kfd_cache_properties)); 1564 pcache->processor_id_low = cu_processor_id + (first_active_cu - 1); 1565 pcache->cache_level = pcache_info[cache_type].cache_level; 1566 pcache->cache_size = pcache_info[cache_type].cache_size; 1567 pcache->cacheline_size = pcache_info[cache_type].cache_line_size; 1568 1569 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE) 1570 pcache->cache_type |= HSA_CACHE_TYPE_DATA; 1571 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE) 1572 pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; 1573 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE) 1574 pcache->cache_type |= HSA_CACHE_TYPE_CPU; 1575 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE) 1576 pcache->cache_type |= HSA_CACHE_TYPE_HSACU; 1577 1578 /* Sibling map is w.r.t processor_id_low, so shift out 1579 * inactive CU 1580 */ 1581 cu_sibling_map_mask = 1582 cu_sibling_map_mask >> (first_active_cu - 1); 1583 1584 pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF); 1585 pcache->sibling_map[1] = 1586 (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); 1587 pcache->sibling_map[2] = 1588 (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); 1589 pcache->sibling_map[3] = 1590 (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); 1591 1592 pcache->sibling_map_size = 4; 1593 *props_ext = pcache; 1594 1595 return 0; 1596 } 1597 return 1; 1598} 1599 1600/* Helper function. See kfd_fill_gpu_cache_info for parameter description */ 1601static int fill_in_l2_l3_pcache(struct kfd_cache_properties **props_ext, 1602 struct kfd_gpu_cache_info *pcache_info, 1603 struct amdgpu_cu_info *cu_info, 1604 struct amdgpu_gfx_config *gfx_info, 1605 int cache_type, unsigned int cu_processor_id, 1606 struct kfd_node *knode) 1607{ 1608 unsigned int cu_sibling_map_mask; 1609 int first_active_cu; 1610 int i, j, k, xcc, start, end; 1611 int num_xcc = NUM_XCC(knode->xcc_mask); 1612 struct kfd_cache_properties *pcache = NULL; 1613 enum amdgpu_memory_partition mode; 1614 struct amdgpu_device *adev = knode->adev; 1615 1616 start = ffs(knode->xcc_mask) - 1; 1617 end = start + num_xcc; 1618 cu_sibling_map_mask = cu_info->bitmap[start][0][0]; 1619 cu_sibling_map_mask &= 1620 ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1621 first_active_cu = ffs(cu_sibling_map_mask); 1622 1623 /* CU could be inactive. In case of shared cache find the first active 1624 * CU. and incase of non-shared cache check if the CU is inactive. If 1625 * inactive active skip it 1626 */ 1627 if (first_active_cu) { 1628 pcache = kfd_alloc_struct(pcache); 1629 if (!pcache) 1630 return -ENOMEM; 1631 1632 memset(pcache, 0, sizeof(struct kfd_cache_properties)); 1633 pcache->processor_id_low = cu_processor_id 1634 + (first_active_cu - 1); 1635 pcache->cache_level = pcache_info[cache_type].cache_level; 1636 pcache->cacheline_size = pcache_info[cache_type].cache_line_size; 1637 1638 if (KFD_GC_VERSION(knode) == IP_VERSION(9, 4, 3)) 1639 mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 1640 else 1641 mode = UNKNOWN_MEMORY_PARTITION_MODE; 1642 1643 pcache->cache_size = pcache_info[cache_type].cache_size; 1644 /* Partition mode only affects L3 cache size */ 1645 if (mode && pcache->cache_level == 3) 1646 pcache->cache_size /= mode; 1647 1648 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE) 1649 pcache->cache_type |= HSA_CACHE_TYPE_DATA; 1650 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE) 1651 pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; 1652 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE) 1653 pcache->cache_type |= HSA_CACHE_TYPE_CPU; 1654 if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE) 1655 pcache->cache_type |= HSA_CACHE_TYPE_HSACU; 1656 1657 /* Sibling map is w.r.t processor_id_low, so shift out 1658 * inactive CU 1659 */ 1660 cu_sibling_map_mask = cu_sibling_map_mask >> (first_active_cu - 1); 1661 k = 0; 1662 1663 for (xcc = start; xcc < end; xcc++) { 1664 for (i = 0; i < gfx_info->max_shader_engines; i++) { 1665 for (j = 0; j < gfx_info->max_sh_per_se; j++) { 1666 pcache->sibling_map[k] = (uint8_t)(cu_sibling_map_mask & 0xFF); 1667 pcache->sibling_map[k+1] = (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); 1668 pcache->sibling_map[k+2] = (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); 1669 pcache->sibling_map[k+3] = (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); 1670 k += 4; 1671 1672 cu_sibling_map_mask = cu_info->bitmap[xcc][i % 4][j + i / 4]; 1673 cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1); 1674 } 1675 } 1676 } 1677 pcache->sibling_map_size = k; 1678 *props_ext = pcache; 1679 return 0; 1680 } 1681 return 1; 1682} 1683 1684#define KFD_MAX_CACHE_TYPES 6 1685 1686/* kfd_fill_cache_non_crat_info - Fill GPU cache info using kfd_gpu_cache_info 1687 * tables 1688 */ 1689static void kfd_fill_cache_non_crat_info(struct kfd_topology_device *dev, struct kfd_node *kdev) 1690{ 1691 struct kfd_gpu_cache_info *pcache_info = NULL; 1692 int i, j, k, xcc, start, end; 1693 int ct = 0; 1694 unsigned int cu_processor_id; 1695 int ret; 1696 unsigned int num_cu_shared; 1697 struct amdgpu_cu_info *cu_info = &kdev->adev->gfx.cu_info; 1698 struct amdgpu_gfx_config *gfx_info = &kdev->adev->gfx.config; 1699 int gpu_processor_id; 1700 struct kfd_cache_properties *props_ext; 1701 int num_of_entries = 0; 1702 int num_of_cache_types = 0; 1703 struct kfd_gpu_cache_info cache_info[KFD_MAX_CACHE_TYPES]; 1704 1705 1706 gpu_processor_id = dev->node_props.simd_id_base; 1707 1708 memset(cache_info, 0, sizeof(cache_info)); 1709 pcache_info = cache_info; 1710 num_of_cache_types = kfd_get_gpu_cache_info(kdev, &pcache_info); 1711 if (!num_of_cache_types) { 1712 pr_warn("no cache info found\n"); 1713 return; 1714 } 1715 1716 /* For each type of cache listed in the kfd_gpu_cache_info table, 1717 * go through all available Compute Units. 1718 * The [i,j,k] loop will 1719 * if kfd_gpu_cache_info.num_cu_shared = 1 1720 * will parse through all available CU 1721 * If (kfd_gpu_cache_info.num_cu_shared != 1) 1722 * then it will consider only one CU from 1723 * the shared unit 1724 */ 1725 start = ffs(kdev->xcc_mask) - 1; 1726 end = start + NUM_XCC(kdev->xcc_mask); 1727 1728 for (ct = 0; ct < num_of_cache_types; ct++) { 1729 cu_processor_id = gpu_processor_id; 1730 if (pcache_info[ct].cache_level == 1) { 1731 for (xcc = start; xcc < end; xcc++) { 1732 for (i = 0; i < gfx_info->max_shader_engines; i++) { 1733 for (j = 0; j < gfx_info->max_sh_per_se; j++) { 1734 for (k = 0; k < gfx_info->max_cu_per_sh; k += pcache_info[ct].num_cu_shared) { 1735 1736 ret = fill_in_l1_pcache(&props_ext, pcache_info, 1737 cu_info->bitmap[xcc][i % 4][j + i / 4], ct, 1738 cu_processor_id, k); 1739 1740 if (ret < 0) 1741 break; 1742 1743 if (!ret) { 1744 num_of_entries++; 1745 list_add_tail(&props_ext->list, &dev->cache_props); 1746 } 1747 1748 /* Move to next CU block */ 1749 num_cu_shared = ((k + pcache_info[ct].num_cu_shared) <= 1750 gfx_info->max_cu_per_sh) ? 1751 pcache_info[ct].num_cu_shared : 1752 (gfx_info->max_cu_per_sh - k); 1753 cu_processor_id += num_cu_shared; 1754 } 1755 } 1756 } 1757 } 1758 } else { 1759 ret = fill_in_l2_l3_pcache(&props_ext, pcache_info, 1760 cu_info, gfx_info, ct, cu_processor_id, kdev); 1761 1762 if (ret < 0) 1763 break; 1764 1765 if (!ret) { 1766 num_of_entries++; 1767 list_add_tail(&props_ext->list, &dev->cache_props); 1768 } 1769 } 1770 } 1771 dev->node_props.caches_count += num_of_entries; 1772 pr_debug("Added [%d] GPU cache entries\n", num_of_entries); 1773} 1774 1775static int kfd_topology_add_device_locked(struct kfd_node *gpu, uint32_t gpu_id, 1776 struct kfd_topology_device **dev) 1777{ 1778 int proximity_domain = ++topology_crat_proximity_domain; 1779 struct list_head temp_topology_device_list; 1780 void *crat_image = NULL; 1781 size_t image_size = 0; 1782 int res; 1783 1784 res = kfd_create_crat_image_virtual(&crat_image, &image_size, 1785 COMPUTE_UNIT_GPU, gpu, 1786 proximity_domain); 1787 if (res) { 1788 pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n", 1789 gpu_id); 1790 topology_crat_proximity_domain--; 1791 goto err; 1792 } 1793 1794 INIT_LIST_HEAD(&temp_topology_device_list); 1795 1796 res = kfd_parse_crat_table(crat_image, 1797 &temp_topology_device_list, 1798 proximity_domain); 1799 if (res) { 1800 pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n", 1801 gpu_id); 1802 topology_crat_proximity_domain--; 1803 goto err; 1804 } 1805 1806 kfd_topology_update_device_list(&temp_topology_device_list, 1807 &topology_device_list); 1808 1809 *dev = kfd_assign_gpu(gpu); 1810 if (WARN_ON(!*dev)) { 1811 res = -ENODEV; 1812 goto err; 1813 } 1814 1815 /* Fill the cache affinity information here for the GPUs 1816 * using VCRAT 1817 */ 1818 kfd_fill_cache_non_crat_info(*dev, gpu); 1819 1820 /* Update the SYSFS tree, since we added another topology 1821 * device 1822 */ 1823 res = kfd_topology_update_sysfs(); 1824 if (!res) 1825 sys_props.generation_count++; 1826 else 1827 pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n", 1828 gpu_id, res); 1829 1830err: 1831 kfd_destroy_crat_image(crat_image); 1832 return res; 1833} 1834 1835static void kfd_topology_set_dbg_firmware_support(struct kfd_topology_device *dev) 1836{ 1837 bool firmware_supported = true; 1838 1839 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(11, 0, 0) && 1840 KFD_GC_VERSION(dev->gpu) < IP_VERSION(12, 0, 0)) { 1841 uint32_t mes_api_rev = (dev->gpu->adev->mes.sched_version & 1842 AMDGPU_MES_API_VERSION_MASK) >> 1843 AMDGPU_MES_API_VERSION_SHIFT; 1844 uint32_t mes_rev = dev->gpu->adev->mes.sched_version & 1845 AMDGPU_MES_VERSION_MASK; 1846 1847 firmware_supported = (mes_api_rev >= 14) && (mes_rev >= 64); 1848 goto out; 1849 } 1850 1851 /* 1852 * Note: Any unlisted devices here are assumed to support exception handling. 1853 * Add additional checks here as needed. 1854 */ 1855 switch (KFD_GC_VERSION(dev->gpu)) { 1856 case IP_VERSION(9, 0, 1): 1857 firmware_supported = dev->gpu->kfd->mec_fw_version >= 459 + 32768; 1858 break; 1859 case IP_VERSION(9, 1, 0): 1860 case IP_VERSION(9, 2, 1): 1861 case IP_VERSION(9, 2, 2): 1862 case IP_VERSION(9, 3, 0): 1863 case IP_VERSION(9, 4, 0): 1864 firmware_supported = dev->gpu->kfd->mec_fw_version >= 459; 1865 break; 1866 case IP_VERSION(9, 4, 1): 1867 firmware_supported = dev->gpu->kfd->mec_fw_version >= 60; 1868 break; 1869 case IP_VERSION(9, 4, 2): 1870 firmware_supported = dev->gpu->kfd->mec_fw_version >= 51; 1871 break; 1872 case IP_VERSION(10, 1, 10): 1873 case IP_VERSION(10, 1, 2): 1874 case IP_VERSION(10, 1, 1): 1875 firmware_supported = dev->gpu->kfd->mec_fw_version >= 144; 1876 break; 1877 case IP_VERSION(10, 3, 0): 1878 case IP_VERSION(10, 3, 2): 1879 case IP_VERSION(10, 3, 1): 1880 case IP_VERSION(10, 3, 4): 1881 case IP_VERSION(10, 3, 5): 1882 firmware_supported = dev->gpu->kfd->mec_fw_version >= 89; 1883 break; 1884 case IP_VERSION(10, 1, 3): 1885 case IP_VERSION(10, 3, 3): 1886 firmware_supported = false; 1887 break; 1888 default: 1889 break; 1890 } 1891 1892out: 1893 if (firmware_supported) 1894 dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_FIRMWARE_SUPPORTED; 1895} 1896 1897static void kfd_topology_set_capabilities(struct kfd_topology_device *dev) 1898{ 1899 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_2_0 << 1900 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 1901 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 1902 1903 dev->node_props.capability |= HSA_CAP_TRAP_DEBUG_SUPPORT | 1904 HSA_CAP_TRAP_DEBUG_WAVE_LAUNCH_TRAP_OVERRIDE_SUPPORTED | 1905 HSA_CAP_TRAP_DEBUG_WAVE_LAUNCH_MODE_SUPPORTED; 1906 1907 if (kfd_dbg_has_ttmps_always_setup(dev->gpu)) 1908 dev->node_props.debug_prop |= HSA_DBG_DISPATCH_INFO_ALWAYS_VALID; 1909 1910 if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0)) { 1911 if (KFD_GC_VERSION(dev->gpu) == IP_VERSION(9, 4, 3)) 1912 dev->node_props.debug_prop |= 1913 HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX9_4_3 | 1914 HSA_DBG_WATCH_ADDR_MASK_HI_BIT_GFX9_4_3; 1915 else 1916 dev->node_props.debug_prop |= 1917 HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX9 | 1918 HSA_DBG_WATCH_ADDR_MASK_HI_BIT; 1919 1920 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(9, 4, 2)) 1921 dev->node_props.capability |= 1922 HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED; 1923 } else { 1924 dev->node_props.debug_prop |= HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX10 | 1925 HSA_DBG_WATCH_ADDR_MASK_HI_BIT; 1926 1927 if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(11, 0, 0)) 1928 dev->node_props.capability |= 1929 HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED; 1930 } 1931 1932 kfd_topology_set_dbg_firmware_support(dev); 1933} 1934 1935int kfd_topology_add_device(struct kfd_node *gpu) 1936{ 1937 uint32_t gpu_id; 1938 struct kfd_topology_device *dev; 1939 int res = 0; 1940 int i; 1941 const char *asic_name = amdgpu_asic_name[gpu->adev->asic_type]; 1942 struct amdgpu_gfx_config *gfx_info = &gpu->adev->gfx.config; 1943 struct amdgpu_cu_info *cu_info = &gpu->adev->gfx.cu_info; 1944 1945 gpu_id = kfd_generate_gpu_id(gpu); 1946 if (gpu->xcp && !gpu->xcp->ddev) { 1947 dev_warn(gpu->adev->dev, 1948 "Won't add GPU (ID: 0x%x) to topology since it has no drm node assigned.", 1949 gpu_id); 1950 return 0; 1951 } else { 1952 pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id); 1953 } 1954 1955 /* Check to see if this gpu device exists in the topology_device_list. 1956 * If so, assign the gpu to that device, 1957 * else create a Virtual CRAT for this gpu device and then parse that 1958 * CRAT to create a new topology device. Once created assign the gpu to 1959 * that topology device 1960 */ 1961 down_write(&topology_lock); 1962 dev = kfd_assign_gpu(gpu); 1963 if (!dev) 1964 res = kfd_topology_add_device_locked(gpu, gpu_id, &dev); 1965 up_write(&topology_lock); 1966 if (res) 1967 return res; 1968 1969 dev->gpu_id = gpu_id; 1970 gpu->id = gpu_id; 1971 1972 kfd_dev_create_p2p_links(); 1973 1974 /* TODO: Move the following lines to function 1975 * kfd_add_non_crat_information 1976 */ 1977 1978 /* Fill-in additional information that is not available in CRAT but 1979 * needed for the topology 1980 */ 1981 for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1; i++) { 1982 dev->node_props.name[i] = __tolower(asic_name[i]); 1983 if (asic_name[i] == '\0') 1984 break; 1985 } 1986 dev->node_props.name[i] = '\0'; 1987 1988 dev->node_props.simd_arrays_per_engine = 1989 gfx_info->max_sh_per_se; 1990 1991 dev->node_props.gfx_target_version = 1992 gpu->kfd->device_info.gfx_target_version; 1993 dev->node_props.vendor_id = gpu->adev->pdev->vendor; 1994 dev->node_props.device_id = gpu->adev->pdev->device; 1995 dev->node_props.capability |= 1996 ((dev->gpu->adev->rev_id << HSA_CAP_ASIC_REVISION_SHIFT) & 1997 HSA_CAP_ASIC_REVISION_MASK); 1998 1999 dev->node_props.location_id = pci_dev_id(gpu->adev->pdev); 2000 /* On multi-partition nodes, node id = location_id[31:28] */ 2001 if (gpu->kfd->num_nodes > 1) 2002 dev->node_props.location_id |= (dev->gpu->node_id << 28); 2003 2004 dev->node_props.domain = pci_domain_nr(gpu->adev->pdev->bus); 2005 dev->node_props.max_engine_clk_fcompute = 2006 amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->adev); 2007 dev->node_props.max_engine_clk_ccompute = 2008 cpufreq_quick_get_max(0) / 1000; 2009 2010 if (gpu->xcp) 2011 dev->node_props.drm_render_minor = gpu->xcp->ddev->render->index; 2012 else 2013 dev->node_props.drm_render_minor = 2014 gpu->kfd->shared_resources.drm_render_minor; 2015 2016 dev->node_props.hive_id = gpu->kfd->hive_id; 2017 dev->node_props.num_sdma_engines = kfd_get_num_sdma_engines(gpu); 2018 dev->node_props.num_sdma_xgmi_engines = 2019 kfd_get_num_xgmi_sdma_engines(gpu); 2020 dev->node_props.num_sdma_queues_per_engine = 2021 gpu->kfd->device_info.num_sdma_queues_per_engine - 2022 gpu->kfd->device_info.num_reserved_sdma_queues_per_engine; 2023 dev->node_props.num_gws = (dev->gpu->gws && 2024 dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ? 2025 dev->gpu->adev->gds.gws_size : 0; 2026 dev->node_props.num_cp_queues = get_cp_queues_num(dev->gpu->dqm); 2027 2028 kfd_fill_mem_clk_max_info(dev); 2029 kfd_fill_iolink_non_crat_info(dev); 2030 2031 switch (dev->gpu->adev->asic_type) { 2032 case CHIP_KAVERI: 2033 case CHIP_HAWAII: 2034 case CHIP_TONGA: 2035 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 << 2036 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 2037 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 2038 break; 2039 case CHIP_CARRIZO: 2040 case CHIP_FIJI: 2041 case CHIP_POLARIS10: 2042 case CHIP_POLARIS11: 2043 case CHIP_POLARIS12: 2044 case CHIP_VEGAM: 2045 pr_debug("Adding doorbell packet type capability\n"); 2046 dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 << 2047 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) & 2048 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK); 2049 break; 2050 default: 2051 if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(9, 0, 1)) 2052 WARN(1, "Unexpected ASIC family %u", 2053 dev->gpu->adev->asic_type); 2054 else 2055 kfd_topology_set_capabilities(dev); 2056 } 2057 2058 /* 2059 * Overwrite ATS capability according to needs_iommu_device to fix 2060 * potential missing corresponding bit in CRAT of BIOS. 2061 */ 2062 dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT; 2063 2064 /* Fix errors in CZ CRAT. 2065 * simd_count: Carrizo CRAT reports wrong simd_count, probably 2066 * because it doesn't consider masked out CUs 2067 * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd 2068 */ 2069 if (dev->gpu->adev->asic_type == CHIP_CARRIZO) { 2070 dev->node_props.simd_count = 2071 cu_info->simd_per_cu * cu_info->number; 2072 dev->node_props.max_waves_per_simd = 10; 2073 } 2074 2075 /* kfd only concerns sram ecc on GFX and HBM ecc on UMC */ 2076 dev->node_props.capability |= 2077 ((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ? 2078 HSA_CAP_SRAM_EDCSUPPORTED : 0; 2079 dev->node_props.capability |= 2080 ((dev->gpu->adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ? 2081 HSA_CAP_MEM_EDCSUPPORTED : 0; 2082 2083 if (KFD_GC_VERSION(dev->gpu) != IP_VERSION(9, 0, 1)) 2084 dev->node_props.capability |= (dev->gpu->adev->ras_enabled != 0) ? 2085 HSA_CAP_RASEVENTNOTIFY : 0; 2086 2087 if (KFD_IS_SVM_API_SUPPORTED(dev->gpu->adev)) 2088 dev->node_props.capability |= HSA_CAP_SVMAPI_SUPPORTED; 2089 2090 if (dev->gpu->adev->gmc.is_app_apu || 2091 dev->gpu->adev->gmc.xgmi.connected_to_cpu) 2092 dev->node_props.capability |= HSA_CAP_FLAGS_COHERENTHOSTACCESS; 2093 2094 kfd_debug_print_topology(); 2095 2096 kfd_notify_gpu_change(gpu_id, 1); 2097 2098 return 0; 2099} 2100 2101/** 2102 * kfd_topology_update_io_links() - Update IO links after device removal. 2103 * @proximity_domain: Proximity domain value of the dev being removed. 2104 * 2105 * The topology list currently is arranged in increasing order of 2106 * proximity domain. 2107 * 2108 * Two things need to be done when a device is removed: 2109 * 1. All the IO links to this device need to be removed. 2110 * 2. All nodes after the current device node need to move 2111 * up once this device node is removed from the topology 2112 * list. As a result, the proximity domain values for 2113 * all nodes after the node being deleted reduce by 1. 2114 * This would also cause the proximity domain values for 2115 * io links to be updated based on new proximity domain 2116 * values. 2117 * 2118 * Context: The caller must hold write topology_lock. 2119 */ 2120static void kfd_topology_update_io_links(int proximity_domain) 2121{ 2122 struct kfd_topology_device *dev; 2123 struct kfd_iolink_properties *iolink, *p2plink, *tmp; 2124 2125 list_for_each_entry(dev, &topology_device_list, list) { 2126 if (dev->proximity_domain > proximity_domain) 2127 dev->proximity_domain--; 2128 2129 list_for_each_entry_safe(iolink, tmp, &dev->io_link_props, list) { 2130 /* 2131 * If there is an io link to the dev being deleted 2132 * then remove that IO link also. 2133 */ 2134 if (iolink->node_to == proximity_domain) { 2135 list_del(&iolink->list); 2136 dev->node_props.io_links_count--; 2137 } else { 2138 if (iolink->node_from > proximity_domain) 2139 iolink->node_from--; 2140 if (iolink->node_to > proximity_domain) 2141 iolink->node_to--; 2142 } 2143 } 2144 2145 list_for_each_entry_safe(p2plink, tmp, &dev->p2p_link_props, list) { 2146 /* 2147 * If there is a p2p link to the dev being deleted 2148 * then remove that p2p link also. 2149 */ 2150 if (p2plink->node_to == proximity_domain) { 2151 list_del(&p2plink->list); 2152 dev->node_props.p2p_links_count--; 2153 } else { 2154 if (p2plink->node_from > proximity_domain) 2155 p2plink->node_from--; 2156 if (p2plink->node_to > proximity_domain) 2157 p2plink->node_to--; 2158 } 2159 } 2160 } 2161} 2162 2163int kfd_topology_remove_device(struct kfd_node *gpu) 2164{ 2165 struct kfd_topology_device *dev, *tmp; 2166 uint32_t gpu_id; 2167 int res = -ENODEV; 2168 int i = 0; 2169 2170 down_write(&topology_lock); 2171 2172 list_for_each_entry_safe(dev, tmp, &topology_device_list, list) { 2173 if (dev->gpu == gpu) { 2174 gpu_id = dev->gpu_id; 2175 kfd_remove_sysfs_node_entry(dev); 2176 kfd_release_topology_device(dev); 2177 sys_props.num_devices--; 2178 kfd_topology_update_io_links(i); 2179 topology_crat_proximity_domain = sys_props.num_devices-1; 2180 sys_props.generation_count++; 2181 res = 0; 2182 if (kfd_topology_update_sysfs() < 0) 2183 kfd_topology_release_sysfs(); 2184 break; 2185 } 2186 i++; 2187 } 2188 2189 up_write(&topology_lock); 2190 2191 if (!res) 2192 kfd_notify_gpu_change(gpu_id, 0); 2193 2194 return res; 2195} 2196 2197/* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD 2198 * topology. If GPU device is found @idx, then valid kfd_dev pointer is 2199 * returned through @kdev 2200 * Return - 0: On success (@kdev will be NULL for non GPU nodes) 2201 * -1: If end of list 2202 */ 2203int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_node **kdev) 2204{ 2205 2206 struct kfd_topology_device *top_dev; 2207 uint8_t device_idx = 0; 2208 2209 *kdev = NULL; 2210 down_read(&topology_lock); 2211 2212 list_for_each_entry(top_dev, &topology_device_list, list) { 2213 if (device_idx == idx) { 2214 *kdev = top_dev->gpu; 2215 up_read(&topology_lock); 2216 return 0; 2217 } 2218 2219 device_idx++; 2220 } 2221 2222 up_read(&topology_lock); 2223 2224 return -1; 2225 2226} 2227 2228static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask) 2229{ 2230 int first_cpu_of_numa_node; 2231 2232 if (!cpumask || cpumask == cpu_none_mask) 2233 return -1; 2234 first_cpu_of_numa_node = cpumask_first(cpumask); 2235 if (first_cpu_of_numa_node >= nr_cpu_ids) 2236 return -1; 2237#ifdef CONFIG_X86_64 2238 return cpu_data(first_cpu_of_numa_node).topo.apicid; 2239#else 2240 return first_cpu_of_numa_node; 2241#endif 2242} 2243 2244/* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor 2245 * of the given NUMA node (numa_node_id) 2246 * Return -1 on failure 2247 */ 2248int kfd_numa_node_to_apic_id(int numa_node_id) 2249{ 2250 if (numa_node_id == -1) { 2251 pr_warn("Invalid NUMA Node. Use online CPU mask\n"); 2252 return kfd_cpumask_to_apic_id(cpu_online_mask); 2253 } 2254 return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id)); 2255} 2256 2257#if defined(CONFIG_DEBUG_FS) 2258 2259int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data) 2260{ 2261 struct kfd_topology_device *dev; 2262 unsigned int i = 0; 2263 int r = 0; 2264 2265 down_read(&topology_lock); 2266 2267 list_for_each_entry(dev, &topology_device_list, list) { 2268 if (!dev->gpu) { 2269 i++; 2270 continue; 2271 } 2272 2273 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); 2274 r = dqm_debugfs_hqds(m, dev->gpu->dqm); 2275 if (r) 2276 break; 2277 } 2278 2279 up_read(&topology_lock); 2280 2281 return r; 2282} 2283 2284int kfd_debugfs_rls_by_device(struct seq_file *m, void *data) 2285{ 2286 struct kfd_topology_device *dev; 2287 unsigned int i = 0; 2288 int r = 0; 2289 2290 down_read(&topology_lock); 2291 2292 list_for_each_entry(dev, &topology_device_list, list) { 2293 if (!dev->gpu) { 2294 i++; 2295 continue; 2296 } 2297 2298 seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); 2299 r = pm_debugfs_runlist(m, &dev->gpu->dqm->packet_mgr); 2300 if (r) 2301 break; 2302 } 2303 2304 up_read(&topology_lock); 2305 2306 return r; 2307} 2308 2309#endif 2310