1// SPDX-License-Identifier: GPL-2.0 2 3/* 4 * Copyright 2021 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8#include <linux/vmalloc.h> 9#include <uapi/drm/habanalabs_accel.h> 10#include "habanalabs.h" 11 12/** 13 * hl_format_as_binary - helper function, format an integer as binary 14 * using supplied scratch buffer 15 * @buf: the buffer to use 16 * @buf_len: buffer capacity 17 * @n: number to format 18 * 19 * Returns pointer to buffer 20 */ 21char *hl_format_as_binary(char *buf, size_t buf_len, u32 n) 22{ 23 int i; 24 u32 bit; 25 bool leading0 = true; 26 char *wrptr = buf; 27 28 if (buf_len > 0 && buf_len < 3) { 29 *wrptr = '\0'; 30 return buf; 31 } 32 33 wrptr[0] = '0'; 34 wrptr[1] = 'b'; 35 wrptr += 2; 36 /* Remove 3 characters from length for '0b' and '\0' termination */ 37 buf_len -= 3; 38 39 for (i = 0; i < sizeof(n) * BITS_PER_BYTE && buf_len; ++i, n <<= 1) { 40 /* Writing bit calculation in one line would cause a false 41 * positive static code analysis error, so splitting. 42 */ 43 bit = n & (1 << (sizeof(n) * BITS_PER_BYTE - 1)); 44 bit = !!bit; 45 leading0 &= !bit; 46 if (!leading0) { 47 *wrptr = '0' + bit; 48 ++wrptr; 49 } 50 } 51 52 *wrptr = '\0'; 53 54 return buf; 55} 56 57/** 58 * resize_to_fit - helper function, resize buffer to fit given amount of data 59 * @buf: destination buffer double pointer 60 * @size: pointer to the size container 61 * @desired_size: size the buffer must contain 62 * 63 * Returns 0 on success or error code on failure. 64 * On success, the size of buffer is at least desired_size. Buffer is allocated 65 * via vmalloc and must be freed with vfree. 66 */ 67static int resize_to_fit(char **buf, size_t *size, size_t desired_size) 68{ 69 char *resized_buf; 70 size_t new_size; 71 72 if (*size >= desired_size) 73 return 0; 74 75 /* Not enough space to print all, have to resize */ 76 new_size = max_t(size_t, PAGE_SIZE, round_up(desired_size, PAGE_SIZE)); 77 resized_buf = vmalloc(new_size); 78 if (!resized_buf) 79 return -ENOMEM; 80 memcpy(resized_buf, *buf, *size); 81 vfree(*buf); 82 *buf = resized_buf; 83 *size = new_size; 84 85 return 1; 86} 87 88/** 89 * hl_snprintf_resize() - print formatted data to buffer, resize as needed 90 * @buf: buffer double pointer, to be written to and resized, must be either 91 * NULL or allocated with vmalloc. 92 * @size: current size of the buffer 93 * @offset: current offset to write to 94 * @format: format of the data 95 * 96 * This function will write formatted data into the buffer. If buffer is not 97 * large enough, it will be resized using vmalloc. Size may be modified if the 98 * buffer was resized, offset will be advanced by the number of bytes written 99 * not including the terminating character 100 * 101 * Returns 0 on success or error code on failure 102 * 103 * Note that the buffer has to be manually released using vfree. 104 */ 105int hl_snprintf_resize(char **buf, size_t *size, size_t *offset, 106 const char *format, ...) 107{ 108 va_list args; 109 size_t length; 110 int rc; 111 112 if (*buf == NULL && (*size != 0 || *offset != 0)) 113 return -EINVAL; 114 115 va_start(args, format); 116 length = vsnprintf(*buf + *offset, *size - *offset, format, args); 117 va_end(args); 118 119 rc = resize_to_fit(buf, size, *offset + length + 1); 120 if (rc < 0) 121 return rc; 122 else if (rc > 0) { 123 /* Resize was needed, write again */ 124 va_start(args, format); 125 length = vsnprintf(*buf + *offset, *size - *offset, format, 126 args); 127 va_end(args); 128 } 129 130 *offset += length; 131 132 return 0; 133} 134 135/** 136 * hl_sync_engine_to_string - convert engine type enum to string literal 137 * @engine_type: engine type (TPC/MME/DMA) 138 * 139 * Return the resolved string literal 140 */ 141const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type) 142{ 143 switch (engine_type) { 144 case ENGINE_DMA: 145 return "DMA"; 146 case ENGINE_MME: 147 return "MME"; 148 case ENGINE_TPC: 149 return "TPC"; 150 } 151 return "Invalid Engine Type"; 152} 153 154/** 155 * hl_print_resize_sync_engine - helper function, format engine name and ID 156 * using hl_snprintf_resize 157 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 158 * @size: pointer to the size container 159 * @offset: pointer to the offset container 160 * @engine_type: engine type (TPC/MME/DMA) 161 * @engine_id: engine numerical id 162 * 163 * Returns 0 on success or error code on failure 164 */ 165static int hl_print_resize_sync_engine(char **buf, size_t *size, size_t *offset, 166 enum hl_sync_engine_type engine_type, 167 u32 engine_id) 168{ 169 return hl_snprintf_resize(buf, size, offset, "%s%u", 170 hl_sync_engine_to_string(engine_type), engine_id); 171} 172 173/** 174 * hl_state_dump_get_sync_name - transform sync object id to name if available 175 * @hdev: pointer to the device 176 * @sync_id: sync object id 177 * 178 * Returns a name literal or NULL if not resolved. 179 * Note: returning NULL shall not be considered as a failure, as not all 180 * sync objects are named. 181 */ 182const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id) 183{ 184 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 185 struct hl_hw_obj_name_entry *entry; 186 187 hash_for_each_possible(sds->so_id_to_str_tb, entry, 188 node, sync_id) 189 if (sync_id == entry->id) 190 return entry->name; 191 192 return NULL; 193} 194 195/** 196 * hl_state_dump_get_monitor_name - transform monitor object dump to monitor 197 * name if available 198 * @hdev: pointer to the device 199 * @mon: monitor state dump 200 * 201 * Returns a name literal or NULL if not resolved. 202 * Note: returning NULL shall not be considered as a failure, as not all 203 * monitors are named. 204 */ 205const char *hl_state_dump_get_monitor_name(struct hl_device *hdev, 206 struct hl_mon_state_dump *mon) 207{ 208 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 209 struct hl_hw_obj_name_entry *entry; 210 211 hash_for_each_possible(sds->monitor_id_to_str_tb, 212 entry, node, mon->id) 213 if (mon->id == entry->id) 214 return entry->name; 215 216 return NULL; 217} 218 219/** 220 * hl_state_dump_free_sync_to_engine_map - free sync object to engine map 221 * @map: sync object to engine map 222 * 223 * Note: generic free implementation, the allocation is implemented per ASIC. 224 */ 225void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map) 226{ 227 struct hl_sync_to_engine_map_entry *entry; 228 struct hlist_node *tmp_node; 229 int i; 230 231 hash_for_each_safe(map->tb, i, tmp_node, entry, node) { 232 hash_del(&entry->node); 233 kfree(entry); 234 } 235} 236 237/** 238 * hl_state_dump_get_sync_to_engine - transform sync_id to 239 * hl_sync_to_engine_map_entry if available for current id 240 * @map: sync object to engine map 241 * @sync_id: sync object id 242 * 243 * Returns the translation entry if found or NULL if not. 244 * Note, returned NULL shall not be considered as a failure as the map 245 * does not cover all possible, it is a best effort sync ids. 246 */ 247static struct hl_sync_to_engine_map_entry * 248hl_state_dump_get_sync_to_engine(struct hl_sync_to_engine_map *map, u32 sync_id) 249{ 250 struct hl_sync_to_engine_map_entry *entry; 251 252 hash_for_each_possible(map->tb, entry, node, sync_id) 253 if (entry->sync_id == sync_id) 254 return entry; 255 return NULL; 256} 257 258/** 259 * hl_state_dump_read_sync_objects - read sync objects array 260 * @hdev: pointer to the device 261 * @index: sync manager block index starting with E_N 262 * 263 * Returns array of size SP_SYNC_OBJ_AMOUNT on success or NULL on failure 264 */ 265static u32 *hl_state_dump_read_sync_objects(struct hl_device *hdev, u32 index) 266{ 267 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 268 u32 *sync_objects; 269 s64 base_addr; /* Base addr can be negative */ 270 int i; 271 272 base_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] + 273 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index; 274 275 sync_objects = vmalloc(sds->props[SP_SYNC_OBJ_AMOUNT] * sizeof(u32)); 276 if (!sync_objects) 277 return NULL; 278 279 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) 280 sync_objects[i] = RREG32(base_addr + i * sizeof(u32)); 281 282 return sync_objects; 283} 284 285/** 286 * hl_state_dump_free_sync_objects - free sync objects array allocated by 287 * hl_state_dump_read_sync_objects 288 * @sync_objects: sync objects array 289 */ 290static void hl_state_dump_free_sync_objects(u32 *sync_objects) 291{ 292 vfree(sync_objects); 293} 294 295 296/** 297 * hl_state_dump_print_syncs_single_block - print active sync objects on a 298 * single block 299 * @hdev: pointer to the device 300 * @index: sync manager block index starting with E_N 301 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 302 * @size: pointer to the size container 303 * @offset: pointer to the offset container 304 * @map: sync engines names map 305 * 306 * Returns 0 on success or error code on failure 307 */ 308static int 309hl_state_dump_print_syncs_single_block(struct hl_device *hdev, u32 index, 310 char **buf, size_t *size, size_t *offset, 311 struct hl_sync_to_engine_map *map) 312{ 313 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 314 const char *sync_name; 315 u32 *sync_objects = NULL; 316 int rc = 0, i; 317 318 if (sds->sync_namager_names) { 319 rc = hl_snprintf_resize( 320 buf, size, offset, "%s\n", 321 sds->sync_namager_names[index]); 322 if (rc) 323 goto out; 324 } 325 326 sync_objects = hl_state_dump_read_sync_objects(hdev, index); 327 if (!sync_objects) { 328 rc = -ENOMEM; 329 goto out; 330 } 331 332 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) { 333 struct hl_sync_to_engine_map_entry *entry; 334 u64 sync_object_addr; 335 336 if (!sync_objects[i]) 337 continue; 338 339 sync_object_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] + 340 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index + 341 i * sizeof(u32); 342 343 rc = hl_snprintf_resize(buf, size, offset, "sync id: %u", i); 344 if (rc) 345 goto free_sync_objects; 346 sync_name = hl_state_dump_get_sync_name(hdev, i); 347 if (sync_name) { 348 rc = hl_snprintf_resize(buf, size, offset, " %s", 349 sync_name); 350 if (rc) 351 goto free_sync_objects; 352 } 353 rc = hl_snprintf_resize(buf, size, offset, ", value: %u", 354 sync_objects[i]); 355 if (rc) 356 goto free_sync_objects; 357 358 /* Append engine string */ 359 entry = hl_state_dump_get_sync_to_engine(map, 360 (u32)sync_object_addr); 361 if (entry) { 362 rc = hl_snprintf_resize(buf, size, offset, 363 ", Engine: "); 364 if (rc) 365 goto free_sync_objects; 366 rc = hl_print_resize_sync_engine(buf, size, offset, 367 entry->engine_type, 368 entry->engine_id); 369 if (rc) 370 goto free_sync_objects; 371 } 372 373 rc = hl_snprintf_resize(buf, size, offset, "\n"); 374 if (rc) 375 goto free_sync_objects; 376 } 377 378free_sync_objects: 379 hl_state_dump_free_sync_objects(sync_objects); 380out: 381 return rc; 382} 383 384/** 385 * hl_state_dump_print_syncs - print active sync objects 386 * @hdev: pointer to the device 387 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 388 * @size: pointer to the size container 389 * @offset: pointer to the offset container 390 * 391 * Returns 0 on success or error code on failure 392 */ 393static int hl_state_dump_print_syncs(struct hl_device *hdev, 394 char **buf, size_t *size, 395 size_t *offset) 396 397{ 398 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 399 struct hl_sync_to_engine_map *map; 400 u32 index; 401 int rc = 0; 402 403 map = kzalloc(sizeof(*map), GFP_KERNEL); 404 if (!map) 405 return -ENOMEM; 406 407 rc = sds->funcs.gen_sync_to_engine_map(hdev, map); 408 if (rc) 409 goto free_map_mem; 410 411 rc = hl_snprintf_resize(buf, size, offset, "Non zero sync objects:\n"); 412 if (rc) 413 goto out; 414 415 if (sds->sync_namager_names) { 416 for (index = 0; sds->sync_namager_names[index]; ++index) { 417 rc = hl_state_dump_print_syncs_single_block( 418 hdev, index, buf, size, offset, map); 419 if (rc) 420 goto out; 421 } 422 } else { 423 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) { 424 rc = hl_state_dump_print_syncs_single_block( 425 hdev, index, buf, size, offset, map); 426 if (rc) 427 goto out; 428 } 429 } 430 431out: 432 hl_state_dump_free_sync_to_engine_map(map); 433free_map_mem: 434 kfree(map); 435 436 return rc; 437} 438 439/** 440 * hl_state_dump_alloc_read_sm_block_monitors - read monitors for a specific 441 * block 442 * @hdev: pointer to the device 443 * @index: sync manager block index starting with E_N 444 * 445 * Returns an array of monitor data of size SP_MONITORS_AMOUNT or NULL 446 * on error 447 */ 448static struct hl_mon_state_dump * 449hl_state_dump_alloc_read_sm_block_monitors(struct hl_device *hdev, u32 index) 450{ 451 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 452 struct hl_mon_state_dump *monitors; 453 s64 base_addr; /* Base addr can be negative */ 454 int i; 455 456 monitors = vmalloc(sds->props[SP_MONITORS_AMOUNT] * 457 sizeof(struct hl_mon_state_dump)); 458 if (!monitors) 459 return NULL; 460 461 base_addr = sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index; 462 463 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) { 464 monitors[i].id = i; 465 monitors[i].wr_addr_low = 466 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_LOW] + 467 i * sizeof(u32)); 468 469 monitors[i].wr_addr_high = 470 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_HIGH] + 471 i * sizeof(u32)); 472 473 monitors[i].wr_data = 474 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_DATA] + 475 i * sizeof(u32)); 476 477 monitors[i].arm_data = 478 RREG32(base_addr + sds->props[SP_MON_OBJ_ARM_DATA] + 479 i * sizeof(u32)); 480 481 monitors[i].status = 482 RREG32(base_addr + sds->props[SP_MON_OBJ_STATUS] + 483 i * sizeof(u32)); 484 } 485 486 return monitors; 487} 488 489/** 490 * hl_state_dump_free_monitors - free the monitors structure 491 * @monitors: monitors array created with 492 * hl_state_dump_alloc_read_sm_block_monitors 493 */ 494static void hl_state_dump_free_monitors(struct hl_mon_state_dump *monitors) 495{ 496 vfree(monitors); 497} 498 499/** 500 * hl_state_dump_print_monitors_single_block - print active monitors on a 501 * single block 502 * @hdev: pointer to the device 503 * @index: sync manager block index starting with E_N 504 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 505 * @size: pointer to the size container 506 * @offset: pointer to the offset container 507 * 508 * Returns 0 on success or error code on failure 509 */ 510static int hl_state_dump_print_monitors_single_block(struct hl_device *hdev, 511 u32 index, 512 char **buf, size_t *size, 513 size_t *offset) 514{ 515 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 516 struct hl_mon_state_dump *monitors = NULL; 517 int rc = 0, i; 518 519 if (sds->sync_namager_names) { 520 rc = hl_snprintf_resize( 521 buf, size, offset, "%s\n", 522 sds->sync_namager_names[index]); 523 if (rc) 524 goto out; 525 } 526 527 monitors = hl_state_dump_alloc_read_sm_block_monitors(hdev, index); 528 if (!monitors) { 529 rc = -ENOMEM; 530 goto out; 531 } 532 533 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) { 534 if (!(sds->funcs.monitor_valid(&monitors[i]))) 535 continue; 536 537 /* Monitor is valid, dump it */ 538 rc = sds->funcs.print_single_monitor(buf, size, offset, hdev, 539 &monitors[i]); 540 if (rc) 541 goto free_monitors; 542 543 hl_snprintf_resize(buf, size, offset, "\n"); 544 } 545 546free_monitors: 547 hl_state_dump_free_monitors(monitors); 548out: 549 return rc; 550} 551 552/** 553 * hl_state_dump_print_monitors - print active monitors 554 * @hdev: pointer to the device 555 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 556 * @size: pointer to the size container 557 * @offset: pointer to the offset container 558 * 559 * Returns 0 on success or error code on failure 560 */ 561static int hl_state_dump_print_monitors(struct hl_device *hdev, 562 char **buf, size_t *size, 563 size_t *offset) 564{ 565 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 566 u32 index; 567 int rc = 0; 568 569 rc = hl_snprintf_resize(buf, size, offset, 570 "Valid (armed) monitor objects:\n"); 571 if (rc) 572 goto out; 573 574 if (sds->sync_namager_names) { 575 for (index = 0; sds->sync_namager_names[index]; ++index) { 576 rc = hl_state_dump_print_monitors_single_block( 577 hdev, index, buf, size, offset); 578 if (rc) 579 goto out; 580 } 581 } else { 582 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) { 583 rc = hl_state_dump_print_monitors_single_block( 584 hdev, index, buf, size, offset); 585 if (rc) 586 goto out; 587 } 588 } 589 590out: 591 return rc; 592} 593 594/** 595 * hl_state_dump_print_engine_fences - print active fences for a specific 596 * engine 597 * @hdev: pointer to the device 598 * @engine_type: engine type to use 599 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 600 * @size: pointer to the size container 601 * @offset: pointer to the offset container 602 */ 603static int 604hl_state_dump_print_engine_fences(struct hl_device *hdev, 605 enum hl_sync_engine_type engine_type, 606 char **buf, size_t *size, size_t *offset) 607{ 608 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 609 int rc = 0, i, n_fences; 610 u64 base_addr, next_fence; 611 612 switch (engine_type) { 613 case ENGINE_TPC: 614 n_fences = sds->props[SP_NUM_OF_TPC_ENGINES]; 615 base_addr = sds->props[SP_TPC0_CMDQ]; 616 next_fence = sds->props[SP_NEXT_TPC]; 617 break; 618 case ENGINE_MME: 619 n_fences = sds->props[SP_NUM_OF_MME_ENGINES]; 620 base_addr = sds->props[SP_MME_CMDQ]; 621 next_fence = sds->props[SP_NEXT_MME]; 622 break; 623 case ENGINE_DMA: 624 n_fences = sds->props[SP_NUM_OF_DMA_ENGINES]; 625 base_addr = sds->props[SP_DMA_CMDQ]; 626 next_fence = sds->props[SP_DMA_QUEUES_OFFSET]; 627 break; 628 default: 629 return -EINVAL; 630 } 631 for (i = 0; i < n_fences; ++i) { 632 rc = sds->funcs.print_fences_single_engine( 633 hdev, 634 base_addr + next_fence * i + 635 sds->props[SP_FENCE0_CNT_OFFSET], 636 base_addr + next_fence * i + 637 sds->props[SP_CP_STS_OFFSET], 638 engine_type, i, buf, size, offset); 639 if (rc) 640 goto out; 641 } 642out: 643 return rc; 644} 645 646/** 647 * hl_state_dump_print_fences - print active fences 648 * @hdev: pointer to the device 649 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 650 * @size: pointer to the size container 651 * @offset: pointer to the offset container 652 */ 653static int hl_state_dump_print_fences(struct hl_device *hdev, char **buf, 654 size_t *size, size_t *offset) 655{ 656 int rc = 0; 657 658 rc = hl_snprintf_resize(buf, size, offset, "Valid (armed) fences:\n"); 659 if (rc) 660 goto out; 661 662 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_TPC, buf, size, offset); 663 if (rc) 664 goto out; 665 666 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_MME, buf, size, offset); 667 if (rc) 668 goto out; 669 670 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_DMA, buf, size, offset); 671 if (rc) 672 goto out; 673 674out: 675 return rc; 676} 677 678/** 679 * hl_state_dump() - dump system state 680 * @hdev: pointer to device structure 681 */ 682int hl_state_dump(struct hl_device *hdev) 683{ 684 char *buf = NULL; 685 size_t offset = 0, size = 0; 686 int rc; 687 688 rc = hl_snprintf_resize(&buf, &size, &offset, 689 "Timestamp taken on: %llu\n\n", 690 ktime_to_ns(ktime_get())); 691 if (rc) 692 goto err; 693 694 rc = hl_state_dump_print_syncs(hdev, &buf, &size, &offset); 695 if (rc) 696 goto err; 697 698 hl_snprintf_resize(&buf, &size, &offset, "\n"); 699 700 rc = hl_state_dump_print_monitors(hdev, &buf, &size, &offset); 701 if (rc) 702 goto err; 703 704 hl_snprintf_resize(&buf, &size, &offset, "\n"); 705 706 rc = hl_state_dump_print_fences(hdev, &buf, &size, &offset); 707 if (rc) 708 goto err; 709 710 hl_snprintf_resize(&buf, &size, &offset, "\n"); 711 712 hl_debugfs_set_state_dump(hdev, buf, size); 713 714 return 0; 715err: 716 vfree(buf); 717 return rc; 718} 719