1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * sorttable.h 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <shile.zhang@linux.alibaba.com> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by: 12 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com> 13 * 14 * Some of this code was taken out of recordmcount.h written by: 15 * 16 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved. 17 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc. 18 */ 19 20#undef extable_ent_size 21#undef compare_extable 22#undef get_mcount_loc 23#undef sort_mcount_loc 24#undef elf_mcount_loc 25#undef do_sort 26#undef Elf_Addr 27#undef Elf_Ehdr 28#undef Elf_Shdr 29#undef Elf_Rel 30#undef Elf_Rela 31#undef Elf_Sym 32#undef ELF_R_SYM 33#undef Elf_r_sym 34#undef ELF_R_INFO 35#undef Elf_r_info 36#undef ELF_ST_BIND 37#undef ELF_ST_TYPE 38#undef fn_ELF_R_SYM 39#undef fn_ELF_R_INFO 40#undef uint_t 41#undef _r 42#undef _w 43 44#ifdef SORTTABLE_64 45# define extable_ent_size 16 46# define compare_extable compare_extable_64 47# define get_mcount_loc get_mcount_loc_64 48# define sort_mcount_loc sort_mcount_loc_64 49# define elf_mcount_loc elf_mcount_loc_64 50# define do_sort do_sort_64 51# define Elf_Addr Elf64_Addr 52# define Elf_Ehdr Elf64_Ehdr 53# define Elf_Shdr Elf64_Shdr 54# define Elf_Rel Elf64_Rel 55# define Elf_Rela Elf64_Rela 56# define Elf_Sym Elf64_Sym 57# define ELF_R_SYM ELF64_R_SYM 58# define Elf_r_sym Elf64_r_sym 59# define ELF_R_INFO ELF64_R_INFO 60# define Elf_r_info Elf64_r_info 61# define ELF_ST_BIND ELF64_ST_BIND 62# define ELF_ST_TYPE ELF64_ST_TYPE 63# define fn_ELF_R_SYM fn_ELF64_R_SYM 64# define fn_ELF_R_INFO fn_ELF64_R_INFO 65# define uint_t uint64_t 66# define _r r8 67# define _w w8 68#else 69# define extable_ent_size 8 70# define compare_extable compare_extable_32 71# define get_mcount_loc get_mcount_loc_32 72# define sort_mcount_loc sort_mcount_loc_32 73# define elf_mcount_loc elf_mcount_loc_32 74# define do_sort do_sort_32 75# define Elf_Addr Elf32_Addr 76# define Elf_Ehdr Elf32_Ehdr 77# define Elf_Shdr Elf32_Shdr 78# define Elf_Rel Elf32_Rel 79# define Elf_Rela Elf32_Rela 80# define Elf_Sym Elf32_Sym 81# define ELF_R_SYM ELF32_R_SYM 82# define Elf_r_sym Elf32_r_sym 83# define ELF_R_INFO ELF32_R_INFO 84# define Elf_r_info Elf32_r_info 85# define ELF_ST_BIND ELF32_ST_BIND 86# define ELF_ST_TYPE ELF32_ST_TYPE 87# define fn_ELF_R_SYM fn_ELF32_R_SYM 88# define fn_ELF_R_INFO fn_ELF32_R_INFO 89# define uint_t uint32_t 90# define _r r 91# define _w w 92#endif 93 94#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 95/* ORC unwinder only support X86_64 */ 96#include <asm/orc_types.h> 97 98#define ERRSTR_MAXSZ 256 99 100char g_err[ERRSTR_MAXSZ]; 101int *g_orc_ip_table; 102struct orc_entry *g_orc_table; 103 104pthread_t orc_sort_thread; 105 106static inline unsigned long orc_ip(const int *ip) 107{ 108 return (unsigned long)ip + *ip; 109} 110 111static int orc_sort_cmp(const void *_a, const void *_b) 112{ 113 struct orc_entry *orc_a; 114 const int *a = g_orc_ip_table + *(int *)_a; 115 const int *b = g_orc_ip_table + *(int *)_b; 116 unsigned long a_val = orc_ip(a); 117 unsigned long b_val = orc_ip(b); 118 119 if (a_val > b_val) 120 return 1; 121 if (a_val < b_val) 122 return -1; 123 124 /* 125 * The "weak" section terminator entries need to always be on the left 126 * to ensure the lookup code skips them in favor of real entries. 127 * These terminator entries exist to handle any gaps created by 128 * whitelisted .o files which didn't get objtool generation. 129 */ 130 orc_a = g_orc_table + (a - g_orc_ip_table); 131 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 132} 133 134static void *sort_orctable(void *arg) 135{ 136 int i; 137 int *idxs = NULL; 138 int *tmp_orc_ip_table = NULL; 139 struct orc_entry *tmp_orc_table = NULL; 140 unsigned int *orc_ip_size = (unsigned int *)arg; 141 unsigned int num_entries = *orc_ip_size / sizeof(int); 142 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 143 144 idxs = (int *)malloc(*orc_ip_size); 145 if (!idxs) { 146 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 147 strerror(errno)); 148 pthread_exit(g_err); 149 } 150 151 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 152 if (!tmp_orc_ip_table) { 153 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 154 strerror(errno)); 155 pthread_exit(g_err); 156 } 157 158 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 159 if (!tmp_orc_table) { 160 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 161 strerror(errno)); 162 pthread_exit(g_err); 163 } 164 165 /* initialize indices array, convert ip_table to absolute address */ 166 for (i = 0; i < num_entries; i++) { 167 idxs[i] = i; 168 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 169 } 170 memcpy(tmp_orc_table, g_orc_table, orc_size); 171 172 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 173 174 for (i = 0; i < num_entries; i++) { 175 if (idxs[i] == i) 176 continue; 177 178 /* convert back to relative address */ 179 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 180 g_orc_table[i] = tmp_orc_table[idxs[i]]; 181 } 182 183 free(idxs); 184 free(tmp_orc_ip_table); 185 free(tmp_orc_table); 186 pthread_exit(NULL); 187} 188#endif 189 190static int compare_extable(const void *a, const void *b) 191{ 192 Elf_Addr av = _r(a); 193 Elf_Addr bv = _r(b); 194 195 if (av < bv) 196 return -1; 197 if (av > bv) 198 return 1; 199 return 0; 200} 201#ifdef MCOUNT_SORT_ENABLED 202pthread_t mcount_sort_thread; 203 204struct elf_mcount_loc { 205 Elf_Ehdr *ehdr; 206 Elf_Shdr *init_data_sec; 207 uint_t start_mcount_loc; 208 uint_t stop_mcount_loc; 209}; 210 211/* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 212static void *sort_mcount_loc(void *arg) 213{ 214 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 215 uint_t offset = emloc->start_mcount_loc - _r(&(emloc->init_data_sec)->sh_addr) 216 + _r(&(emloc->init_data_sec)->sh_offset); 217 uint_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc; 218 unsigned char *start_loc = (void *)emloc->ehdr + offset; 219 220 qsort(start_loc, count/sizeof(uint_t), sizeof(uint_t), compare_extable); 221 return NULL; 222} 223 224/* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 225static void get_mcount_loc(uint_t *_start, uint_t *_stop) 226{ 227 FILE *file_start, *file_stop; 228 char start_buff[20]; 229 char stop_buff[20]; 230 int len = 0; 231 232 file_start = popen(" grep start_mcount System.map | awk '{print $1}' ", "r"); 233 if (!file_start) { 234 fprintf(stderr, "get start_mcount_loc error!"); 235 return; 236 } 237 238 file_stop = popen(" grep stop_mcount System.map | awk '{print $1}' ", "r"); 239 if (!file_stop) { 240 fprintf(stderr, "get stop_mcount_loc error!"); 241 pclose(file_start); 242 return; 243 } 244 245 while (fgets(start_buff, sizeof(start_buff), file_start) != NULL) { 246 len = strlen(start_buff); 247 start_buff[len - 1] = '\0'; 248 } 249 *_start = strtoul(start_buff, NULL, 16); 250 251 while (fgets(stop_buff, sizeof(stop_buff), file_stop) != NULL) { 252 len = strlen(stop_buff); 253 stop_buff[len - 1] = '\0'; 254 } 255 *_stop = strtoul(stop_buff, NULL, 16); 256 257 pclose(file_start); 258 pclose(file_stop); 259} 260#endif 261static int do_sort(Elf_Ehdr *ehdr, 262 char const *const fname, 263 table_sort_t custom_sort) 264{ 265 int rc = -1; 266 Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff)); 267 Elf_Shdr *strtab_sec = NULL; 268 Elf_Shdr *symtab_sec = NULL; 269 Elf_Shdr *extab_sec = NULL; 270 Elf_Sym *sym; 271 const Elf_Sym *symtab; 272 Elf32_Word *symtab_shndx = NULL; 273 Elf_Sym *sort_needed_sym = NULL; 274 Elf_Shdr *sort_needed_sec; 275 Elf_Rel *relocs = NULL; 276 int relocs_size = 0; 277 uint32_t *sort_needed_loc; 278 const char *secstrings; 279 const char *strtab; 280 char *extab_image; 281 int extab_index = 0; 282 int i; 283 int idx; 284 unsigned int shnum; 285 unsigned int shstrndx; 286#ifdef MCOUNT_SORT_ENABLED 287 struct elf_mcount_loc mstruct = {0}; 288 uint_t _start_mcount_loc = 0; 289 uint_t _stop_mcount_loc = 0; 290#endif 291#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 292 unsigned int orc_ip_size = 0; 293 unsigned int orc_size = 0; 294 unsigned int orc_num_entries = 0; 295#endif 296 297 shstrndx = r2(&ehdr->e_shstrndx); 298 if (shstrndx == SHN_XINDEX) 299 shstrndx = r(&shdr[0].sh_link); 300 secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset); 301 302 shnum = r2(&ehdr->e_shnum); 303 if (shnum == SHN_UNDEF) 304 shnum = _r(&shdr[0].sh_size); 305 306 for (i = 0, s = shdr; s < shdr + shnum; i++, s++) { 307 idx = r(&s->sh_name); 308 if (!strcmp(secstrings + idx, "__ex_table")) { 309 extab_sec = s; 310 extab_index = i; 311 } 312 if (!strcmp(secstrings + idx, ".symtab")) 313 symtab_sec = s; 314 if (!strcmp(secstrings + idx, ".strtab")) 315 strtab_sec = s; 316 317 if ((r(&s->sh_type) == SHT_REL || 318 r(&s->sh_type) == SHT_RELA) && 319 r(&s->sh_info) == extab_index) { 320 relocs = (void *)ehdr + _r(&s->sh_offset); 321 relocs_size = _r(&s->sh_size); 322 } 323 if (r(&s->sh_type) == SHT_SYMTAB_SHNDX) 324 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 325 _r(&s->sh_offset)); 326 327#ifdef MCOUNT_SORT_ENABLED 328 /* locate the .init.data section in vmlinux */ 329 if (!strcmp(secstrings + idx, ".init.data")) { 330 get_mcount_loc(&_start_mcount_loc, &_stop_mcount_loc); 331 mstruct.ehdr = ehdr; 332 mstruct.init_data_sec = s; 333 mstruct.start_mcount_loc = _start_mcount_loc; 334 mstruct.stop_mcount_loc = _stop_mcount_loc; 335 } 336#endif 337 338#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 339 /* locate the ORC unwind tables */ 340 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 341 orc_ip_size = s->sh_size; 342 g_orc_ip_table = (int *)((void *)ehdr + 343 s->sh_offset); 344 } 345 if (!strcmp(secstrings + idx, ".orc_unwind")) { 346 orc_size = s->sh_size; 347 g_orc_table = (struct orc_entry *)((void *)ehdr + 348 s->sh_offset); 349 } 350#endif 351 } /* for loop */ 352 353#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 354 if (!g_orc_ip_table || !g_orc_table) { 355 fprintf(stderr, 356 "incomplete ORC unwind tables in file: %s\n", fname); 357 goto out; 358 } 359 360 orc_num_entries = orc_ip_size / sizeof(int); 361 if (orc_ip_size % sizeof(int) != 0 || 362 orc_size % sizeof(struct orc_entry) != 0 || 363 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 364 fprintf(stderr, 365 "inconsistent ORC unwind table entries in file: %s\n", 366 fname); 367 goto out; 368 } 369 370 /* create thread to sort ORC unwind tables concurrently */ 371 if (pthread_create(&orc_sort_thread, NULL, 372 sort_orctable, &orc_ip_size)) { 373 fprintf(stderr, 374 "pthread_create orc_sort_thread failed '%s': %s\n", 375 strerror(errno), fname); 376 goto out; 377 } 378#endif 379 380#ifdef MCOUNT_SORT_ENABLED 381 if (!mstruct.init_data_sec || !_start_mcount_loc || !_stop_mcount_loc) { 382 fprintf(stderr, 383 "incomplete mcount's sort in file: %s\n", 384 fname); 385 goto out; 386 } 387 388 /* create thread to sort mcount_loc concurrently */ 389 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 390 fprintf(stderr, 391 "pthread_create mcount_sort_thread failed '%s': %s\n", 392 strerror(errno), fname); 393 goto out; 394 } 395#endif 396 if (!extab_sec) { 397 fprintf(stderr, "no __ex_table in file: %s\n", fname); 398 goto out; 399 } 400 401 if (!symtab_sec) { 402 fprintf(stderr, "no .symtab in file: %s\n", fname); 403 goto out; 404 } 405 406 if (!strtab_sec) { 407 fprintf(stderr, "no .strtab in file: %s\n", fname); 408 goto out; 409 } 410 411 extab_image = (void *)ehdr + _r(&extab_sec->sh_offset); 412 strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset); 413 symtab = (const Elf_Sym *)((const char *)ehdr + 414 _r(&symtab_sec->sh_offset)); 415 416 if (custom_sort) { 417 custom_sort(extab_image, _r(&extab_sec->sh_size)); 418 } else { 419 int num_entries = _r(&extab_sec->sh_size) / extable_ent_size; 420 qsort(extab_image, num_entries, 421 extable_ent_size, compare_extable); 422 } 423 424 /* If there were relocations, we no longer need them. */ 425 if (relocs) 426 memset(relocs, 0, relocs_size); 427 428 /* find the flag main_extable_sort_needed */ 429 for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset); 430 sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym); 431 sym++) { 432 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT) 433 continue; 434 if (!strcmp(strtab + r(&sym->st_name), 435 "main_extable_sort_needed")) { 436 sort_needed_sym = sym; 437 break; 438 } 439 } 440 441 if (!sort_needed_sym) { 442 fprintf(stderr, 443 "no main_extable_sort_needed symbol in file: %s\n", 444 fname); 445 goto out; 446 } 447 448 sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx), 449 sort_needed_sym - symtab, 450 symtab_shndx)]; 451 sort_needed_loc = (void *)ehdr + 452 _r(&sort_needed_sec->sh_offset) + 453 _r(&sort_needed_sym->st_value) - 454 _r(&sort_needed_sec->sh_addr); 455 456 /* extable has been sorted, clear the flag */ 457 w(0, sort_needed_loc); 458 rc = 0; 459 460out: 461#if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 462 if (orc_sort_thread) { 463 void *retval = NULL; 464 /* wait for ORC tables sort done */ 465 rc = pthread_join(orc_sort_thread, &retval); 466 if (rc) { 467 fprintf(stderr, 468 "pthread_join failed '%s': %s\n", 469 strerror(errno), fname); 470 } else if (retval) { 471 rc = -1; 472 fprintf(stderr, 473 "failed to sort ORC tables '%s': %s\n", 474 (char *)retval, fname); 475 } 476 } 477#endif 478 479#ifdef MCOUNT_SORT_ENABLED 480 if (mcount_sort_thread) { 481 void *retval = NULL; 482 /* wait for mcount sort done */ 483 rc = pthread_join(mcount_sort_thread, &retval); 484 if (rc) { 485 fprintf(stderr, 486 "pthread_join failed '%s': %s\n", 487 strerror(errno), fname); 488 } else if (retval) { 489 rc = -1; 490 fprintf(stderr, 491 "failed to sort mcount '%s': %s\n", 492 (char *)retval, fname); 493 } 494 } 495#endif 496 return rc; 497} 498