138514Sdfr/*- 259603Sdfr * Copyright (c) 1998-2000 Doug Rabson 338514Sdfr * All rights reserved. 438514Sdfr * 538514Sdfr * Redistribution and use in source and binary forms, with or without 638514Sdfr * modification, are permitted provided that the following conditions 738514Sdfr * are met: 838514Sdfr * 1. Redistributions of source code must retain the above copyright 938514Sdfr * notice, this list of conditions and the following disclaimer. 1038514Sdfr * 2. Redistributions in binary form must reproduce the above copyright 1138514Sdfr * notice, this list of conditions and the following disclaimer in the 1238514Sdfr * documentation and/or other materials provided with the distribution. 1338514Sdfr * 1438514Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1538514Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1638514Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1738514Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1838514Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1938514Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2038514Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2138514Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2238514Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2338514Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2438514Sdfr * SUCH DAMAGE. 2538514Sdfr */ 2638514Sdfr 27116182Sobrien#include <sys/cdefs.h> 28116182Sobrien__FBSDID("$FreeBSD: stable/11/sys/kern/link_elf.c 359652 2020-04-06 07:16:31Z hselasky $"); 29116182Sobrien 30179223Sjb#include "opt_ddb.h" 31131928Smarcel#include "opt_gdb.h" 3259603Sdfr 3338514Sdfr#include <sys/param.h> 3476166Smarkm#include <sys/systm.h> 35129443Sbde#ifdef GPROF 36129443Sbde#include <sys/gmon.h> 37129443Sbde#endif 3838514Sdfr#include <sys/kernel.h> 3976166Smarkm#include <sys/lock.h> 4038514Sdfr#include <sys/malloc.h> 4177642Sdd#include <sys/mutex.h> 42159808Sjhb#include <sys/mount.h> 43194784Sjeff#include <sys/pcpu.h> 4438514Sdfr#include <sys/proc.h> 4538514Sdfr#include <sys/namei.h> 4638514Sdfr#include <sys/fcntl.h> 4738514Sdfr#include <sys/vnode.h> 4838514Sdfr#include <sys/linker.h> 4976166Smarkm 5038514Sdfr#include <machine/elf.h> 5138514Sdfr 52195699Srwatson#include <net/vnet.h> 53195699Srwatson 54163606Srwatson#include <security/mac/mac_framework.h> 55163606Srwatson 5639071Sdfr#include <vm/vm.h> 5739071Sdfr#include <vm/vm_param.h> 5852128Speter#ifdef SPARSE_MAPPING 5939071Sdfr#include <vm/vm_object.h> 6039071Sdfr#include <vm/vm_kern.h> 6139071Sdfr#include <vm/vm_extern.h> 6252128Speter#endif 6339071Sdfr#include <vm/pmap.h> 6439071Sdfr#include <vm/vm_map.h> 6576166Smarkm 66102288Speter#include <sys/link_elf.h> 6739071Sdfr 68179223Sjb#ifdef DDB_CTF 69281855Srodrigc#include <sys/zlib.h> 70179223Sjb#endif 71179223Sjb 7259603Sdfr#include "linker_if.h" 7338514Sdfr 74151430Speter#define MAXSEGS 4 75151430Speter 76180438Sobrientypedef struct elf_file { 77215013Smdf struct linker_file lf; /* Common fields */ 78215013Smdf int preloaded; /* Was file pre-loaded */ 79215013Smdf caddr_t address; /* Relocation address */ 80180438Sobrien#ifdef SPARSE_MAPPING 81215013Smdf vm_object_t object; /* VM object to hold file pages */ 82180438Sobrien#endif 83215013Smdf Elf_Dyn *dynamic; /* Symbol table etc. */ 84215013Smdf Elf_Hashelt nbuckets; /* DT_HASH info */ 85215013Smdf Elf_Hashelt nchains; 86215013Smdf const Elf_Hashelt *buckets; 87215013Smdf const Elf_Hashelt *chains; 88215013Smdf caddr_t hash; 89215013Smdf caddr_t strtab; /* DT_STRTAB */ 90215013Smdf int strsz; /* DT_STRSZ */ 91215013Smdf const Elf_Sym *symtab; /* DT_SYMTAB */ 92215013Smdf Elf_Addr *got; /* DT_PLTGOT */ 93215013Smdf const Elf_Rel *pltrel; /* DT_JMPREL */ 94215013Smdf int pltrelsize; /* DT_PLTRELSZ */ 95215013Smdf const Elf_Rela *pltrela; /* DT_JMPREL */ 96215013Smdf int pltrelasize; /* DT_PLTRELSZ */ 97215013Smdf const Elf_Rel *rel; /* DT_REL */ 98215013Smdf int relsize; /* DT_RELSZ */ 99215013Smdf const Elf_Rela *rela; /* DT_RELA */ 100215013Smdf int relasize; /* DT_RELASZ */ 101215013Smdf caddr_t modptr; 102215013Smdf const Elf_Sym *ddbsymtab; /* The symbol table we are using */ 103215013Smdf long ddbsymcnt; /* Number of symbols */ 104215013Smdf caddr_t ddbstrtab; /* String table */ 105215013Smdf long ddbstrcnt; /* number of bytes in string table */ 106215013Smdf caddr_t symbase; /* malloc'ed symbold base */ 107215013Smdf caddr_t strbase; /* malloc'ed string base */ 108215013Smdf caddr_t ctftab; /* CTF table */ 109215013Smdf long ctfcnt; /* number of bytes in CTF table */ 110215013Smdf caddr_t ctfoff; /* CTF offset table */ 111215013Smdf caddr_t typoff; /* Type offset table */ 112215013Smdf long typlen; /* Number of type entries. */ 113215013Smdf Elf_Addr pcpu_start; /* Pre-relocation pcpu set start. */ 114215013Smdf Elf_Addr pcpu_stop; /* Pre-relocation pcpu set stop. */ 115215013Smdf Elf_Addr pcpu_base; /* Relocated pcpu set address. */ 116195699Srwatson#ifdef VIMAGE 117215013Smdf Elf_Addr vnet_start; /* Pre-relocation vnet set start. */ 118215013Smdf Elf_Addr vnet_stop; /* Pre-relocation vnet set stop. */ 119215013Smdf Elf_Addr vnet_base; /* Relocated vnet set address. */ 120195699Srwatson#endif 121180438Sobrien#ifdef GDB 122215013Smdf struct link_map gdb; /* hooks for gdb */ 123180438Sobrien#endif 124180438Sobrien} *elf_file_t; 125180438Sobrien 126240997Strocinystruct elf_set { 127240997Strociny Elf_Addr es_start; 128240997Strociny Elf_Addr es_stop; 129240997Strociny Elf_Addr es_base; 130240997Strociny TAILQ_ENTRY(elf_set) es_link; 131240997Strociny}; 132240997Strociny 133240997StrocinyTAILQ_HEAD(elf_set_head, elf_set); 134240997Strociny 135179223Sjb#include <kern/kern_ctf.c> 136179223Sjb 137105468Smarcelstatic int link_elf_link_common_finish(linker_file_t); 13859751Speterstatic int link_elf_link_preload(linker_class_t cls, 139215013Smdf const char *, linker_file_t *); 14059751Speterstatic int link_elf_link_preload_finish(linker_file_t); 141215013Smdfstatic int link_elf_load_file(linker_class_t, const char *, 142215013Smdf linker_file_t *); 143215013Smdfstatic int link_elf_lookup_symbol(linker_file_t, const char *, 144215013Smdf c_linker_sym_t *); 145215013Smdfstatic int link_elf_symbol_values(linker_file_t, c_linker_sym_t, 146215013Smdf linker_symval_t *); 147215013Smdfstatic int link_elf_search_symbol(linker_file_t, caddr_t, 148215013Smdf c_linker_sym_t *, long *); 14938514Sdfr 15059603Sdfrstatic void link_elf_unload_file(linker_file_t); 15159751Speterstatic void link_elf_unload_preload(linker_file_t); 15278161Speterstatic int link_elf_lookup_set(linker_file_t, const char *, 153215013Smdf void ***, void ***, int *); 15485736Sgreenstatic int link_elf_each_function_name(linker_file_t, 155215013Smdf int (*)(const char *, void *), void *); 156179223Sjbstatic int link_elf_each_function_nameval(linker_file_t, 157215013Smdf linker_function_nameval_callback_t, void *); 158109605Sjakestatic void link_elf_reloc_local(linker_file_t); 159192859Sssonstatic long link_elf_symtab_get(linker_file_t, const Elf_Sym **); 160192859Sssonstatic long link_elf_strtab_get(linker_file_t, caddr_t *); 161288000Skibstatic int elf_lookup(linker_file_t, Elf_Size, int, Elf_Addr *); 16259603Sdfr 16359603Sdfrstatic kobj_method_t link_elf_methods[] = { 164215013Smdf KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol), 165215013Smdf KOBJMETHOD(linker_symbol_values, link_elf_symbol_values), 166215013Smdf KOBJMETHOD(linker_search_symbol, link_elf_search_symbol), 167215013Smdf KOBJMETHOD(linker_unload, link_elf_unload_file), 168215013Smdf KOBJMETHOD(linker_load_file, link_elf_load_file), 169215013Smdf KOBJMETHOD(linker_link_preload, link_elf_link_preload), 170215013Smdf KOBJMETHOD(linker_link_preload_finish, link_elf_link_preload_finish), 171215013Smdf KOBJMETHOD(linker_lookup_set, link_elf_lookup_set), 172215013Smdf KOBJMETHOD(linker_each_function_name, link_elf_each_function_name), 173215013Smdf KOBJMETHOD(linker_each_function_nameval, link_elf_each_function_nameval), 174215013Smdf KOBJMETHOD(linker_ctf_get, link_elf_ctf_get), 175215013Smdf KOBJMETHOD(linker_symtab_get, link_elf_symtab_get), 176215013Smdf KOBJMETHOD(linker_strtab_get, link_elf_strtab_get), 177215013Smdf { 0, 0 } 17859603Sdfr}; 17959603Sdfr 18059603Sdfrstatic struct linker_class link_elf_class = { 18159603Sdfr#if ELF_TARG_CLASS == ELFCLASS32 182215013Smdf "elf32", 18359603Sdfr#else 184215013Smdf "elf64", 18559603Sdfr#endif 186215013Smdf link_elf_methods, sizeof(struct elf_file) 18759603Sdfr}; 18859603Sdfr 189338867Smarkjtypedef int (*elf_reloc_fn)(linker_file_t lf, Elf_Addr relocbase, 190338867Smarkj const void *data, int type, elf_lookup_fn lookup); 191338867Smarkj 192215013Smdfstatic int parse_dynamic(elf_file_t); 193215013Smdfstatic int relocate_file(elf_file_t); 194338867Smarkjstatic int relocate_file1(elf_file_t ef, elf_lookup_fn lookup, 195338867Smarkj elf_reloc_fn reloc, bool ifuncs); 196215013Smdfstatic int link_elf_preload_parse_symbols(elf_file_t); 19759603Sdfr 198240997Strocinystatic struct elf_set_head set_pcpu_list; 199240997Strociny#ifdef VIMAGE 200240997Strocinystatic struct elf_set_head set_vnet_list; 201240997Strociny#endif 202240997Strociny 203240997Strocinystatic void 204240997Strocinyelf_set_add(struct elf_set_head *list, Elf_Addr start, Elf_Addr stop, Elf_Addr base) 205240997Strociny{ 206240997Strociny struct elf_set *set, *iter; 207240997Strociny 208240997Strociny set = malloc(sizeof(*set), M_LINKER, M_WAITOK); 209240997Strociny set->es_start = start; 210240997Strociny set->es_stop = stop; 211240997Strociny set->es_base = base; 212240997Strociny 213240997Strociny TAILQ_FOREACH(iter, list, es_link) { 214240997Strociny 215240997Strociny KASSERT((set->es_start < iter->es_start && set->es_stop < iter->es_stop) || 216240997Strociny (set->es_start > iter->es_start && set->es_stop > iter->es_stop), 217240997Strociny ("linker sets intersection: to insert: 0x%jx-0x%jx; inserted: 0x%jx-0x%jx", 218240997Strociny (uintmax_t)set->es_start, (uintmax_t)set->es_stop, 219240997Strociny (uintmax_t)iter->es_start, (uintmax_t)iter->es_stop)); 220240997Strociny 221240997Strociny if (iter->es_start > set->es_start) { 222240997Strociny TAILQ_INSERT_BEFORE(iter, set, es_link); 223240997Strociny break; 224240997Strociny } 225240997Strociny } 226240997Strociny 227240997Strociny if (iter == NULL) 228240997Strociny TAILQ_INSERT_TAIL(list, set, es_link); 229240997Strociny} 230240997Strociny 231240997Strocinystatic int 232240997Strocinyelf_set_find(struct elf_set_head *list, Elf_Addr addr, Elf_Addr *start, Elf_Addr *base) 233240997Strociny{ 234240997Strociny struct elf_set *set; 235240997Strociny 236240997Strociny TAILQ_FOREACH(set, list, es_link) { 237240997Strociny if (addr < set->es_start) 238240997Strociny return (0); 239240997Strociny if (addr < set->es_stop) { 240240997Strociny *start = set->es_start; 241240997Strociny *base = set->es_base; 242240997Strociny return (1); 243240997Strociny } 244240997Strociny } 245240997Strociny 246240997Strociny return (0); 247240997Strociny} 248240997Strociny 249240997Strocinystatic void 250240997Strocinyelf_set_delete(struct elf_set_head *list, Elf_Addr start) 251240997Strociny{ 252240997Strociny struct elf_set *set; 253240997Strociny 254240997Strociny TAILQ_FOREACH(set, list, es_link) { 255240997Strociny if (start < set->es_start) 256240997Strociny break; 257240997Strociny if (start == set->es_start) { 258240997Strociny TAILQ_REMOVE(list, set, es_link); 259240997Strociny free(set, M_LINKER); 260240997Strociny return; 261240997Strociny } 262240997Strociny } 263240997Strociny KASSERT(0, ("deleting unknown linker set (start = 0x%jx)", 264240997Strociny (uintmax_t)start)); 265240997Strociny} 266240997Strociny 267131928Smarcel#ifdef GDB 268215013Smdfstatic void r_debug_state(struct r_debug *, struct link_map *); 26959603Sdfr 27038514Sdfr/* 27159603Sdfr * A list of loaded modules for GDB to use for loading symbols. 27259603Sdfr */ 27359603Sdfrstruct r_debug r_debug; 27459603Sdfr 275215013Smdf#define GDB_STATE(s) do { \ 276215013Smdf r_debug.r_state = s; r_debug_state(NULL, NULL); \ 277215013Smdf} while (0) 27859603Sdfr 27959603Sdfr/* 28059603Sdfr * Function for the debugger to set a breakpoint on to gain control. 28159603Sdfr */ 282104094Sphkstatic void 28366719Sjhbr_debug_state(struct r_debug *dummy_one __unused, 28466719Sjhb struct link_map *dummy_two __unused) 28559603Sdfr{ 28659603Sdfr} 28759603Sdfr 288105467Smarcelstatic void 289105467Smarcellink_elf_add_gdb(struct link_map *l) 290105467Smarcel{ 291215013Smdf struct link_map *prev; 29259603Sdfr 293215013Smdf l->l_next = NULL; 294105467Smarcel 295215013Smdf if (r_debug.r_map == NULL) { 296215013Smdf /* Add first. */ 297215013Smdf l->l_prev = NULL; 298215013Smdf r_debug.r_map = l; 299215013Smdf } else { 300215013Smdf /* Append to list. */ 301215013Smdf for (prev = r_debug.r_map; 302215013Smdf prev->l_next != NULL; 303215013Smdf prev = prev->l_next) 304215013Smdf ; 305215013Smdf l->l_prev = prev; 306215013Smdf prev->l_next = l; 307215013Smdf } 308105467Smarcel} 309105467Smarcel 310105467Smarcelstatic void 311105467Smarcellink_elf_delete_gdb(struct link_map *l) 312105467Smarcel{ 313215013Smdf if (l->l_prev == NULL) { 314215013Smdf /* Remove first. */ 315215013Smdf if ((r_debug.r_map = l->l_next) != NULL) 316215013Smdf l->l_next->l_prev = NULL; 317215013Smdf } else { 318215013Smdf /* Remove any but first. */ 319215013Smdf if ((l->l_prev->l_next = l->l_next) != NULL) 320215013Smdf l->l_next->l_prev = l->l_prev; 321215013Smdf } 322105467Smarcel} 323131928Smarcel#endif /* GDB */ 324105467Smarcel 325180438Sobrien/* 326180438Sobrien * The kernel symbol table starts here. 327180438Sobrien */ 328180438Sobrienextern struct _dynamic _DYNAMIC; 329180438Sobrien 33038514Sdfrstatic void 331180374Sedwinlink_elf_error(const char *filename, const char *s) 332105467Smarcel{ 333180374Sedwin if (filename == NULL) 334180374Sedwin printf("kldload: %s\n", s); 335180374Sedwin else 336180374Sedwin printf("kldload: %s: %s\n", filename, s); 337105467Smarcel} 338105467Smarcel 339273334Smarcelstatic void 340273334Smarcellink_elf_invoke_ctors(caddr_t addr, size_t size) 341273334Smarcel{ 342273334Smarcel void (**ctor)(void); 343273334Smarcel size_t i, cnt; 344273334Smarcel 345273334Smarcel if (addr == NULL || size == 0) 346273334Smarcel return; 347273334Smarcel cnt = size / sizeof(*ctor); 348273334Smarcel ctor = (void *)addr; 349273334Smarcel for (i = 0; i < cnt; i++) { 350273334Smarcel if (ctor[i] != NULL) 351273334Smarcel (*ctor[i])(); 352273334Smarcel } 353273334Smarcel} 354273334Smarcel 355105468Smarcel/* 356105468Smarcel * Actions performed after linking/loading both the preloaded kernel and any 357105468Smarcel * modules; whether preloaded or dynamicly loaded. 358105468Smarcel */ 359105468Smarcelstatic int 360105468Smarcellink_elf_link_common_finish(linker_file_t lf) 361105468Smarcel{ 362131928Smarcel#ifdef GDB 363215013Smdf elf_file_t ef = (elf_file_t)lf; 364215013Smdf char *newfilename; 365105468Smarcel#endif 366215013Smdf int error; 367105468Smarcel 368215013Smdf /* Notify MD code that a module is being loaded. */ 369215013Smdf error = elf_cpu_load_file(lf); 370215013Smdf if (error != 0) 371215013Smdf return (error); 372105469Smarcel 373131928Smarcel#ifdef GDB 374215013Smdf GDB_STATE(RT_ADD); 375215013Smdf ef->gdb.l_addr = lf->address; 376215013Smdf newfilename = malloc(strlen(lf->filename) + 1, M_LINKER, M_WAITOK); 377215013Smdf strcpy(newfilename, lf->filename); 378215013Smdf ef->gdb.l_name = newfilename; 379215013Smdf ef->gdb.l_ld = ef->dynamic; 380215013Smdf link_elf_add_gdb(&ef->gdb); 381215013Smdf GDB_STATE(RT_CONSISTENT); 382105468Smarcel#endif 383105468Smarcel 384273334Smarcel /* Invoke .ctors */ 385273334Smarcel link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size); 386215013Smdf return (0); 387105468Smarcel} 388105468Smarcel 389279119Snwhitehornextern vm_offset_t __startkernel; 390279119Snwhitehorn 391105467Smarcelstatic void 39238514Sdfrlink_elf_init(void* arg) 39338514Sdfr{ 394215013Smdf Elf_Dyn *dp; 395273334Smarcel Elf_Addr *ctors_addrp; 396273334Smarcel Elf_Size *ctors_sizep; 397215013Smdf caddr_t modptr, baseptr, sizeptr; 398215013Smdf elf_file_t ef; 399215013Smdf char *modname; 40038514Sdfr 401215013Smdf linker_add_class(&link_elf_class); 40238514Sdfr 403215013Smdf dp = (Elf_Dyn *)&_DYNAMIC; 404215013Smdf modname = NULL; 405215013Smdf modptr = preload_search_by_type("elf" __XSTRING(__ELF_WORD_SIZE) " kernel"); 406215013Smdf if (modptr == NULL) 407215013Smdf modptr = preload_search_by_type("elf kernel"); 408287000Sroyger modname = (char *)preload_search_info(modptr, MODINFO_NAME); 409215013Smdf if (modname == NULL) 410215013Smdf modname = "kernel"; 411215013Smdf linker_kernel_file = linker_make_file(modname, &link_elf_class); 412215013Smdf if (linker_kernel_file == NULL) 413215013Smdf panic("%s: Can't create linker structures for kernel", 414215013Smdf __func__); 415105468Smarcel 416215013Smdf ef = (elf_file_t) linker_kernel_file; 417215013Smdf ef->preloaded = 1; 418279750Snwhitehorn#ifdef __powerpc__ 419279119Snwhitehorn ef->address = (caddr_t) (__startkernel - KERNBASE); 420279119Snwhitehorn#else 421215013Smdf ef->address = 0; 422279119Snwhitehorn#endif 42359603Sdfr#ifdef SPARSE_MAPPING 424215013Smdf ef->object = 0; 42559603Sdfr#endif 426215013Smdf ef->dynamic = dp; 42759603Sdfr 428215013Smdf if (dp != NULL) 429215013Smdf parse_dynamic(ef); 430279119Snwhitehorn linker_kernel_file->address += KERNBASE; 431215013Smdf linker_kernel_file->size = -(intptr_t)linker_kernel_file->address; 43240156Speter 433215013Smdf if (modptr != NULL) { 434215013Smdf ef->modptr = modptr; 435215013Smdf baseptr = preload_search_info(modptr, MODINFO_ADDR); 436215013Smdf if (baseptr != NULL) 437215013Smdf linker_kernel_file->address = *(caddr_t *)baseptr; 438215013Smdf sizeptr = preload_search_info(modptr, MODINFO_SIZE); 439215013Smdf if (sizeptr != NULL) 440215013Smdf linker_kernel_file->size = *(size_t *)sizeptr; 441273334Smarcel ctors_addrp = (Elf_Addr *)preload_search_info(modptr, 442273334Smarcel MODINFO_METADATA | MODINFOMD_CTORS_ADDR); 443273334Smarcel ctors_sizep = (Elf_Size *)preload_search_info(modptr, 444273334Smarcel MODINFO_METADATA | MODINFOMD_CTORS_SIZE); 445273334Smarcel if (ctors_addrp != NULL && ctors_sizep != NULL) { 446292641Sngie linker_kernel_file->ctors_addr = ef->address + 447273334Smarcel *ctors_addrp; 448273334Smarcel linker_kernel_file->ctors_size = *ctors_sizep; 449273334Smarcel } 450215013Smdf } 451215013Smdf (void)link_elf_preload_parse_symbols(ef); 45259603Sdfr 453131928Smarcel#ifdef GDB 454215013Smdf r_debug.r_map = NULL; 455215013Smdf r_debug.r_brk = r_debug_state; 456215013Smdf r_debug.r_state = RT_CONSISTENT; 457105468Smarcel#endif 45859603Sdfr 459215013Smdf (void)link_elf_link_common_finish(linker_kernel_file); 460215013Smdf linker_kernel_file->flags |= LINKER_FILE_LINKED; 461240997Strociny TAILQ_INIT(&set_pcpu_list); 462240997Strociny#ifdef VIMAGE 463240997Strociny TAILQ_INIT(&set_vnet_list); 464240997Strociny#endif 46538514Sdfr} 46638514Sdfr 467359652ShselaskySYSINIT(link_elf, SI_SUB_KLD, SI_ORDER_THIRD, link_elf_init, NULL); 46838514Sdfr 46938514Sdfrstatic int 47059751Speterlink_elf_preload_parse_symbols(elf_file_t ef) 47140254Speter{ 472215013Smdf caddr_t pointer; 473215013Smdf caddr_t ssym, esym, base; 474215013Smdf caddr_t strtab; 475215013Smdf int strcnt; 476215013Smdf Elf_Sym *symtab; 477215013Smdf int symcnt; 47840254Speter 479215013Smdf if (ef->modptr == NULL) 480215013Smdf return (0); 481215013Smdf pointer = preload_search_info(ef->modptr, 482215013Smdf MODINFO_METADATA | MODINFOMD_SSYM); 483215013Smdf if (pointer == NULL) 484215013Smdf return (0); 485215013Smdf ssym = *(caddr_t *)pointer; 486215013Smdf pointer = preload_search_info(ef->modptr, 487215013Smdf MODINFO_METADATA | MODINFOMD_ESYM); 488215013Smdf if (pointer == NULL) 489215013Smdf return (0); 490215013Smdf esym = *(caddr_t *)pointer; 49140254Speter 492215013Smdf base = ssym; 49340254Speter 494215013Smdf symcnt = *(long *)base; 495215013Smdf base += sizeof(long); 496215013Smdf symtab = (Elf_Sym *)base; 497215013Smdf base += roundup(symcnt, sizeof(long)); 49840254Speter 499215013Smdf if (base > esym || base < ssym) { 500215013Smdf printf("Symbols are corrupt!\n"); 501215013Smdf return (EINVAL); 502215013Smdf } 50340254Speter 504215013Smdf strcnt = *(long *)base; 505215013Smdf base += sizeof(long); 506215013Smdf strtab = base; 507215013Smdf base += roundup(strcnt, sizeof(long)); 50840254Speter 509215013Smdf if (base > esym || base < ssym) { 510215013Smdf printf("Symbols are corrupt!\n"); 511215013Smdf return (EINVAL); 512215013Smdf } 51340254Speter 514215013Smdf ef->ddbsymtab = symtab; 515215013Smdf ef->ddbsymcnt = symcnt / sizeof(Elf_Sym); 516215013Smdf ef->ddbstrtab = strtab; 517215013Smdf ef->ddbstrcnt = strcnt; 51840254Speter 519215013Smdf return (0); 52040254Speter} 52140254Speter 52240254Speterstatic int 52359603Sdfrparse_dynamic(elf_file_t ef) 52438514Sdfr{ 525215013Smdf Elf_Dyn *dp; 526215013Smdf int plttype = DT_REL; 52738514Sdfr 528215013Smdf for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) { 529215013Smdf switch (dp->d_tag) { 530215013Smdf case DT_HASH: 531215013Smdf { 532215013Smdf /* From src/libexec/rtld-elf/rtld.c */ 533215013Smdf const Elf_Hashelt *hashtab = (const Elf_Hashelt *) 534215013Smdf (ef->address + dp->d_un.d_ptr); 535215013Smdf ef->nbuckets = hashtab[0]; 536215013Smdf ef->nchains = hashtab[1]; 537215013Smdf ef->buckets = hashtab + 2; 538215013Smdf ef->chains = ef->buckets + ef->nbuckets; 539215013Smdf break; 540215013Smdf } 541215013Smdf case DT_STRTAB: 542215013Smdf ef->strtab = (caddr_t) (ef->address + dp->d_un.d_ptr); 543215013Smdf break; 544215013Smdf case DT_STRSZ: 545215013Smdf ef->strsz = dp->d_un.d_val; 546215013Smdf break; 547215013Smdf case DT_SYMTAB: 548215013Smdf ef->symtab = (Elf_Sym*) (ef->address + dp->d_un.d_ptr); 549215013Smdf break; 550215013Smdf case DT_SYMENT: 551215013Smdf if (dp->d_un.d_val != sizeof(Elf_Sym)) 552215013Smdf return (ENOEXEC); 553215013Smdf break; 554215013Smdf case DT_PLTGOT: 555215013Smdf ef->got = (Elf_Addr *) (ef->address + dp->d_un.d_ptr); 556215013Smdf break; 557215013Smdf case DT_REL: 558215013Smdf ef->rel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr); 559215013Smdf break; 560215013Smdf case DT_RELSZ: 561215013Smdf ef->relsize = dp->d_un.d_val; 562215013Smdf break; 563215013Smdf case DT_RELENT: 564215013Smdf if (dp->d_un.d_val != sizeof(Elf_Rel)) 565215013Smdf return (ENOEXEC); 566215013Smdf break; 567215013Smdf case DT_JMPREL: 568215013Smdf ef->pltrel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr); 569215013Smdf break; 570215013Smdf case DT_PLTRELSZ: 571215013Smdf ef->pltrelsize = dp->d_un.d_val; 572215013Smdf break; 573215013Smdf case DT_RELA: 574215013Smdf ef->rela = (const Elf_Rela *) (ef->address + dp->d_un.d_ptr); 575215013Smdf break; 576215013Smdf case DT_RELASZ: 577215013Smdf ef->relasize = dp->d_un.d_val; 578215013Smdf break; 579215013Smdf case DT_RELAENT: 580215013Smdf if (dp->d_un.d_val != sizeof(Elf_Rela)) 581215013Smdf return (ENOEXEC); 582215013Smdf break; 583215013Smdf case DT_PLTREL: 584215013Smdf plttype = dp->d_un.d_val; 585215013Smdf if (plttype != DT_REL && plttype != DT_RELA) 586215013Smdf return (ENOEXEC); 587215013Smdf break; 588131928Smarcel#ifdef GDB 589215013Smdf case DT_DEBUG: 590215013Smdf dp->d_un.d_ptr = (Elf_Addr)&r_debug; 591215013Smdf break; 59259603Sdfr#endif 593215013Smdf } 59438514Sdfr } 59539071Sdfr 596215013Smdf if (plttype == DT_RELA) { 597215013Smdf ef->pltrela = (const Elf_Rela *)ef->pltrel; 598215013Smdf ef->pltrel = NULL; 599215013Smdf ef->pltrelasize = ef->pltrelsize; 600215013Smdf ef->pltrelsize = 0; 601215013Smdf } 60239071Sdfr 603215013Smdf ef->ddbsymtab = ef->symtab; 604215013Smdf ef->ddbsymcnt = ef->nchains; 605215013Smdf ef->ddbstrtab = ef->strtab; 606215013Smdf ef->ddbstrcnt = ef->strsz; 60740254Speter 608215013Smdf return (0); 60938514Sdfr} 61038514Sdfr 61138514Sdfrstatic int 612194784Sjeffparse_dpcpu(elf_file_t ef) 613292640Sngie{ 614340053Sbz int error, size; 615194784Sjeff 616215013Smdf ef->pcpu_start = 0; 617215013Smdf ef->pcpu_stop = 0; 618215013Smdf error = link_elf_lookup_set(&ef->lf, "pcpu", (void ***)&ef->pcpu_start, 619340053Sbz (void ***)&ef->pcpu_stop, NULL); 620215013Smdf /* Error just means there is no pcpu set to relocate. */ 621215013Smdf if (error != 0) 622215013Smdf return (0); 623340053Sbz size = (uintptr_t)ef->pcpu_stop - (uintptr_t)ef->pcpu_start; 624340053Sbz /* Empty set? */ 625340053Sbz if (size < 1) 626340053Sbz return (0); 627215013Smdf /* 628215013Smdf * Allocate space in the primary pcpu area. Copy in our 629215013Smdf * initialization from the data section and then initialize 630215013Smdf * all per-cpu storage from that. 631215013Smdf */ 632340053Sbz ef->pcpu_base = (Elf_Addr)(uintptr_t)dpcpu_alloc(size); 633340054Sbz if (ef->pcpu_base == 0) { 634340054Sbz printf("%s: pcpu module space is out of space; " 635340054Sbz "cannot allocate %d for %s\n", 636340054Sbz __func__, size, ef->lf.pathname); 637215013Smdf return (ENOSPC); 638340054Sbz } 639340053Sbz memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, size); 640340053Sbz dpcpu_copy((void *)ef->pcpu_base, size); 641240997Strociny elf_set_add(&set_pcpu_list, ef->pcpu_start, ef->pcpu_stop, 642240997Strociny ef->pcpu_base); 643194784Sjeff 644215013Smdf return (0); 645194784Sjeff} 646194784Sjeff 647195699Srwatson#ifdef VIMAGE 648194784Sjeffstatic int 649195699Srwatsonparse_vnet(elf_file_t ef) 650292640Sngie{ 651340053Sbz int error, size; 652195699Srwatson 653215013Smdf ef->vnet_start = 0; 654215013Smdf ef->vnet_stop = 0; 655215013Smdf error = link_elf_lookup_set(&ef->lf, "vnet", (void ***)&ef->vnet_start, 656340053Sbz (void ***)&ef->vnet_stop, NULL); 657215013Smdf /* Error just means there is no vnet data set to relocate. */ 658215013Smdf if (error != 0) 659215013Smdf return (0); 660340053Sbz size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start; 661340053Sbz /* Empty set? */ 662340053Sbz if (size < 1) 663340053Sbz return (0); 664215013Smdf /* 665215013Smdf * Allocate space in the primary vnet area. Copy in our 666215013Smdf * initialization from the data section and then initialize 667215013Smdf * all per-vnet storage from that. 668215013Smdf */ 669340053Sbz ef->vnet_base = (Elf_Addr)(uintptr_t)vnet_data_alloc(size); 670340054Sbz if (ef->vnet_base == 0) { 671340054Sbz printf("%s: vnet module space is out of space; " 672340054Sbz "cannot allocate %d for %s\n", 673340054Sbz __func__, size, ef->lf.pathname); 674215013Smdf return (ENOSPC); 675340054Sbz } 676340053Sbz memcpy((void *)ef->vnet_base, (void *)ef->vnet_start, size); 677340053Sbz vnet_data_copy((void *)ef->vnet_base, size); 678240997Strociny elf_set_add(&set_vnet_list, ef->vnet_start, ef->vnet_stop, 679240997Strociny ef->vnet_base); 680195699Srwatson 681215013Smdf return (0); 682195699Srwatson} 683195699Srwatson#endif 684195699Srwatson 685195699Srwatsonstatic int 68659751Speterlink_elf_link_preload(linker_class_t cls, 687215013Smdf const char* filename, linker_file_t *result) 68840156Speter{ 689273334Smarcel Elf_Addr *ctors_addrp; 690273334Smarcel Elf_Size *ctors_sizep; 691215013Smdf caddr_t modptr, baseptr, sizeptr, dynptr; 692215013Smdf char *type; 693215013Smdf elf_file_t ef; 694215013Smdf linker_file_t lf; 695215013Smdf int error; 696215013Smdf vm_offset_t dp; 69740156Speter 698215013Smdf /* Look to see if we have the file preloaded */ 699215013Smdf modptr = preload_search_by_name(filename); 700215013Smdf if (modptr == NULL) 701215013Smdf return (ENOENT); 70240156Speter 703215013Smdf type = (char *)preload_search_info(modptr, MODINFO_TYPE); 704215013Smdf baseptr = preload_search_info(modptr, MODINFO_ADDR); 705215013Smdf sizeptr = preload_search_info(modptr, MODINFO_SIZE); 706215013Smdf dynptr = preload_search_info(modptr, 707215013Smdf MODINFO_METADATA | MODINFOMD_DYNAMIC); 708215013Smdf if (type == NULL || 709215013Smdf (strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE) " module") != 0 && 710215013Smdf strcmp(type, "elf module") != 0)) 711215013Smdf return (EFTYPE); 712215013Smdf if (baseptr == NULL || sizeptr == NULL || dynptr == NULL) 713215013Smdf return (EINVAL); 71440156Speter 715215013Smdf lf = linker_make_file(filename, &link_elf_class); 716215013Smdf if (lf == NULL) 717215013Smdf return (ENOMEM); 71859603Sdfr 719215013Smdf ef = (elf_file_t) lf; 720215013Smdf ef->preloaded = 1; 721215013Smdf ef->modptr = modptr; 722215013Smdf ef->address = *(caddr_t *)baseptr; 72340156Speter#ifdef SPARSE_MAPPING 724215013Smdf ef->object = 0; 72540156Speter#endif 726215013Smdf dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr; 727215013Smdf ef->dynamic = (Elf_Dyn *)dp; 728215013Smdf lf->address = ef->address; 729215013Smdf lf->size = *(size_t *)sizeptr; 73040156Speter 731273334Smarcel ctors_addrp = (Elf_Addr *)preload_search_info(modptr, 732273334Smarcel MODINFO_METADATA | MODINFOMD_CTORS_ADDR); 733273334Smarcel ctors_sizep = (Elf_Size *)preload_search_info(modptr, 734273334Smarcel MODINFO_METADATA | MODINFOMD_CTORS_SIZE); 735273334Smarcel if (ctors_addrp != NULL && ctors_sizep != NULL) { 736273334Smarcel lf->ctors_addr = ef->address + *ctors_addrp; 737273334Smarcel lf->ctors_size = *ctors_sizep; 738273334Smarcel } 739273334Smarcel 740215013Smdf error = parse_dynamic(ef); 741215013Smdf if (error == 0) 742215013Smdf error = parse_dpcpu(ef); 743195699Srwatson#ifdef VIMAGE 744215013Smdf if (error == 0) 745215013Smdf error = parse_vnet(ef); 746195699Srwatson#endif 747215013Smdf if (error != 0) { 748215013Smdf linker_file_unload(lf, LINKER_UNLOAD_FORCE); 749215013Smdf return (error); 750215013Smdf } 751215013Smdf link_elf_reloc_local(lf); 752215013Smdf *result = lf; 753215013Smdf return (0); 75459751Speter} 75559751Speter 75659751Speterstatic int 75759751Speterlink_elf_link_preload_finish(linker_file_t lf) 75859751Speter{ 759215013Smdf elf_file_t ef; 760215013Smdf int error; 76159751Speter 762215013Smdf ef = (elf_file_t) lf; 763215013Smdf error = relocate_file(ef); 764215013Smdf if (error != 0) 765215013Smdf return (error); 766215013Smdf (void)link_elf_preload_parse_symbols(ef); 76759603Sdfr 768215013Smdf return (link_elf_link_common_finish(lf)); 76940156Speter} 77040156Speter 77140156Speterstatic int 772105468Smarcellink_elf_load_file(linker_class_t cls, const char* filename, 773215013Smdf linker_file_t* result) 77438514Sdfr{ 775215013Smdf struct nameidata nd; 776215013Smdf struct thread* td = curthread; /* XXX */ 777215013Smdf Elf_Ehdr *hdr; 778215013Smdf caddr_t firstpage; 779215013Smdf int nbytes, i; 780215013Smdf Elf_Phdr *phdr; 781215013Smdf Elf_Phdr *phlimit; 782215013Smdf Elf_Phdr *segs[MAXSEGS]; 783215013Smdf int nsegs; 784215013Smdf Elf_Phdr *phdyn; 785215013Smdf Elf_Phdr *phphdr; 786215013Smdf caddr_t mapbase; 787215013Smdf size_t mapsize; 788215013Smdf Elf_Off base_offset; 789215013Smdf Elf_Addr base_vaddr; 790215013Smdf Elf_Addr base_vlimit; 791215013Smdf int error = 0; 792231949Skib ssize_t resid; 793231949Skib int flags; 794215013Smdf elf_file_t ef; 795215013Smdf linker_file_t lf; 796215013Smdf Elf_Shdr *shdr; 797215013Smdf int symtabindex; 798215013Smdf int symstrindex; 799273334Smarcel int shstrindex; 800215013Smdf int symcnt; 801215013Smdf int strcnt; 802273334Smarcel char *shstrs; 80338514Sdfr 804215013Smdf shdr = NULL; 805215013Smdf lf = NULL; 806273334Smarcel shstrs = NULL; 80740292Speter 808241896Skib NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td); 809215013Smdf flags = FREAD; 810215013Smdf error = vn_open(&nd, &flags, 0, NULL); 811215013Smdf if (error != 0) 812215013Smdf return (error); 813215013Smdf NDFREE(&nd, NDF_ONLY_PNBUF); 814215013Smdf if (nd.ni_vp->v_type != VREG) { 815215013Smdf error = ENOEXEC; 816215013Smdf firstpage = NULL; 817215013Smdf goto out; 818215013Smdf } 819107089Srwatson#ifdef MAC 820215013Smdf error = mac_kld_check_load(curthread->td_ucred, nd.ni_vp); 821215013Smdf if (error != 0) { 822215013Smdf firstpage = NULL; 823215013Smdf goto out; 824215013Smdf } 825107089Srwatson#endif 82638514Sdfr 827215013Smdf /* 828215013Smdf * Read the elf header from the file. 829215013Smdf */ 830215013Smdf firstpage = malloc(PAGE_SIZE, M_LINKER, M_WAITOK); 831215013Smdf hdr = (Elf_Ehdr *)firstpage; 832215013Smdf error = vn_rdwr(UIO_READ, nd.ni_vp, firstpage, PAGE_SIZE, 0, 833215013Smdf UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 834215013Smdf &resid, td); 835215013Smdf nbytes = PAGE_SIZE - resid; 836215013Smdf if (error != 0) 837215013Smdf goto out; 83838514Sdfr 839215013Smdf if (!IS_ELF(*hdr)) { 840215013Smdf error = ENOEXEC; 841215013Smdf goto out; 842215013Smdf } 84338514Sdfr 844215013Smdf if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || 845215013Smdf hdr->e_ident[EI_DATA] != ELF_TARG_DATA) { 846215013Smdf link_elf_error(filename, "Unsupported file layout"); 847215013Smdf error = ENOEXEC; 848215013Smdf goto out; 849215013Smdf } 850215013Smdf if (hdr->e_ident[EI_VERSION] != EV_CURRENT || 851215013Smdf hdr->e_version != EV_CURRENT) { 852215013Smdf link_elf_error(filename, "Unsupported file version"); 853215013Smdf error = ENOEXEC; 854215013Smdf goto out; 855215013Smdf } 856215013Smdf if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) { 857215013Smdf error = ENOSYS; 858215013Smdf goto out; 859215013Smdf } 860215013Smdf if (hdr->e_machine != ELF_TARG_MACH) { 861215013Smdf link_elf_error(filename, "Unsupported machine"); 862215013Smdf error = ENOEXEC; 863215013Smdf goto out; 864215013Smdf } 86539071Sdfr 866215013Smdf /* 867215013Smdf * We rely on the program header being in the first page. 868215013Smdf * This is not strictly required by the ABI specification, but 869215013Smdf * it seems to always true in practice. And, it simplifies 870215013Smdf * things considerably. 871215013Smdf */ 872215013Smdf if (!((hdr->e_phentsize == sizeof(Elf_Phdr)) && 873215013Smdf (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= PAGE_SIZE) && 874215013Smdf (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= nbytes))) 875215013Smdf link_elf_error(filename, "Unreadable program headers"); 87639071Sdfr 877215013Smdf /* 878215013Smdf * Scan the program header entries, and save key information. 879215013Smdf * 880215013Smdf * We rely on there being exactly two load segments, text and data, 881215013Smdf * in that order. 882215013Smdf */ 883215013Smdf phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff); 884215013Smdf phlimit = phdr + hdr->e_phnum; 885215013Smdf nsegs = 0; 886215013Smdf phdyn = NULL; 887215013Smdf phphdr = NULL; 888215013Smdf while (phdr < phlimit) { 889215013Smdf switch (phdr->p_type) { 890215013Smdf case PT_LOAD: 891215013Smdf if (nsegs == MAXSEGS) { 892215013Smdf link_elf_error(filename, "Too many sections"); 893215013Smdf error = ENOEXEC; 894215013Smdf goto out; 895215013Smdf } 896215013Smdf /* 897215013Smdf * XXX: We just trust they come in right order ?? 898215013Smdf */ 899215013Smdf segs[nsegs] = phdr; 900215013Smdf ++nsegs; 901215013Smdf break; 90239071Sdfr 903215013Smdf case PT_PHDR: 904215013Smdf phphdr = phdr; 905215013Smdf break; 90639071Sdfr 907215013Smdf case PT_DYNAMIC: 908215013Smdf phdyn = phdr; 909215013Smdf break; 91039071Sdfr 911215013Smdf case PT_INTERP: 912215013Smdf error = ENOSYS; 913215013Smdf goto out; 914215013Smdf } 91565503Sbp 916215013Smdf ++phdr; 91739071Sdfr } 918215013Smdf if (phdyn == NULL) { 919215013Smdf link_elf_error(filename, "Object is not dynamically-linked"); 920215013Smdf error = ENOEXEC; 921215013Smdf goto out; 922215013Smdf } 923215013Smdf if (nsegs == 0) { 924215013Smdf link_elf_error(filename, "No sections"); 925215013Smdf error = ENOEXEC; 926215013Smdf goto out; 927215013Smdf } 92839071Sdfr 929215013Smdf /* 930215013Smdf * Allocate the entire address space of the object, to stake 931215013Smdf * out our contiguous region, and to establish the base 932215013Smdf * address for relocation. 933215013Smdf */ 934215013Smdf base_offset = trunc_page(segs[0]->p_offset); 935215013Smdf base_vaddr = trunc_page(segs[0]->p_vaddr); 936292640Sngie base_vlimit = round_page(segs[nsegs - 1]->p_vaddr + 937215013Smdf segs[nsegs - 1]->p_memsz); 938215013Smdf mapsize = base_vlimit - base_vaddr; 93938514Sdfr 940215013Smdf lf = linker_make_file(filename, &link_elf_class); 941215013Smdf if (lf == NULL) { 942215013Smdf error = ENOMEM; 943215013Smdf goto out; 944215013Smdf } 94539071Sdfr 946215013Smdf ef = (elf_file_t) lf; 94739071Sdfr#ifdef SPARSE_MAPPING 948215013Smdf ef->object = vm_object_allocate(OBJT_DEFAULT, mapsize >> PAGE_SHIFT); 949215013Smdf if (ef->object == NULL) { 950215013Smdf error = ENOMEM; 951215013Smdf goto out; 952215013Smdf } 953215013Smdf ef->address = (caddr_t) vm_map_min(kernel_map); 954215013Smdf error = vm_map_find(kernel_map, ef->object, 0, 955255426Sjhb (vm_offset_t *) &ef->address, mapsize, 0, VMFS_OPTIMAL_SPACE, 956215013Smdf VM_PROT_ALL, VM_PROT_ALL, 0); 957215013Smdf if (error != 0) { 958215013Smdf vm_object_deallocate(ef->object); 959215013Smdf ef->object = 0; 960215013Smdf goto out; 961215013Smdf } 96239071Sdfr#else 963215013Smdf ef->address = malloc(mapsize, M_LINKER, M_WAITOK); 96439071Sdfr#endif 965215013Smdf mapbase = ef->address; 96638514Sdfr 967215013Smdf /* 968215013Smdf * Read the text and data sections and zero the bss. 969215013Smdf */ 970215013Smdf for (i = 0; i < nsegs; i++) { 971215013Smdf caddr_t segbase = mapbase + segs[i]->p_vaddr - base_vaddr; 972215013Smdf error = vn_rdwr(UIO_READ, nd.ni_vp, 973215013Smdf segbase, segs[i]->p_filesz, segs[i]->p_offset, 974215013Smdf UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 975215013Smdf &resid, td); 976215013Smdf if (error != 0) 977215013Smdf goto out; 978215013Smdf bzero(segbase + segs[i]->p_filesz, 979215013Smdf segs[i]->p_memsz - segs[i]->p_filesz); 980149539Salc 981149539Salc#ifdef SPARSE_MAPPING 982215013Smdf /* 983215013Smdf * Wire down the pages 984215013Smdf */ 985215013Smdf error = vm_map_wire(kernel_map, 986149539Salc (vm_offset_t) segbase, 987149539Salc (vm_offset_t) segbase + segs[i]->p_memsz, 988149539Salc VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES); 989215013Smdf if (error != KERN_SUCCESS) { 990215013Smdf error = ENOMEM; 991215013Smdf goto out; 992215013Smdf } 993215013Smdf#endif 994149544Salc } 99539071Sdfr 99685734Sgreen#ifdef GPROF 997215013Smdf /* Update profiling information with the new text segment. */ 998215013Smdf mtx_lock(&Giant); 999292641Sngie kmupetext((uintfptr_t)(mapbase + segs[0]->p_vaddr - base_vaddr + 1000215013Smdf segs[0]->p_memsz)); 1001215013Smdf mtx_unlock(&Giant); 100285734Sgreen#endif 100385734Sgreen 1004215013Smdf ef->dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr); 100539071Sdfr 1006215013Smdf lf->address = ef->address; 1007215013Smdf lf->size = mapsize; 100838514Sdfr 1009215013Smdf error = parse_dynamic(ef); 1010215013Smdf if (error != 0) 1011215013Smdf goto out; 1012215013Smdf error = parse_dpcpu(ef); 1013215013Smdf if (error != 0) 1014215013Smdf goto out; 1015195699Srwatson#ifdef VIMAGE 1016215013Smdf error = parse_vnet(ef); 1017215013Smdf if (error != 0) 1018215013Smdf goto out; 1019195699Srwatson#endif 1020215013Smdf link_elf_reloc_local(lf); 1021109605Sjake 1022215013Smdf VOP_UNLOCK(nd.ni_vp, 0); 1023215013Smdf error = linker_load_dependencies(lf); 1024215013Smdf vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY); 1025215013Smdf if (error != 0) 1026215013Smdf goto out; 1027215013Smdf error = relocate_file(ef); 1028215013Smdf if (error != 0) 1029215013Smdf goto out; 103040292Speter 1031215013Smdf /* 1032215013Smdf * Try and load the symbol table if it's present. (you can 1033215013Smdf * strip it!) 1034215013Smdf */ 1035215013Smdf nbytes = hdr->e_shnum * hdr->e_shentsize; 1036215013Smdf if (nbytes == 0 || hdr->e_shoff == 0) 1037215013Smdf goto nosyms; 1038215013Smdf shdr = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO); 1039215013Smdf error = vn_rdwr(UIO_READ, nd.ni_vp, 1040215013Smdf (caddr_t)shdr, nbytes, hdr->e_shoff, 1041215013Smdf UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1042215013Smdf &resid, td); 1043215013Smdf if (error != 0) 1044215013Smdf goto out; 1045273334Smarcel 1046273334Smarcel /* Read section string table */ 1047273334Smarcel shstrindex = hdr->e_shstrndx; 1048273334Smarcel if (shstrindex != 0 && shdr[shstrindex].sh_type == SHT_STRTAB && 1049273334Smarcel shdr[shstrindex].sh_size != 0) { 1050273334Smarcel nbytes = shdr[shstrindex].sh_size; 1051273334Smarcel shstrs = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO); 1052273334Smarcel error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shstrs, nbytes, 1053273334Smarcel shdr[shstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, 1054273334Smarcel td->td_ucred, NOCRED, &resid, td); 1055273334Smarcel if (error) 1056273334Smarcel goto out; 1057273334Smarcel } 1058273334Smarcel 1059215013Smdf symtabindex = -1; 1060215013Smdf symstrindex = -1; 1061215013Smdf for (i = 0; i < hdr->e_shnum; i++) { 1062215013Smdf if (shdr[i].sh_type == SHT_SYMTAB) { 1063215013Smdf symtabindex = i; 1064215013Smdf symstrindex = shdr[i].sh_link; 1065273334Smarcel } else if (shstrs != NULL && shdr[i].sh_name != 0 && 1066273334Smarcel strcmp(shstrs + shdr[i].sh_name, ".ctors") == 0) { 1067273334Smarcel /* Record relocated address and size of .ctors. */ 1068273334Smarcel lf->ctors_addr = mapbase + shdr[i].sh_addr - base_vaddr; 1069273334Smarcel lf->ctors_size = shdr[i].sh_size; 1070215013Smdf } 1071215013Smdf } 1072215013Smdf if (symtabindex < 0 || symstrindex < 0) 1073215013Smdf goto nosyms; 107440156Speter 1075215013Smdf symcnt = shdr[symtabindex].sh_size; 1076215013Smdf ef->symbase = malloc(symcnt, M_LINKER, M_WAITOK); 1077215013Smdf strcnt = shdr[symstrindex].sh_size; 1078215013Smdf ef->strbase = malloc(strcnt, M_LINKER, M_WAITOK); 107940292Speter 1080215013Smdf error = vn_rdwr(UIO_READ, nd.ni_vp, 1081215013Smdf ef->symbase, symcnt, shdr[symtabindex].sh_offset, 1082215013Smdf UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1083215013Smdf &resid, td); 1084215013Smdf if (error != 0) 1085215013Smdf goto out; 1086215013Smdf error = vn_rdwr(UIO_READ, nd.ni_vp, 1087215013Smdf ef->strbase, strcnt, shdr[symstrindex].sh_offset, 1088215013Smdf UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1089215013Smdf &resid, td); 1090215013Smdf if (error != 0) 1091215013Smdf goto out; 109240292Speter 1093215013Smdf ef->ddbsymcnt = symcnt / sizeof(Elf_Sym); 1094215013Smdf ef->ddbsymtab = (const Elf_Sym *)ef->symbase; 1095215013Smdf ef->ddbstrcnt = strcnt; 1096215013Smdf ef->ddbstrtab = ef->strbase; 109740292Speter 1098223155Smarcelnosyms: 1099215013Smdf error = link_elf_link_common_finish(lf); 1100215013Smdf if (error != 0) 1101215013Smdf goto out; 110259603Sdfr 1103215013Smdf *result = lf; 110440292Speter 110538514Sdfrout: 1106215013Smdf VOP_UNLOCK(nd.ni_vp, 0); 1107215013Smdf vn_close(nd.ni_vp, FREAD, td->td_ucred, td); 1108215013Smdf if (error != 0 && lf != NULL) 1109215013Smdf linker_file_unload(lf, LINKER_UNLOAD_FORCE); 1110295489Skib free(shdr, M_LINKER); 1111295489Skib free(firstpage, M_LINKER); 1112295489Skib free(shstrs, M_LINKER); 111338514Sdfr 1114215013Smdf return (error); 111538514Sdfr} 111638514Sdfr 1117194784SjeffElf_Addr 1118194784Sjeffelf_relocaddr(linker_file_t lf, Elf_Addr x) 1119194784Sjeff{ 1120215013Smdf elf_file_t ef; 1121194784Sjeff 1122215013Smdf ef = (elf_file_t)lf; 1123215013Smdf if (x >= ef->pcpu_start && x < ef->pcpu_stop) 1124215013Smdf return ((x - ef->pcpu_start) + ef->pcpu_base); 1125195699Srwatson#ifdef VIMAGE 1126215013Smdf if (x >= ef->vnet_start && x < ef->vnet_stop) 1127215013Smdf return ((x - ef->vnet_start) + ef->vnet_base); 1128195699Srwatson#endif 1129215013Smdf return (x); 1130194784Sjeff} 1131194784Sjeff 1132194784Sjeff 113338514Sdfrstatic void 113440156Speterlink_elf_unload_file(linker_file_t file) 113538514Sdfr{ 1136215013Smdf elf_file_t ef = (elf_file_t) file; 113738514Sdfr 1138215013Smdf if (ef->pcpu_base != 0) { 1139215013Smdf dpcpu_free((void *)ef->pcpu_base, 1140215013Smdf ef->pcpu_stop - ef->pcpu_start); 1141240997Strociny elf_set_delete(&set_pcpu_list, ef->pcpu_start); 1142215013Smdf } 1143195699Srwatson#ifdef VIMAGE 1144215013Smdf if (ef->vnet_base != 0) { 1145215013Smdf vnet_data_free((void *)ef->vnet_base, 1146215013Smdf ef->vnet_stop - ef->vnet_start); 1147240997Strociny elf_set_delete(&set_vnet_list, ef->vnet_start); 1148215013Smdf } 1149195699Srwatson#endif 1150131928Smarcel#ifdef GDB 1151215013Smdf if (ef->gdb.l_ld != NULL) { 1152215013Smdf GDB_STATE(RT_DELETE); 1153215013Smdf free((void *)(uintptr_t)ef->gdb.l_name, M_LINKER); 1154215013Smdf link_elf_delete_gdb(&ef->gdb); 1155215013Smdf GDB_STATE(RT_CONSISTENT); 1156215013Smdf } 115759603Sdfr#endif 115859603Sdfr 1159215013Smdf /* Notify MD code that a module is being unloaded. */ 1160215013Smdf elf_cpu_unload_file(file); 1161105469Smarcel 1162215013Smdf if (ef->preloaded) { 1163215013Smdf link_elf_unload_preload(file); 1164215013Smdf return; 1165215013Smdf } 1166105468Smarcel 116739071Sdfr#ifdef SPARSE_MAPPING 1168215013Smdf if (ef->object != NULL) { 1169215013Smdf vm_map_remove(kernel_map, (vm_offset_t) ef->address, 1170215013Smdf (vm_offset_t) ef->address 1171215013Smdf + (ef->object->size << PAGE_SHIFT)); 1172215013Smdf } 117339071Sdfr#else 1174295489Skib free(ef->address, M_LINKER); 117539071Sdfr#endif 1176295489Skib free(ef->symbase, M_LINKER); 1177295489Skib free(ef->strbase, M_LINKER); 1178295489Skib free(ef->ctftab, M_LINKER); 1179295489Skib free(ef->ctfoff, M_LINKER); 1180295489Skib free(ef->typoff, M_LINKER); 118138514Sdfr} 118238514Sdfr 118340156Speterstatic void 118459751Speterlink_elf_unload_preload(linker_file_t file) 118540156Speter{ 1186336749Smarkj if (file->pathname != NULL) 1187336749Smarkj preload_delete_name(file->pathname); 118840156Speter} 118940156Speter 119039071Sdfrstatic const char * 1191153504Smarcelsymbol_name(elf_file_t ef, Elf_Size r_info) 119238514Sdfr{ 1193215013Smdf const Elf_Sym *ref; 119438514Sdfr 1195215013Smdf if (ELF_R_SYM(r_info)) { 1196215013Smdf ref = ef->symtab + ELF_R_SYM(r_info); 1197215013Smdf return (ef->strtab + ref->st_name); 1198215013Smdf } 1199215013Smdf return (NULL); 120038514Sdfr} 120138514Sdfr 120238514Sdfrstatic int 1203338867Smarkjsymbol_type(elf_file_t ef, Elf_Size r_info) 120438514Sdfr{ 1205338867Smarkj const Elf_Sym *ref; 1206338867Smarkj 1207338867Smarkj if (ELF_R_SYM(r_info)) { 1208338867Smarkj ref = ef->symtab + ELF_R_SYM(r_info); 1209338867Smarkj return (ELF_ST_TYPE(ref->st_info)); 1210338867Smarkj } 1211338867Smarkj return (STT_NOTYPE); 1212338867Smarkj} 1213338867Smarkj 1214338867Smarkjstatic int 1215338867Smarkjrelocate_file1(elf_file_t ef, elf_lookup_fn lookup, elf_reloc_fn reloc, 1216338867Smarkj bool ifuncs) 1217338867Smarkj{ 1218215013Smdf const Elf_Rel *rel; 1219215013Smdf const Elf_Rela *rela; 1220215013Smdf const char *symname; 122138514Sdfr 1222338867Smarkj#define APPLY_RELOCS(iter, tbl, tblsize, type) do { \ 1223338867Smarkj for ((iter) = (tbl); (iter) != NULL && \ 1224338867Smarkj (iter) < (tbl) + (tblsize) / sizeof(*(iter)); (iter)++) { \ 1225338867Smarkj if ((symbol_type(ef, (iter)->r_info) == \ 1226338867Smarkj STT_GNU_IFUNC || \ 1227338867Smarkj elf_is_ifunc_reloc((iter)->r_info)) != ifuncs) \ 1228338867Smarkj continue; \ 1229338867Smarkj if (reloc(&ef->lf, (Elf_Addr)ef->address, \ 1230338867Smarkj (iter), (type), lookup)) { \ 1231338867Smarkj symname = symbol_name(ef, (iter)->r_info); \ 1232338867Smarkj printf("link_elf: symbol %s undefined\n", \ 1233338867Smarkj symname); \ 1234338867Smarkj return (ENOENT); \ 1235338867Smarkj } \ 1236338867Smarkj } \ 1237338867Smarkj} while (0) 123838514Sdfr 1239338867Smarkj APPLY_RELOCS(rel, ef->rel, ef->relsize, ELF_RELOC_REL); 1240338867Smarkj APPLY_RELOCS(rela, ef->rela, ef->relasize, ELF_RELOC_RELA); 1241338867Smarkj APPLY_RELOCS(rel, ef->pltrel, ef->pltrelsize, ELF_RELOC_REL); 1242338867Smarkj APPLY_RELOCS(rela, ef->pltrela, ef->pltrelasize, ELF_RELOC_RELA); 124338514Sdfr 1244338867Smarkj#undef APPLY_RELOCS 124538514Sdfr 1246215013Smdf return (0); 124738514Sdfr} 124838514Sdfr 1249333720Skibstatic int 1250333720Skibrelocate_file(elf_file_t ef) 1251333720Skib{ 1252338867Smarkj int error; 1253333720Skib 1254338867Smarkj error = relocate_file1(ef, elf_lookup, elf_reloc, false); 1255338867Smarkj if (error == 0) 1256338867Smarkj error = relocate_file1(ef, elf_lookup, elf_reloc, true); 1257338867Smarkj return (error); 1258333720Skib} 1259333720Skib 126039071Sdfr/* 126139071Sdfr * Hash function for symbol table lookup. Don't even think about changing 126239071Sdfr * this. It is specified by the System V ABI. 126339071Sdfr */ 126439071Sdfrstatic unsigned long 126539071Sdfrelf_hash(const char *name) 126638514Sdfr{ 1267215013Smdf const unsigned char *p = (const unsigned char *) name; 1268215013Smdf unsigned long h = 0; 1269215013Smdf unsigned long g; 127038514Sdfr 1271215013Smdf while (*p != '\0') { 1272215013Smdf h = (h << 4) + *p++; 1273215013Smdf if ((g = h & 0xf0000000) != 0) 1274215013Smdf h ^= g >> 24; 1275215013Smdf h &= ~g; 1276215013Smdf } 1277215013Smdf return (h); 127838514Sdfr} 127938514Sdfr 1280104094Sphkstatic int 1281338867Smarkjlink_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t *sym) 128238514Sdfr{ 1283215013Smdf elf_file_t ef = (elf_file_t) lf; 1284215013Smdf unsigned long symnum; 1285215013Smdf const Elf_Sym* symp; 1286215013Smdf const char *strp; 1287215013Smdf unsigned long hash; 1288215013Smdf int i; 128938514Sdfr 1290215013Smdf /* If we don't have a hash, bail. */ 1291215013Smdf if (ef->buckets == NULL || ef->nbuckets == 0) { 1292215013Smdf printf("link_elf_lookup_symbol: missing symbol hash table\n"); 1293215013Smdf return (ENOENT); 1294215013Smdf } 1295151902Sjhb 1296215013Smdf /* First, search hashed global symbols */ 1297215013Smdf hash = elf_hash(name); 1298215013Smdf symnum = ef->buckets[hash % ef->nbuckets]; 129939071Sdfr 1300215013Smdf while (symnum != STN_UNDEF) { 1301215013Smdf if (symnum >= ef->nchains) { 1302215013Smdf printf("%s: corrupt symbol table\n", __func__); 1303215013Smdf return (ENOENT); 1304215013Smdf } 130538514Sdfr 1306215013Smdf symp = ef->symtab + symnum; 1307215013Smdf if (symp->st_name == 0) { 1308215013Smdf printf("%s: corrupt symbol table\n", __func__); 1309215013Smdf return (ENOENT); 1310215013Smdf } 131139071Sdfr 1312215013Smdf strp = ef->strtab + symp->st_name; 131339071Sdfr 1314215013Smdf if (strcmp(name, strp) == 0) { 1315215013Smdf if (symp->st_shndx != SHN_UNDEF || 1316215013Smdf (symp->st_value != 0 && 1317333720Skib (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1318333720Skib ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { 1319215013Smdf *sym = (c_linker_sym_t) symp; 1320215013Smdf return (0); 1321215013Smdf } 1322215013Smdf return (ENOENT); 1323215013Smdf } 1324215013Smdf 1325215013Smdf symnum = ef->chains[symnum]; 132639071Sdfr } 132739071Sdfr 1328215013Smdf /* If we have not found it, look at the full table (if loaded) */ 1329215013Smdf if (ef->symtab == ef->ddbsymtab) 1330215013Smdf return (ENOENT); 133139071Sdfr 1332215013Smdf /* Exhaustive search */ 1333215013Smdf for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1334215013Smdf strp = ef->ddbstrtab + symp->st_name; 1335215013Smdf if (strcmp(name, strp) == 0) { 1336215013Smdf if (symp->st_shndx != SHN_UNDEF || 1337215013Smdf (symp->st_value != 0 && 1338333720Skib (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1339333720Skib ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { 1340215013Smdf *sym = (c_linker_sym_t) symp; 1341215013Smdf return (0); 1342215013Smdf } 1343215013Smdf return (ENOENT); 1344215013Smdf } 134540254Speter } 134640254Speter 1347215013Smdf return (ENOENT); 134838514Sdfr} 134938514Sdfr 135040156Speterstatic int 1351215013Smdflink_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, 1352215013Smdf linker_symval_t *symval) 135338514Sdfr{ 1354333720Skib elf_file_t ef; 1355333720Skib const Elf_Sym *es; 1356333720Skib caddr_t val; 135738514Sdfr 1358333720Skib ef = (elf_file_t)lf; 1359333720Skib es = (const Elf_Sym *)sym; 1360102348Smarcel if (es >= ef->symtab && es < (ef->symtab + ef->nchains)) { 1361215013Smdf symval->name = ef->strtab + es->st_name; 1362333720Skib val = (caddr_t)ef->address + es->st_value; 1363333720Skib if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) 1364333720Skib val = ((caddr_t (*)(void))val)(); 1365333720Skib symval->value = val; 1366215013Smdf symval->size = es->st_size; 1367215013Smdf return (0); 136840254Speter } 136940254Speter if (ef->symtab == ef->ddbsymtab) 1370215013Smdf return (ENOENT); 1371102348Smarcel if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) { 1372215013Smdf symval->name = ef->ddbstrtab + es->st_name; 1373333720Skib val = (caddr_t)ef->address + es->st_value; 1374333720Skib if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) 1375333720Skib val = ((caddr_t (*)(void))val)(); 1376333720Skib symval->value = val; 1377215013Smdf symval->size = es->st_size; 1378215013Smdf return (0); 137940254Speter } 1380215013Smdf return (ENOENT); 138138514Sdfr} 138238514Sdfr 138338514Sdfrstatic int 138438514Sdfrlink_elf_search_symbol(linker_file_t lf, caddr_t value, 1385215013Smdf c_linker_sym_t *sym, long *diffp) 138638514Sdfr{ 138759603Sdfr elf_file_t ef = (elf_file_t) lf; 138855090Sbde u_long off = (uintptr_t) (void *) value; 138938514Sdfr u_long diff = off; 139055090Sbde u_long st_value; 139139071Sdfr const Elf_Sym* es; 1392298069Spfg const Elf_Sym* best = NULL; 139338514Sdfr int i; 139438514Sdfr 139540254Speter for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) { 139638514Sdfr if (es->st_name == 0) 139738514Sdfr continue; 139855090Sbde st_value = es->st_value + (uintptr_t) (void *) ef->address; 139953820Speter if (off >= st_value) { 140053820Speter if (off - st_value < diff) { 140153820Speter diff = off - st_value; 140238514Sdfr best = es; 140338514Sdfr if (diff == 0) 140438514Sdfr break; 140553820Speter } else if (off - st_value == diff) { 140638514Sdfr best = es; 140738514Sdfr } 140838514Sdfr } 140938514Sdfr } 1410298069Spfg if (best == NULL) 141138514Sdfr *diffp = off; 141238514Sdfr else 141338514Sdfr *diffp = diff; 141443301Sdillon *sym = (c_linker_sym_t) best; 141538514Sdfr 1416215013Smdf return (0); 141738514Sdfr} 141878161Speter 141978161Speter/* 142078161Speter * Look up a linker set on an ELF system. 142178161Speter */ 142278161Speterstatic int 142378161Speterlink_elf_lookup_set(linker_file_t lf, const char *name, 1424215013Smdf void ***startp, void ***stopp, int *countp) 142578161Speter{ 142678161Speter c_linker_sym_t sym; 142778161Speter linker_symval_t symval; 142878161Speter char *setsym; 142978161Speter void **start, **stop; 143078161Speter int len, error = 0, count; 143178161Speter 143278161Speter len = strlen(name) + sizeof("__start_set_"); /* sizeof includes \0 */ 1433111119Simp setsym = malloc(len, M_LINKER, M_WAITOK); 143478161Speter 143578161Speter /* get address of first entry */ 143678161Speter snprintf(setsym, len, "%s%s", "__start_set_", name); 143778161Speter error = link_elf_lookup_symbol(lf, setsym, &sym); 1438215013Smdf if (error != 0) 143978161Speter goto out; 144078161Speter link_elf_symbol_values(lf, sym, &symval); 144178161Speter if (symval.value == 0) { 144278161Speter error = ESRCH; 144378161Speter goto out; 144478161Speter } 144578161Speter start = (void **)symval.value; 144678161Speter 144778161Speter /* get address of last entry */ 144878161Speter snprintf(setsym, len, "%s%s", "__stop_set_", name); 144978161Speter error = link_elf_lookup_symbol(lf, setsym, &sym); 1450215013Smdf if (error != 0) 145178161Speter goto out; 145278161Speter link_elf_symbol_values(lf, sym, &symval); 145378161Speter if (symval.value == 0) { 145478161Speter error = ESRCH; 145578161Speter goto out; 145678161Speter } 145778161Speter stop = (void **)symval.value; 145878161Speter 145978161Speter /* and the number of entries */ 146078161Speter count = stop - start; 146178161Speter 146278161Speter /* and copy out */ 1463215013Smdf if (startp != NULL) 146478161Speter *startp = start; 1465215013Smdf if (stopp != NULL) 146678161Speter *stopp = stop; 1467215013Smdf if (countp != NULL) 146878161Speter *countp = count; 146978161Speter 147078161Speterout: 147178161Speter free(setsym, M_LINKER); 1472215013Smdf return (error); 147378161Speter} 147485736Sgreen 147585736Sgreenstatic int 147685736Sgreenlink_elf_each_function_name(linker_file_t file, 1477215013Smdf int (*callback)(const char *, void *), void *opaque) 1478215013Smdf{ 1479215013Smdf elf_file_t ef = (elf_file_t)file; 1480215013Smdf const Elf_Sym *symp; 1481215013Smdf int i, error; 1482292640Sngie 1483215013Smdf /* Exhaustive search */ 1484215013Smdf for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1485215013Smdf if (symp->st_value != 0 && 1486333720Skib (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1487333720Skib ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { 1488215013Smdf error = callback(ef->ddbstrtab + symp->st_name, opaque); 1489215013Smdf if (error != 0) 1490215013Smdf return (error); 1491215013Smdf } 149285736Sgreen } 1493215013Smdf return (0); 149485736Sgreen} 149595228Smarcel 1496179223Sjbstatic int 1497179223Sjblink_elf_each_function_nameval(linker_file_t file, 1498179223Sjb linker_function_nameval_callback_t callback, void *opaque) 1499179223Sjb{ 1500179223Sjb linker_symval_t symval; 1501179223Sjb elf_file_t ef = (elf_file_t)file; 1502179223Sjb const Elf_Sym* symp; 1503179223Sjb int i, error; 1504179223Sjb 1505179223Sjb /* Exhaustive search */ 1506179223Sjb for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1507179223Sjb if (symp->st_value != 0 && 1508333720Skib (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1509333720Skib ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { 1510215013Smdf error = link_elf_symbol_values(file, 1511215013Smdf (c_linker_sym_t) symp, &symval); 1512215013Smdf if (error != 0) 1513179223Sjb return (error); 1514179223Sjb error = callback(file, i, &symval, opaque); 1515215013Smdf if (error != 0) 1516179223Sjb return (error); 1517179223Sjb } 1518179223Sjb } 1519179223Sjb return (0); 1520179223Sjb} 1521179223Sjb 1522104072Sjakeconst Elf_Sym * 1523153504Smarcelelf_get_sym(linker_file_t lf, Elf_Size symidx) 1524104072Sjake{ 1525104072Sjake elf_file_t ef = (elf_file_t)lf; 1526104072Sjake 1527104072Sjake if (symidx >= ef->nchains) 1528104072Sjake return (NULL); 1529104072Sjake return (ef->symtab + symidx); 1530104072Sjake} 1531104072Sjake 1532105147Smarcelconst char * 1533153504Smarcelelf_get_symname(linker_file_t lf, Elf_Size symidx) 1534105147Smarcel{ 1535105147Smarcel elf_file_t ef = (elf_file_t)lf; 1536105147Smarcel const Elf_Sym *sym; 1537105147Smarcel 1538105147Smarcel if (symidx >= ef->nchains) 1539105147Smarcel return (NULL); 1540105147Smarcel sym = ef->symtab + symidx; 1541105147Smarcel return (ef->strtab + sym->st_name); 1542105147Smarcel} 1543105147Smarcel 154495410Smarcel/* 154595410Smarcel * Symbol lookup function that can be used when the symbol index is known (ie 154695410Smarcel * in relocations). It uses the symbol index instead of doing a fully fledged 154795410Smarcel * hash table based lookup when such is valid. For example for local symbols. 154895410Smarcel * This is not only more efficient, it's also more correct. It's not always 154995410Smarcel * the case that the symbol can be found through the hash table. 155095410Smarcel */ 1551288000Skibstatic int 1552288000Skibelf_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res) 155395410Smarcel{ 155495410Smarcel elf_file_t ef = (elf_file_t)lf; 155595410Smarcel const Elf_Sym *sym; 155695410Smarcel const char *symbol; 1557240997Strociny Elf_Addr addr, start, base; 155895410Smarcel 155995410Smarcel /* Don't even try to lookup the symbol if the index is bogus. */ 1560288000Skib if (symidx >= ef->nchains) { 1561288000Skib *res = 0; 1562288000Skib return (EINVAL); 1563288000Skib } 156495410Smarcel 156595410Smarcel sym = ef->symtab + symidx; 156695410Smarcel 156795410Smarcel /* 156895410Smarcel * Don't do a full lookup when the symbol is local. It may even 156995410Smarcel * fail because it may not be found through the hash table. 157095410Smarcel */ 157195410Smarcel if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) { 157295410Smarcel /* Force lookup failure when we have an insanity. */ 1573288000Skib if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0) { 1574288000Skib *res = 0; 1575288000Skib return (EINVAL); 1576288000Skib } 1577288000Skib *res = ((Elf_Addr)ef->address + sym->st_value); 1578288000Skib return (0); 157995410Smarcel } 158095410Smarcel 158195410Smarcel /* 158295410Smarcel * XXX we can avoid doing a hash table based lookup for global 158395410Smarcel * symbols as well. This however is not always valid, so we'll 158495410Smarcel * just do it the hard way for now. Performance tweaks can 158595410Smarcel * always be added. 158695410Smarcel */ 158795410Smarcel 158895410Smarcel symbol = ef->strtab + sym->st_name; 158995410Smarcel 159095410Smarcel /* Force a lookup failure if the symbol name is bogus. */ 1591288000Skib if (*symbol == 0) { 1592288000Skib *res = 0; 1593288000Skib return (EINVAL); 1594288000Skib } 159595410Smarcel 1596240997Strociny addr = ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps)); 1597290320Smarkj if (addr == 0 && ELF_ST_BIND(sym->st_info) != STB_WEAK) { 1598290320Smarkj *res = 0; 1599290320Smarkj return (EINVAL); 1600290320Smarkj } 1601240997Strociny 1602240997Strociny if (elf_set_find(&set_pcpu_list, addr, &start, &base)) 1603240997Strociny addr = addr - start + base; 1604240997Strociny#ifdef VIMAGE 1605240997Strociny else if (elf_set_find(&set_vnet_list, addr, &start, &base)) 1606240997Strociny addr = addr - start + base; 1607240997Strociny#endif 1608288000Skib *res = addr; 1609288000Skib return (0); 161095410Smarcel} 1611109605Sjake 1612109605Sjakestatic void 1613109605Sjakelink_elf_reloc_local(linker_file_t lf) 1614109605Sjake{ 1615215013Smdf const Elf_Rel *rellim; 1616215013Smdf const Elf_Rel *rel; 1617215013Smdf const Elf_Rela *relalim; 1618215013Smdf const Elf_Rela *rela; 1619215013Smdf elf_file_t ef = (elf_file_t)lf; 1620109605Sjake 1621215013Smdf /* Perform relocations without addend if there are any: */ 1622215013Smdf if ((rel = ef->rel) != NULL) { 1623215013Smdf rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize); 1624215013Smdf while (rel < rellim) { 1625215013Smdf elf_reloc_local(lf, (Elf_Addr)ef->address, rel, 1626215013Smdf ELF_RELOC_REL, elf_lookup); 1627215013Smdf rel++; 1628215013Smdf } 1629109605Sjake } 1630109605Sjake 1631215013Smdf /* Perform relocations with addend if there are any: */ 1632215013Smdf if ((rela = ef->rela) != NULL) { 1633215013Smdf relalim = (const Elf_Rela *) 1634215013Smdf ((const char *)ef->rela + ef->relasize); 1635215013Smdf while (rela < relalim) { 1636215013Smdf elf_reloc_local(lf, (Elf_Addr)ef->address, rela, 1637215013Smdf ELF_RELOC_RELA, elf_lookup); 1638215013Smdf rela++; 1639215013Smdf } 1640109605Sjake } 1641109605Sjake} 1642192859Ssson 1643192859Sssonstatic long 1644192859Sssonlink_elf_symtab_get(linker_file_t lf, const Elf_Sym **symtab) 1645192859Ssson{ 1646215013Smdf elf_file_t ef = (elf_file_t)lf; 1647192859Ssson 1648215013Smdf *symtab = ef->ddbsymtab; 1649215013Smdf 1650215013Smdf if (*symtab == NULL) 1651215013Smdf return (0); 1652215013Smdf 1653215013Smdf return (ef->ddbsymcnt); 1654192859Ssson} 1655292640Sngie 1656192859Sssonstatic long 1657192859Sssonlink_elf_strtab_get(linker_file_t lf, caddr_t *strtab) 1658192859Ssson{ 1659215013Smdf elf_file_t ef = (elf_file_t)lf; 1660192859Ssson 1661215013Smdf *strtab = ef->ddbstrtab; 1662192859Ssson 1663215013Smdf if (*strtab == NULL) 1664215013Smdf return (0); 1665192859Ssson 1666215013Smdf return (ef->ddbstrcnt); 1667192859Ssson} 1668333720Skib 1669333720Skib#if defined(__i386__) || defined(__amd64__) 1670338867Smarkj/* 1671338867Smarkj * Use this lookup routine when performing relocations early during boot. 1672338867Smarkj * The generic lookup routine depends on kobj, which is not initialized 1673338867Smarkj * at that point. 1674338867Smarkj */ 1675338867Smarkjstatic int 1676338867Smarkjelf_lookup_ifunc(linker_file_t lf, Elf_Size symidx, int deps __unused, 1677338867Smarkj Elf_Addr *res) 1678338867Smarkj{ 1679338867Smarkj elf_file_t ef; 1680338867Smarkj const Elf_Sym *symp; 1681338867Smarkj caddr_t val; 1682338867Smarkj 1683338867Smarkj ef = (elf_file_t)lf; 1684338867Smarkj symp = ef->symtab + symidx; 1685338867Smarkj if (ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC) { 1686338867Smarkj val = (caddr_t)ef->address + symp->st_value; 1687338867Smarkj *res = ((Elf_Addr (*)(void))val)(); 1688338867Smarkj return (0); 1689338867Smarkj } 1690338867Smarkj return (ENOENT); 1691338867Smarkj} 1692338867Smarkj 1693333720Skibvoid 1694333720Skiblink_elf_ireloc(caddr_t kmdp) 1695333720Skib{ 1696333720Skib struct elf_file eff; 1697333720Skib elf_file_t ef; 1698333720Skib volatile char *c; 1699333720Skib size_t i; 1700333720Skib 1701338867Smarkj ef = &eff; 1702333720Skib 1703333720Skib /* Do not use bzero/memset before ireloc is done. */ 1704333720Skib for (c = (char *)ef, i = 0; i < sizeof(*ef); i++) 1705333720Skib c[i] = 0; 1706333720Skib 1707333720Skib ef->modptr = kmdp; 1708333720Skib ef->dynamic = (Elf_Dyn *)&_DYNAMIC; 1709333720Skib parse_dynamic(ef); 1710333720Skib ef->address = 0; 1711333720Skib link_elf_preload_parse_symbols(ef); 1712338867Smarkj relocate_file1(ef, elf_lookup_ifunc, elf_reloc, true); 1713333720Skib} 1714333720Skib#endif 1715