link_elf.c revision 151902 
1/*-
2 * Copyright (c) 1998-2000 Doug Rabson
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/kern/link_elf.c 151902 2005-10-31 19:17:32Z jhb $");
29
30#include "opt_gdb.h"
31#include "opt_mac.h"
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#ifdef GPROF
36#include <sys/gmon.h>
37#endif
38#include <sys/kernel.h>
39#include <sys/lock.h>
40#include <sys/mac.h>
41#include <sys/malloc.h>
42#include <sys/mutex.h>
43#include <sys/proc.h>
44#include <sys/namei.h>
45#include <sys/fcntl.h>
46#include <sys/vnode.h>
47#include <sys/linker.h>
48
49#include <machine/elf.h>
50
51#include <vm/vm.h>
52#include <vm/vm_param.h>
53#ifdef SPARSE_MAPPING
54#include <vm/vm_object.h>
55#include <vm/vm_kern.h>
56#include <vm/vm_extern.h>
57#endif
58#include <vm/pmap.h>
59#include <vm/vm_map.h>
60
61#include <sys/link_elf.h>
62
63#include "linker_if.h"
64
65#define MAXSEGS 4
66
67typedef struct elf_file {
68    struct linker_file	lf;		/* Common fields */
69    int			preloaded;	/* Was file pre-loaded */
70    caddr_t		address;	/* Relocation address */
71#ifdef SPARSE_MAPPING
72    vm_object_t		object;		/* VM object to hold file pages */
73#endif
74    Elf_Dyn*		dynamic;	/* Symbol table etc. */
75    Elf_Hashelt		nbuckets;	/* DT_HASH info */
76    Elf_Hashelt		nchains;
77    const Elf_Hashelt*	buckets;
78    const Elf_Hashelt*	chains;
79    caddr_t		hash;
80    caddr_t		strtab;		/* DT_STRTAB */
81    int			strsz;		/* DT_STRSZ */
82    const Elf_Sym*	symtab;		/* DT_SYMTAB */
83    Elf_Addr*		got;		/* DT_PLTGOT */
84    const Elf_Rel*	pltrel;		/* DT_JMPREL */
85    int			pltrelsize;	/* DT_PLTRELSZ */
86    const Elf_Rela*	pltrela;	/* DT_JMPREL */
87    int			pltrelasize;	/* DT_PLTRELSZ */
88    const Elf_Rel*	rel;		/* DT_REL */
89    int			relsize;	/* DT_RELSZ */
90    const Elf_Rela*	rela;		/* DT_RELA */
91    int			relasize;	/* DT_RELASZ */
92    caddr_t		modptr;
93    const Elf_Sym*	ddbsymtab;	/* The symbol table we are using */
94    long		ddbsymcnt;	/* Number of symbols */
95    caddr_t		ddbstrtab;	/* String table */
96    long		ddbstrcnt;	/* number of bytes in string table */
97    caddr_t		symbase;	/* malloc'ed symbold base */
98    caddr_t		strbase;	/* malloc'ed string base */
99#ifdef GDB
100    struct link_map	gdb;		/* hooks for gdb */
101#endif
102} *elf_file_t;
103
104static int	link_elf_link_common_finish(linker_file_t);
105static int	link_elf_link_preload(linker_class_t cls,
106				      const char*, linker_file_t*);
107static int	link_elf_link_preload_finish(linker_file_t);
108static int	link_elf_load_file(linker_class_t, const char*, linker_file_t*);
109static int	link_elf_lookup_symbol(linker_file_t, const char*,
110				       c_linker_sym_t*);
111static int	link_elf_symbol_values(linker_file_t, c_linker_sym_t, linker_symval_t*);
112static int	link_elf_search_symbol(linker_file_t, caddr_t value,
113				       c_linker_sym_t* sym, long* diffp);
114
115static void	link_elf_unload_file(linker_file_t);
116static void	link_elf_unload_preload(linker_file_t);
117static int	link_elf_lookup_set(linker_file_t, const char *,
118				    void ***, void ***, int *);
119static int	link_elf_each_function_name(linker_file_t,
120				int (*)(const char *, void *),
121				void *);
122static void	link_elf_reloc_local(linker_file_t);
123static Elf_Addr	elf_lookup(linker_file_t lf, Elf_Word symidx, int deps);
124
125static kobj_method_t link_elf_methods[] = {
126    KOBJMETHOD(linker_lookup_symbol,	link_elf_lookup_symbol),
127    KOBJMETHOD(linker_symbol_values,	link_elf_symbol_values),
128    KOBJMETHOD(linker_search_symbol,	link_elf_search_symbol),
129    KOBJMETHOD(linker_unload,		link_elf_unload_file),
130    KOBJMETHOD(linker_load_file,	link_elf_load_file),
131    KOBJMETHOD(linker_link_preload,	link_elf_link_preload),
132    KOBJMETHOD(linker_link_preload_finish, link_elf_link_preload_finish),
133    KOBJMETHOD(linker_lookup_set,	link_elf_lookup_set),
134    KOBJMETHOD(linker_each_function_name, link_elf_each_function_name),
135    { 0, 0 }
136};
137
138static struct linker_class link_elf_class = {
139#if ELF_TARG_CLASS == ELFCLASS32
140    "elf32",
141#else
142    "elf64",
143#endif
144    link_elf_methods, sizeof(struct elf_file)
145};
146
147static int		parse_dynamic(elf_file_t ef);
148static int		relocate_file(elf_file_t ef);
149static int		link_elf_preload_parse_symbols(elf_file_t ef);
150
151#ifdef GDB
152static void		r_debug_state(struct r_debug *dummy_one,
153				      struct link_map *dummy_two);
154
155/*
156 * A list of loaded modules for GDB to use for loading symbols.
157 */
158struct r_debug r_debug;
159
160#define GDB_STATE(s)	r_debug.r_state = s; r_debug_state(NULL, NULL);
161
162/*
163 * Function for the debugger to set a breakpoint on to gain control.
164 */
165static void
166r_debug_state(struct r_debug *dummy_one __unused,
167	      struct link_map *dummy_two __unused)
168{
169}
170
171static void
172link_elf_add_gdb(struct link_map *l)
173{
174    struct link_map *prev;
175
176    l->l_next = NULL;
177
178    if (r_debug.r_map == NULL) {
179	/* Add first. */
180	l->l_prev = NULL;
181	r_debug.r_map = l;
182    } else {
183	/* Append to list. */
184	for (prev = r_debug.r_map; prev->l_next != NULL; prev = prev->l_next)
185	    ;
186	l->l_prev = prev;
187	prev->l_next = l;
188    }
189}
190
191static void
192link_elf_delete_gdb(struct link_map *l)
193{
194    if (l->l_prev == NULL) {
195	/* Remove first. */
196	if ((r_debug.r_map = l->l_next) != NULL)
197	    l->l_next->l_prev = NULL;
198    } else {
199	/* Remove any but first. */
200	if ((l->l_prev->l_next = l->l_next) != NULL)
201	    l->l_next->l_prev = l->l_prev;
202    }
203}
204#endif /* GDB */
205
206#ifdef __ia64__
207Elf_Addr link_elf_get_gp(linker_file_t);
208#endif
209
210/*
211 * The kernel symbol table starts here.
212 */
213extern struct _dynamic _DYNAMIC;
214
215static void
216link_elf_error(const char *s)
217{
218    printf("kldload: %s\n", s);
219}
220
221/*
222 * Actions performed after linking/loading both the preloaded kernel and any
223 * modules; whether preloaded or dynamicly loaded.
224 */
225static int
226link_elf_link_common_finish(linker_file_t lf)
227{
228#ifdef GDB
229    elf_file_t ef = (elf_file_t)lf;
230    char *newfilename;
231#endif
232    int error;
233
234    /* Notify MD code that a module is being loaded. */
235    error = elf_cpu_load_file(lf);
236    if (error)
237	return (error);
238
239#ifdef GDB
240    GDB_STATE(RT_ADD);
241    ef->gdb.l_addr = lf->address;
242    newfilename = malloc(strlen(lf->filename) + 1, M_LINKER, M_WAITOK);
243    strcpy(newfilename, lf->filename);
244    ef->gdb.l_name = newfilename;
245    ef->gdb.l_ld = ef->dynamic;
246    link_elf_add_gdb(&ef->gdb);
247    GDB_STATE(RT_CONSISTENT);
248#endif
249
250    return (0);
251}
252
253static void
254link_elf_init(void* arg)
255{
256    Elf_Dyn	*dp;
257    caddr_t	modptr, baseptr, sizeptr;
258    elf_file_t	ef;
259    char	*modname;
260
261    linker_add_class(&link_elf_class);
262
263    dp = (Elf_Dyn*) &_DYNAMIC;
264    modname = NULL;
265    modptr = preload_search_by_type("elf" __XSTRING(__ELF_WORD_SIZE) " kernel");
266    if (modptr == NULL)
267	modptr = preload_search_by_type("elf kernel");
268    if (modptr)
269	modname = (char *)preload_search_info(modptr, MODINFO_NAME);
270    if (modname == NULL)
271	modname = "kernel";
272    linker_kernel_file = linker_make_file(modname, &link_elf_class);
273    if (linker_kernel_file == NULL)
274	panic("link_elf_init: Can't create linker structures for kernel");
275
276    ef = (elf_file_t) linker_kernel_file;
277    ef->preloaded = 1;
278    ef->address = 0;
279#ifdef SPARSE_MAPPING
280    ef->object = 0;
281#endif
282    ef->dynamic = dp;
283
284    if (dp)
285	parse_dynamic(ef);
286    linker_kernel_file->address = (caddr_t) KERNBASE;
287    linker_kernel_file->size = -(intptr_t)linker_kernel_file->address;
288
289    if (modptr) {
290	ef->modptr = modptr;
291	baseptr = preload_search_info(modptr, MODINFO_ADDR);
292	if (baseptr)
293	    linker_kernel_file->address = *(caddr_t *)baseptr;
294	sizeptr = preload_search_info(modptr, MODINFO_SIZE);
295	if (sizeptr)
296	    linker_kernel_file->size = *(size_t *)sizeptr;
297    }
298    (void)link_elf_preload_parse_symbols(ef);
299
300#ifdef GDB
301    r_debug.r_map = NULL;
302    r_debug.r_brk = r_debug_state;
303    r_debug.r_state = RT_CONSISTENT;
304#endif
305
306    (void)link_elf_link_common_finish(linker_kernel_file);
307}
308
309SYSINIT(link_elf, SI_SUB_KLD, SI_ORDER_SECOND, link_elf_init, 0);
310
311static int
312link_elf_preload_parse_symbols(elf_file_t ef)
313{
314    caddr_t	pointer;
315    caddr_t	ssym, esym, base;
316    caddr_t	strtab;
317    int		strcnt;
318    Elf_Sym*	symtab;
319    int		symcnt;
320
321    if (ef->modptr == NULL)
322	return 0;
323    pointer = preload_search_info(ef->modptr, MODINFO_METADATA|MODINFOMD_SSYM);
324    if (pointer == NULL)
325	return 0;
326    ssym = *(caddr_t *)pointer;
327    pointer = preload_search_info(ef->modptr, MODINFO_METADATA|MODINFOMD_ESYM);
328    if (pointer == NULL)
329	return 0;
330    esym = *(caddr_t *)pointer;
331
332    base = ssym;
333
334    symcnt = *(long *)base;
335    base += sizeof(long);
336    symtab = (Elf_Sym *)base;
337    base += roundup(symcnt, sizeof(long));
338
339    if (base > esym || base < ssym) {
340	printf("Symbols are corrupt!\n");
341	return EINVAL;
342    }
343
344    strcnt = *(long *)base;
345    base += sizeof(long);
346    strtab = base;
347    base += roundup(strcnt, sizeof(long));
348
349    if (base > esym || base < ssym) {
350	printf("Symbols are corrupt!\n");
351	return EINVAL;
352    }
353
354    ef->ddbsymtab = symtab;
355    ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);
356    ef->ddbstrtab = strtab;
357    ef->ddbstrcnt = strcnt;
358
359    return 0;
360}
361
362static int
363parse_dynamic(elf_file_t ef)
364{
365    Elf_Dyn *dp;
366    int plttype = DT_REL;
367
368    for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {
369	switch (dp->d_tag) {
370	case DT_HASH:
371	{
372	    /* From src/libexec/rtld-elf/rtld.c */
373	    const Elf_Hashelt *hashtab = (const Elf_Hashelt *)
374		(ef->address + dp->d_un.d_ptr);
375	    ef->nbuckets = hashtab[0];
376	    ef->nchains = hashtab[1];
377	    ef->buckets = hashtab + 2;
378	    ef->chains = ef->buckets + ef->nbuckets;
379	    break;
380	}
381	case DT_STRTAB:
382	    ef->strtab = (caddr_t) (ef->address + dp->d_un.d_ptr);
383	    break;
384	case DT_STRSZ:
385	    ef->strsz = dp->d_un.d_val;
386	    break;
387	case DT_SYMTAB:
388	    ef->symtab = (Elf_Sym*) (ef->address + dp->d_un.d_ptr);
389	    break;
390	case DT_SYMENT:
391	    if (dp->d_un.d_val != sizeof(Elf_Sym))
392		return ENOEXEC;
393	    break;
394	case DT_PLTGOT:
395	    ef->got = (Elf_Addr *) (ef->address + dp->d_un.d_ptr);
396	    break;
397	case DT_REL:
398	    ef->rel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr);
399	    break;
400	case DT_RELSZ:
401	    ef->relsize = dp->d_un.d_val;
402	    break;
403	case DT_RELENT:
404	    if (dp->d_un.d_val != sizeof(Elf_Rel))
405		return ENOEXEC;
406	    break;
407	case DT_JMPREL:
408	    ef->pltrel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr);
409	    break;
410	case DT_PLTRELSZ:
411	    ef->pltrelsize = dp->d_un.d_val;
412	    break;
413	case DT_RELA:
414	    ef->rela = (const Elf_Rela *) (ef->address + dp->d_un.d_ptr);
415	    break;
416	case DT_RELASZ:
417	    ef->relasize = dp->d_un.d_val;
418	    break;
419	case DT_RELAENT:
420	    if (dp->d_un.d_val != sizeof(Elf_Rela))
421		return ENOEXEC;
422	    break;
423	case DT_PLTREL:
424	    plttype = dp->d_un.d_val;
425	    if (plttype != DT_REL && plttype != DT_RELA)
426		return ENOEXEC;
427	    break;
428#ifdef GDB
429	case DT_DEBUG:
430	    dp->d_un.d_ptr = (Elf_Addr) &r_debug;
431	    break;
432#endif
433	}
434    }
435
436    if (plttype == DT_RELA) {
437	ef->pltrela = (const Elf_Rela *) ef->pltrel;
438	ef->pltrel = NULL;
439	ef->pltrelasize = ef->pltrelsize;
440	ef->pltrelsize = 0;
441    }
442
443    ef->ddbsymtab = ef->symtab;
444    ef->ddbsymcnt = ef->nchains;
445    ef->ddbstrtab = ef->strtab;
446    ef->ddbstrcnt = ef->strsz;
447
448    return 0;
449}
450
451static int
452link_elf_link_preload(linker_class_t cls,
453		      const char* filename, linker_file_t *result)
454{
455    caddr_t		modptr, baseptr, sizeptr, dynptr;
456    char		*type;
457    elf_file_t		ef;
458    linker_file_t	lf;
459    int			error;
460    vm_offset_t		dp;
461
462    /* Look to see if we have the file preloaded */
463    modptr = preload_search_by_name(filename);
464    if (modptr == NULL)
465	return ENOENT;
466
467    type = (char *)preload_search_info(modptr, MODINFO_TYPE);
468    baseptr = preload_search_info(modptr, MODINFO_ADDR);
469    sizeptr = preload_search_info(modptr, MODINFO_SIZE);
470    dynptr = preload_search_info(modptr, MODINFO_METADATA|MODINFOMD_DYNAMIC);
471    if (type == NULL ||
472	(strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE) " module") != 0 &&
473	 strcmp(type, "elf module") != 0))
474	return (EFTYPE);
475    if (baseptr == NULL || sizeptr == NULL || dynptr == NULL)
476	return (EINVAL);
477
478    lf = linker_make_file(filename, &link_elf_class);
479    if (lf == NULL) {
480	return ENOMEM;
481    }
482
483    ef = (elf_file_t) lf;
484    ef->preloaded = 1;
485    ef->modptr = modptr;
486    ef->address = *(caddr_t *)baseptr;
487#ifdef SPARSE_MAPPING
488    ef->object = 0;
489#endif
490    dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr;
491    ef->dynamic = (Elf_Dyn *)dp;
492    lf->address = ef->address;
493    lf->size = *(size_t *)sizeptr;
494
495    error = parse_dynamic(ef);
496    if (error) {
497	linker_file_unload(lf, LINKER_UNLOAD_FORCE);
498	return error;
499    }
500    link_elf_reloc_local(lf);
501    *result = lf;
502    return (0);
503}
504
505static int
506link_elf_link_preload_finish(linker_file_t lf)
507{
508    elf_file_t		ef;
509    int error;
510
511    ef = (elf_file_t) lf;
512#if 0	/* this will be more trouble than it's worth for now */
513    for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {
514	if (dp->d_tag != DT_NEEDED)
515	    continue;
516	modname = ef->strtab + dp->d_un.d_val;
517	error = linker_load_module(modname, lf);
518	if (error)
519	    goto out;
520    }
521#endif
522    error = relocate_file(ef);
523    if (error)
524	return error;
525    (void)link_elf_preload_parse_symbols(ef);
526
527    return (link_elf_link_common_finish(lf));
528}
529
530static int
531link_elf_load_file(linker_class_t cls, const char* filename,
532	linker_file_t* result)
533{
534    struct nameidata nd;
535    struct thread* td = curthread;	/* XXX */
536    Elf_Ehdr *hdr;
537    caddr_t firstpage;
538    int nbytes, i;
539    Elf_Phdr *phdr;
540    Elf_Phdr *phlimit;
541    Elf_Phdr *segs[MAXSEGS];
542    int nsegs;
543    Elf_Phdr *phdyn;
544    Elf_Phdr *phphdr;
545    caddr_t mapbase;
546    size_t mapsize;
547    Elf_Off base_offset;
548    Elf_Addr base_vaddr;
549    Elf_Addr base_vlimit;
550    int error = 0;
551    int resid, flags;
552    elf_file_t ef;
553    linker_file_t lf;
554    Elf_Shdr *shdr;
555    int symtabindex;
556    int symstrindex;
557    int symcnt;
558    int strcnt;
559
560    GIANT_REQUIRED;
561
562    shdr = NULL;
563    lf = NULL;
564
565    NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td);
566    flags = FREAD;
567    error = vn_open(&nd, &flags, 0, -1);
568    if (error)
569	return error;
570    NDFREE(&nd, NDF_ONLY_PNBUF);
571#ifdef MAC
572    error = mac_check_kld_load(curthread->td_ucred, nd.ni_vp);
573    if (error) {
574	firstpage = NULL;
575	goto out;
576    }
577#endif
578
579    /*
580     * Read the elf header from the file.
581     */
582    firstpage = malloc(PAGE_SIZE, M_LINKER, M_WAITOK);
583    if (firstpage == NULL) {
584	error = ENOMEM;
585	goto out;
586    }
587    hdr = (Elf_Ehdr *)firstpage;
588    error = vn_rdwr(UIO_READ, nd.ni_vp, firstpage, PAGE_SIZE, 0,
589		    UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
590		    &resid, td);
591    nbytes = PAGE_SIZE - resid;
592    if (error)
593	goto out;
594
595    if (!IS_ELF(*hdr)) {
596	error = ENOEXEC;
597	goto out;
598    }
599
600    if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS
601      || hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
602	link_elf_error("Unsupported file layout");
603	error = ENOEXEC;
604	goto out;
605    }
606    if (hdr->e_ident[EI_VERSION] != EV_CURRENT
607      || hdr->e_version != EV_CURRENT) {
608	link_elf_error("Unsupported file version");
609	error = ENOEXEC;
610	goto out;
611    }
612    if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) {
613	link_elf_error("Unsupported file type");
614	error = ENOEXEC;
615	goto out;
616    }
617    if (hdr->e_machine != ELF_TARG_MACH) {
618	link_elf_error("Unsupported machine");
619	error = ENOEXEC;
620	goto out;
621    }
622
623    /*
624     * We rely on the program header being in the first page.  This is
625     * not strictly required by the ABI specification, but it seems to
626     * always true in practice.  And, it simplifies things considerably.
627     */
628    if (!((hdr->e_phentsize == sizeof(Elf_Phdr)) &&
629	  (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= PAGE_SIZE) &&
630	  (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= nbytes)))
631	link_elf_error("Unreadable program headers");
632
633    /*
634     * Scan the program header entries, and save key information.
635     *
636     * We rely on there being exactly two load segments, text and data,
637     * in that order.
638     */
639    phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff);
640    phlimit = phdr + hdr->e_phnum;
641    nsegs = 0;
642    phdyn = NULL;
643    phphdr = NULL;
644    while (phdr < phlimit) {
645	switch (phdr->p_type) {
646
647	case PT_LOAD:
648	    if (nsegs == MAXSEGS) {
649		link_elf_error("Too many sections");
650		error = ENOEXEC;
651		goto out;
652	    }
653	    /*
654	     * XXX: We just trust they come in right order ??
655	     */
656	    segs[nsegs] = phdr;
657	    ++nsegs;
658	    break;
659
660	case PT_PHDR:
661	    phphdr = phdr;
662	    break;
663
664	case PT_DYNAMIC:
665	    phdyn = phdr;
666	    break;
667
668	case PT_INTERP:
669	    link_elf_error("Unsupported file type");
670	    error = ENOEXEC;
671	    goto out;
672	}
673
674	++phdr;
675    }
676    if (phdyn == NULL) {
677	link_elf_error("Object is not dynamically-linked");
678	error = ENOEXEC;
679	goto out;
680    }
681    if (nsegs == 0) {
682	link_elf_error("No sections");
683	error = ENOEXEC;
684	goto out;
685    }
686
687    /*
688     * Allocate the entire address space of the object, to stake out our
689     * contiguous region, and to establish the base address for relocation.
690     */
691    base_offset = trunc_page(segs[0]->p_offset);
692    base_vaddr = trunc_page(segs[0]->p_vaddr);
693    base_vlimit = round_page(segs[nsegs - 1]->p_vaddr +
694	segs[nsegs - 1]->p_memsz);
695    mapsize = base_vlimit - base_vaddr;
696
697    lf = linker_make_file(filename, &link_elf_class);
698    if (!lf) {
699	error = ENOMEM;
700	goto out;
701    }
702
703    ef = (elf_file_t) lf;
704#ifdef SPARSE_MAPPING
705    ef->object = vm_object_allocate(OBJT_DEFAULT, mapsize >> PAGE_SHIFT);
706    if (ef->object == NULL) {
707	error = ENOMEM;
708	goto out;
709    }
710    ef->address = (caddr_t) vm_map_min(kernel_map);
711    error = vm_map_find(kernel_map, ef->object, 0,
712			(vm_offset_t *) &ef->address,
713			mapsize, 1,
714			VM_PROT_ALL, VM_PROT_ALL, 0);
715    if (error) {
716	vm_object_deallocate(ef->object);
717	ef->object = 0;
718	goto out;
719    }
720#else
721    ef->address = malloc(mapsize, M_LINKER, M_WAITOK);
722    if (!ef->address) {
723	error = ENOMEM;
724	goto out;
725    }
726#endif
727    mapbase = ef->address;
728
729    /*
730     * Read the text and data sections and zero the bss.
731     */
732    for (i = 0; i < nsegs; i++) {
733	caddr_t segbase = mapbase + segs[i]->p_vaddr - base_vaddr;
734	error = vn_rdwr(UIO_READ, nd.ni_vp,
735			segbase, segs[i]->p_filesz, segs[i]->p_offset,
736			UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
737			&resid, td);
738	if (error) {
739	    goto out;
740	}
741	bzero(segbase + segs[i]->p_filesz,
742	      segs[i]->p_memsz - segs[i]->p_filesz);
743
744#ifdef SPARSE_MAPPING
745	/*
746	 * Wire down the pages
747	 */
748	error = vm_map_wire(kernel_map,
749		    (vm_offset_t) segbase,
750		    (vm_offset_t) segbase + segs[i]->p_memsz,
751		    VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES);
752	if (error != KERN_SUCCESS) {
753	    error = ENOMEM;
754	    goto out;
755	}
756#endif
757    }
758
759#ifdef GPROF
760    /* Update profiling information with the new text segment. */
761    kmupetext((uintfptr_t)(mapbase + segs[0]->p_vaddr - base_vaddr +
762	segs[0]->p_memsz));
763#endif
764
765    ef->dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr);
766
767    lf->address = ef->address;
768    lf->size = mapsize;
769
770    error = parse_dynamic(ef);
771    if (error)
772	goto out;
773    link_elf_reloc_local(lf);
774
775    error = linker_load_dependencies(lf);
776    if (error)
777	goto out;
778#if 0	/* this will be more trouble than it's worth for now */
779    for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {
780	if (dp->d_tag != DT_NEEDED)
781	    continue;
782	modname = ef->strtab + dp->d_un.d_val;
783	error = linker_load_module(modname, lf);
784	if (error)
785	    goto out;
786    }
787#endif
788    error = relocate_file(ef);
789    if (error)
790	goto out;
791
792    /* Try and load the symbol table if it's present.  (you can strip it!) */
793    nbytes = hdr->e_shnum * hdr->e_shentsize;
794    if (nbytes == 0 || hdr->e_shoff == 0)
795	goto nosyms;
796    shdr = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO);
797    if (shdr == NULL) {
798	error = ENOMEM;
799	goto out;
800    }
801    error = vn_rdwr(UIO_READ, nd.ni_vp,
802		    (caddr_t)shdr, nbytes, hdr->e_shoff,
803		    UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
804		    &resid, td);
805    if (error)
806	goto out;
807    symtabindex = -1;
808    symstrindex = -1;
809    for (i = 0; i < hdr->e_shnum; i++) {
810	if (shdr[i].sh_type == SHT_SYMTAB) {
811	    symtabindex = i;
812	    symstrindex = shdr[i].sh_link;
813	}
814    }
815    if (symtabindex < 0 || symstrindex < 0)
816	goto nosyms;
817
818    symcnt = shdr[symtabindex].sh_size;
819    ef->symbase = malloc(symcnt, M_LINKER, M_WAITOK);
820    strcnt = shdr[symstrindex].sh_size;
821    ef->strbase = malloc(strcnt, M_LINKER, M_WAITOK);
822
823    if (ef->symbase == NULL || ef->strbase == NULL) {
824	error = ENOMEM;
825	goto out;
826    }
827    error = vn_rdwr(UIO_READ, nd.ni_vp,
828		    ef->symbase, symcnt, shdr[symtabindex].sh_offset,
829		    UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
830		    &resid, td);
831    if (error)
832	goto out;
833    error = vn_rdwr(UIO_READ, nd.ni_vp,
834		    ef->strbase, strcnt, shdr[symstrindex].sh_offset,
835		    UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
836		    &resid, td);
837    if (error)
838	goto out;
839
840    ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);
841    ef->ddbsymtab = (const Elf_Sym *)ef->symbase;
842    ef->ddbstrcnt = strcnt;
843    ef->ddbstrtab = ef->strbase;
844
845    error = link_elf_link_common_finish(lf);
846    if (error)
847	goto out;
848
849nosyms:
850
851    *result = lf;
852
853out:
854    if (error && lf)
855	linker_file_unload(lf, LINKER_UNLOAD_FORCE);
856    if (shdr)
857	free(shdr, M_LINKER);
858    if (firstpage)
859	free(firstpage, M_LINKER);
860    VOP_UNLOCK(nd.ni_vp, 0, td);
861    vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
862
863    return error;
864}
865
866static void
867link_elf_unload_file(linker_file_t file)
868{
869    elf_file_t ef = (elf_file_t) file;
870
871#ifdef GDB
872    if (ef->gdb.l_ld) {
873	GDB_STATE(RT_DELETE);
874	free((void *)(uintptr_t)ef->gdb.l_name, M_LINKER);
875	link_elf_delete_gdb(&ef->gdb);
876	GDB_STATE(RT_CONSISTENT);
877    }
878#endif
879
880    /* Notify MD code that a module is being unloaded. */
881    elf_cpu_unload_file(file);
882
883    if (ef->preloaded) {
884	link_elf_unload_preload(file);
885	return;
886    }
887
888#ifdef SPARSE_MAPPING
889    if (ef->object) {
890	vm_map_remove(kernel_map, (vm_offset_t) ef->address,
891		      (vm_offset_t) ef->address
892		      + (ef->object->size << PAGE_SHIFT));
893    }
894#else
895    if (ef->address)
896	free(ef->address, M_LINKER);
897#endif
898    if (ef->symbase)
899	free(ef->symbase, M_LINKER);
900    if (ef->strbase)
901	free(ef->strbase, M_LINKER);
902}
903
904static void
905link_elf_unload_preload(linker_file_t file)
906{
907    if (file->filename)
908	preload_delete_name(file->filename);
909}
910
911static const char *
912symbol_name(elf_file_t ef, Elf_Word r_info)
913{
914    const Elf_Sym *ref;
915
916    if (ELF_R_SYM(r_info)) {
917	ref = ef->symtab + ELF_R_SYM(r_info);
918	return ef->strtab + ref->st_name;
919    } else
920	return NULL;
921}
922
923static int
924relocate_file(elf_file_t ef)
925{
926    const Elf_Rel *rellim;
927    const Elf_Rel *rel;
928    const Elf_Rela *relalim;
929    const Elf_Rela *rela;
930    const char *symname;
931
932    /* Perform relocations without addend if there are any: */
933    rel = ef->rel;
934    if (rel) {
935	rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize);
936	while (rel < rellim) {
937	    if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rel, ELF_RELOC_REL,
938			  elf_lookup)) {
939		symname = symbol_name(ef, rel->r_info);
940		printf("link_elf: symbol %s undefined\n", symname);
941		return ENOENT;
942	    }
943	    rel++;
944	}
945    }
946
947    /* Perform relocations with addend if there are any: */
948    rela = ef->rela;
949    if (rela) {
950	relalim = (const Elf_Rela *)((const char *)ef->rela + ef->relasize);
951	while (rela < relalim) {
952	    if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rela, ELF_RELOC_RELA,
953			  elf_lookup)) {
954		symname = symbol_name(ef, rela->r_info);
955		printf("link_elf: symbol %s undefined\n", symname);
956		return ENOENT;
957	    }
958	    rela++;
959	}
960    }
961
962    /* Perform PLT relocations without addend if there are any: */
963    rel = ef->pltrel;
964    if (rel) {
965	rellim = (const Elf_Rel *)((const char *)ef->pltrel + ef->pltrelsize);
966	while (rel < rellim) {
967	    if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rel, ELF_RELOC_REL,
968			  elf_lookup)) {
969		symname = symbol_name(ef, rel->r_info);
970		printf("link_elf: symbol %s undefined\n", symname);
971		return ENOENT;
972	    }
973	    rel++;
974	}
975    }
976
977    /* Perform relocations with addend if there are any: */
978    rela = ef->pltrela;
979    if (rela) {
980	relalim = (const Elf_Rela *)((const char *)ef->pltrela + ef->pltrelasize);
981	while (rela < relalim) {
982	    if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rela, ELF_RELOC_RELA,
983			  elf_lookup)) {
984		symname = symbol_name(ef, rela->r_info);
985		printf("link_elf: symbol %s undefined\n", symname);
986		return ENOENT;
987	    }
988	    rela++;
989	}
990    }
991
992    return 0;
993}
994
995/*
996 * Hash function for symbol table lookup.  Don't even think about changing
997 * this.  It is specified by the System V ABI.
998 */
999static unsigned long
1000elf_hash(const char *name)
1001{
1002    const unsigned char *p = (const unsigned char *) name;
1003    unsigned long h = 0;
1004    unsigned long g;
1005
1006    while (*p != '\0') {
1007	h = (h << 4) + *p++;
1008	if ((g = h & 0xf0000000) != 0)
1009	    h ^= g >> 24;
1010	h &= ~g;
1011    }
1012    return h;
1013}
1014
1015static int
1016link_elf_lookup_symbol(linker_file_t lf, const char* name, c_linker_sym_t* sym)
1017{
1018    elf_file_t ef = (elf_file_t) lf;
1019    unsigned long symnum;
1020    const Elf_Sym* symp;
1021    const char *strp;
1022    unsigned long hash;
1023    int i;
1024
1025    /* If we don't have a hash, bail. */
1026    if (ef->buckets == NULL || ef->nbuckets == 0) {
1027	printf("link_elf_lookup_symbol: missing symbol hash table\n");
1028	return ENOENT;
1029    }
1030
1031    /* First, search hashed global symbols */
1032    hash = elf_hash(name);
1033    symnum = ef->buckets[hash % ef->nbuckets];
1034
1035    while (symnum != STN_UNDEF) {
1036	if (symnum >= ef->nchains) {
1037	    printf("link_elf_lookup_symbol: corrupt symbol table\n");
1038	    return ENOENT;
1039	}
1040
1041	symp = ef->symtab + symnum;
1042	if (symp->st_name == 0) {
1043	    printf("link_elf_lookup_symbol: corrupt symbol table\n");
1044	    return ENOENT;
1045	}
1046
1047	strp = ef->strtab + symp->st_name;
1048
1049	if (strcmp(name, strp) == 0) {
1050	    if (symp->st_shndx != SHN_UNDEF ||
1051		(symp->st_value != 0 &&
1052		 ELF_ST_TYPE(symp->st_info) == STT_FUNC)) {
1053		*sym = (c_linker_sym_t) symp;
1054		return 0;
1055	    } else
1056		return ENOENT;
1057	}
1058
1059	symnum = ef->chains[symnum];
1060    }
1061
1062    /* If we have not found it, look at the full table (if loaded) */
1063    if (ef->symtab == ef->ddbsymtab)
1064	return ENOENT;
1065
1066    /* Exhaustive search */
1067    for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
1068	strp = ef->ddbstrtab + symp->st_name;
1069	if (strcmp(name, strp) == 0) {
1070	    if (symp->st_shndx != SHN_UNDEF ||
1071		(symp->st_value != 0 &&
1072		 ELF_ST_TYPE(symp->st_info) == STT_FUNC)) {
1073		*sym = (c_linker_sym_t) symp;
1074		return 0;
1075	    } else
1076		return ENOENT;
1077	}
1078    }
1079
1080    return ENOENT;
1081}
1082
1083static int
1084link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, linker_symval_t* symval)
1085{
1086	elf_file_t ef = (elf_file_t) lf;
1087	const Elf_Sym* es = (const Elf_Sym*) sym;
1088
1089	if (es >= ef->symtab && es < (ef->symtab + ef->nchains)) {
1090	    symval->name = ef->strtab + es->st_name;
1091	    symval->value = (caddr_t) ef->address + es->st_value;
1092	    symval->size = es->st_size;
1093	    return 0;
1094	}
1095	if (ef->symtab == ef->ddbsymtab)
1096	    return ENOENT;
1097	if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {
1098	    symval->name = ef->ddbstrtab + es->st_name;
1099	    symval->value = (caddr_t) ef->address + es->st_value;
1100	    symval->size = es->st_size;
1101	    return 0;
1102	}
1103	return ENOENT;
1104}
1105
1106static int
1107link_elf_search_symbol(linker_file_t lf, caddr_t value,
1108		       c_linker_sym_t* sym, long* diffp)
1109{
1110	elf_file_t ef = (elf_file_t) lf;
1111	u_long off = (uintptr_t) (void *) value;
1112	u_long diff = off;
1113	u_long st_value;
1114	const Elf_Sym* es;
1115	const Elf_Sym* best = 0;
1116	int i;
1117
1118	for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) {
1119		if (es->st_name == 0)
1120			continue;
1121		st_value = es->st_value + (uintptr_t) (void *) ef->address;
1122		if (off >= st_value) {
1123			if (off - st_value < diff) {
1124				diff = off - st_value;
1125				best = es;
1126				if (diff == 0)
1127					break;
1128			} else if (off - st_value == diff) {
1129				best = es;
1130			}
1131		}
1132	}
1133	if (best == 0)
1134		*diffp = off;
1135	else
1136		*diffp = diff;
1137	*sym = (c_linker_sym_t) best;
1138
1139	return 0;
1140}
1141
1142/*
1143 * Look up a linker set on an ELF system.
1144 */
1145static int
1146link_elf_lookup_set(linker_file_t lf, const char *name,
1147		    void ***startp, void ***stopp, int *countp)
1148{
1149	c_linker_sym_t sym;
1150	linker_symval_t symval;
1151	char *setsym;
1152	void **start, **stop;
1153	int len, error = 0, count;
1154
1155	len = strlen(name) + sizeof("__start_set_"); /* sizeof includes \0 */
1156	setsym = malloc(len, M_LINKER, M_WAITOK);
1157	if (setsym == NULL)
1158		return ENOMEM;
1159
1160	/* get address of first entry */
1161	snprintf(setsym, len, "%s%s", "__start_set_", name);
1162	error = link_elf_lookup_symbol(lf, setsym, &sym);
1163	if (error)
1164		goto out;
1165	link_elf_symbol_values(lf, sym, &symval);
1166	if (symval.value == 0) {
1167		error = ESRCH;
1168		goto out;
1169	}
1170	start = (void **)symval.value;
1171
1172	/* get address of last entry */
1173	snprintf(setsym, len, "%s%s", "__stop_set_", name);
1174	error = link_elf_lookup_symbol(lf, setsym, &sym);
1175	if (error)
1176		goto out;
1177	link_elf_symbol_values(lf, sym, &symval);
1178	if (symval.value == 0) {
1179		error = ESRCH;
1180		goto out;
1181	}
1182	stop = (void **)symval.value;
1183
1184	/* and the number of entries */
1185	count = stop - start;
1186
1187	/* and copy out */
1188	if (startp)
1189		*startp = start;
1190	if (stopp)
1191		*stopp = stop;
1192	if (countp)
1193		*countp = count;
1194
1195out:
1196	free(setsym, M_LINKER);
1197	return error;
1198}
1199
1200static int
1201link_elf_each_function_name(linker_file_t file,
1202  int (*callback)(const char *, void *), void *opaque) {
1203    elf_file_t ef = (elf_file_t)file;
1204    const Elf_Sym* symp;
1205    int i, error;
1206
1207    /* Exhaustive search */
1208    for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
1209	if (symp->st_value != 0 &&
1210	    ELF_ST_TYPE(symp->st_info) == STT_FUNC) {
1211		error = callback(ef->ddbstrtab + symp->st_name, opaque);
1212		if (error)
1213		    return (error);
1214	}
1215    }
1216    return (0);
1217}
1218
1219#ifdef __ia64__
1220/*
1221 * Each KLD has its own GP. The GP value for each load module is given by
1222 * DT_PLTGOT on ia64. We need GP to construct function descriptors, but
1223 * don't have direct access to the ELF file structure. The link_elf_get_gp()
1224 * function returns the GP given a pointer to a generic linker file struct.
1225 */
1226Elf_Addr
1227link_elf_get_gp(linker_file_t lf)
1228{
1229	elf_file_t ef = (elf_file_t)lf;
1230	return (Elf_Addr)ef->got;
1231}
1232#endif
1233
1234const Elf_Sym *
1235elf_get_sym(linker_file_t lf, Elf_Word symidx)
1236{
1237	elf_file_t ef = (elf_file_t)lf;
1238
1239	if (symidx >= ef->nchains)
1240		return (NULL);
1241	return (ef->symtab + symidx);
1242}
1243
1244const char *
1245elf_get_symname(linker_file_t lf, Elf_Word symidx)
1246{
1247	elf_file_t ef = (elf_file_t)lf;
1248	const Elf_Sym *sym;
1249
1250	if (symidx >= ef->nchains)
1251		return (NULL);
1252	sym = ef->symtab + symidx;
1253	return (ef->strtab + sym->st_name);
1254}
1255
1256/*
1257 * Symbol lookup function that can be used when the symbol index is known (ie
1258 * in relocations). It uses the symbol index instead of doing a fully fledged
1259 * hash table based lookup when such is valid. For example for local symbols.
1260 * This is not only more efficient, it's also more correct. It's not always
1261 * the case that the symbol can be found through the hash table.
1262 */
1263static Elf_Addr
1264elf_lookup(linker_file_t lf, Elf_Word symidx, int deps)
1265{
1266	elf_file_t ef = (elf_file_t)lf;
1267	const Elf_Sym *sym;
1268	const char *symbol;
1269
1270	/* Don't even try to lookup the symbol if the index is bogus. */
1271	if (symidx >= ef->nchains)
1272		return (0);
1273
1274	sym = ef->symtab + symidx;
1275
1276	/*
1277	 * Don't do a full lookup when the symbol is local. It may even
1278	 * fail because it may not be found through the hash table.
1279	 */
1280	if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) {
1281		/* Force lookup failure when we have an insanity. */
1282		if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0)
1283			return (0);
1284		return ((Elf_Addr)ef->address + sym->st_value);
1285	}
1286
1287	/*
1288	 * XXX we can avoid doing a hash table based lookup for global
1289	 * symbols as well. This however is not always valid, so we'll
1290	 * just do it the hard way for now. Performance tweaks can
1291	 * always be added.
1292	 */
1293
1294	symbol = ef->strtab + sym->st_name;
1295
1296	/* Force a lookup failure if the symbol name is bogus. */
1297	if (*symbol == 0)
1298		return (0);
1299
1300	return ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps));
1301}
1302
1303static void
1304link_elf_reloc_local(linker_file_t lf)
1305{
1306    const Elf_Rel *rellim;
1307    const Elf_Rel *rel;
1308    const Elf_Rela *relalim;
1309    const Elf_Rela *rela;
1310    elf_file_t ef = (elf_file_t)lf;
1311
1312    /* Perform relocations without addend if there are any: */
1313    if ((rel = ef->rel) != NULL) {
1314	rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize);
1315	while (rel < rellim) {
1316	    elf_reloc_local(lf, (Elf_Addr)ef->address, rel, ELF_RELOC_REL,
1317			    elf_lookup);
1318	    rel++;
1319	}
1320    }
1321
1322    /* Perform relocations with addend if there are any: */
1323    if ((rela = ef->rela) != NULL) {
1324	relalim = (const Elf_Rela *)((const char *)ef->rela + ef->relasize);
1325	while (rela < relalim) {
1326	    elf_reloc_local(lf, (Elf_Addr)ef->address, rela, ELF_RELOC_RELA,
1327			    elf_lookup);
1328	    rela++;
1329	}
1330    }
1331}
1332