link_elf.c revision 131928 
1/*-
2 * Copyright (c) 1998-2000 Doug Rabson
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/kern/link_elf.c 131928 2004-07-10 21:37:30Z marcel $");
29
30#include "opt_gdb.h"
31#include "opt_mac.h"
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#ifdef GPROF
36#include <sys/gmon.h>
37#endif
38#include <sys/kernel.h>
39#include <sys/lock.h>
40#include <sys/mac.h>
41#include <sys/malloc.h>
42#include <sys/mutex.h>
43#include <sys/proc.h>
44#include <sys/namei.h>
45#include <sys/fcntl.h>
46#include <sys/vnode.h>
47#include <sys/linker.h>
48
49#include <machine/elf.h>
50
51#include <vm/vm.h>
52#include <vm/vm_param.h>
53#ifdef SPARSE_MAPPING
54#include <vm/vm_object.h>
55#include <vm/vm_kern.h>
56#include <vm/vm_extern.h>
57#endif
58#include <vm/pmap.h>
59#include <vm/vm_map.h>
60
61#include <sys/link_elf.h>
62
63#include "linker_if.h"
64
65typedef struct elf_file {
66    struct linker_file	lf;		/* Common fields */
67    int			preloaded;	/* Was file pre-loaded */
68    caddr_t		address;	/* Relocation address */
69#ifdef SPARSE_MAPPING
70    vm_object_t		object;		/* VM object to hold file pages */
71#endif
72    Elf_Dyn*		dynamic;	/* Symbol table etc. */
73    Elf_Hashelt		nbuckets;	/* DT_HASH info */
74    Elf_Hashelt		nchains;
75    const Elf_Hashelt*	buckets;
76    const Elf_Hashelt*	chains;
77    caddr_t		hash;
78    caddr_t		strtab;		/* DT_STRTAB */
79    int			strsz;		/* DT_STRSZ */
80    const Elf_Sym*	symtab;		/* DT_SYMTAB */
81    Elf_Addr*		got;		/* DT_PLTGOT */
82    const Elf_Rel*	pltrel;		/* DT_JMPREL */
83    int			pltrelsize;	/* DT_PLTRELSZ */
84    const Elf_Rela*	pltrela;	/* DT_JMPREL */
85    int			pltrelasize;	/* DT_PLTRELSZ */
86    const Elf_Rel*	rel;		/* DT_REL */
87    int			relsize;	/* DT_RELSZ */
88    const Elf_Rela*	rela;		/* DT_RELA */
89    int			relasize;	/* DT_RELASZ */
90    caddr_t		modptr;
91    const Elf_Sym*	ddbsymtab;	/* The symbol table we are using */
92    long		ddbsymcnt;	/* Number of symbols */
93    caddr_t		ddbstrtab;	/* String table */
94    long		ddbstrcnt;	/* number of bytes in string table */
95    caddr_t		symbase;	/* malloc'ed symbold base */
96    caddr_t		strbase;	/* malloc'ed string base */
97#ifdef GDB
98    struct link_map	gdb;		/* hooks for gdb */
99#endif
100} *elf_file_t;
101
102static int	link_elf_link_common_finish(linker_file_t);
103static int	link_elf_link_preload(linker_class_t cls,
104				      const char*, linker_file_t*);
105static int	link_elf_link_preload_finish(linker_file_t);
106static int	link_elf_load_file(linker_class_t, const char*, linker_file_t*);
107static int	link_elf_lookup_symbol(linker_file_t, const char*,
108				       c_linker_sym_t*);
109static int	link_elf_symbol_values(linker_file_t, c_linker_sym_t, linker_symval_t*);
110static int	link_elf_search_symbol(linker_file_t, caddr_t value,
111				       c_linker_sym_t* sym, long* diffp);
112
113static void	link_elf_unload_file(linker_file_t);
114static void	link_elf_unload_preload(linker_file_t);
115static int	link_elf_lookup_set(linker_file_t, const char *,
116				    void ***, void ***, int *);
117static int	link_elf_each_function_name(linker_file_t,
118				int (*)(const char *, void *),
119				void *);
120static void	link_elf_reloc_local(linker_file_t);
121static Elf_Addr	elf_lookup(linker_file_t lf, Elf_Word symidx, int deps);
122
123static kobj_method_t link_elf_methods[] = {
124    KOBJMETHOD(linker_lookup_symbol,	link_elf_lookup_symbol),
125    KOBJMETHOD(linker_symbol_values,	link_elf_symbol_values),
126    KOBJMETHOD(linker_search_symbol,	link_elf_search_symbol),
127    KOBJMETHOD(linker_unload,		link_elf_unload_file),
128    KOBJMETHOD(linker_load_file,	link_elf_load_file),
129    KOBJMETHOD(linker_link_preload,	link_elf_link_preload),
130    KOBJMETHOD(linker_link_preload_finish, link_elf_link_preload_finish),
131    KOBJMETHOD(linker_lookup_set,	link_elf_lookup_set),
132    KOBJMETHOD(linker_each_function_name, link_elf_each_function_name),
133    { 0, 0 }
134};
135
136static struct linker_class link_elf_class = {
137#if ELF_TARG_CLASS == ELFCLASS32
138    "elf32",
139#else
140    "elf64",
141#endif
142    link_elf_methods, sizeof(struct elf_file)
143};
144
145static int		parse_dynamic(elf_file_t ef);
146static int		relocate_file(elf_file_t ef);
147static int		link_elf_preload_parse_symbols(elf_file_t ef);
148
149#ifdef GDB
150static void		r_debug_state(struct r_debug *dummy_one,
151				      struct link_map *dummy_two);
152
153/*
154 * A list of loaded modules for GDB to use for loading symbols.
155 */
156struct r_debug r_debug;
157
158#define GDB_STATE(s)	r_debug.r_state = s; r_debug_state(NULL, NULL);
159
160/*
161 * Function for the debugger to set a breakpoint on to gain control.
162 */
163static void
164r_debug_state(struct r_debug *dummy_one __unused,
165	      struct link_map *dummy_two __unused)
166{
167}
168
169static void
170link_elf_add_gdb(struct link_map *l)
171{
172    struct link_map *prev;
173
174    l->l_next = NULL;
175
176    if (r_debug.r_map == NULL) {
177	/* Add first. */
178	l->l_prev = NULL;
179	r_debug.r_map = l;
180    } else {
181	/* Append to list. */
182	for (prev = r_debug.r_map; prev->l_next != NULL; prev = prev->l_next)
183	    ;
184	l->l_prev = prev;
185	prev->l_next = l;
186    }
187}
188
189static void
190link_elf_delete_gdb(struct link_map *l)
191{
192    if (l->l_prev == NULL) {
193	/* Remove first. */
194	if ((r_debug.r_map = l->l_next) != NULL)
195	    l->l_next->l_prev = NULL;
196    } else {
197	/* Remove any but first. */
198	if ((l->l_prev->l_next = l->l_next) != NULL)
199	    l->l_next->l_prev = l->l_prev;
200    }
201}
202#endif /* GDB */
203
204#ifdef __ia64__
205Elf_Addr link_elf_get_gp(linker_file_t);
206#endif
207
208/*
209 * The kernel symbol table starts here.
210 */
211extern struct _dynamic _DYNAMIC;
212
213static void
214link_elf_error(const char *s)
215{
216    printf("kldload: %s\n", s);
217}
218
219/*
220 * Actions performed after linking/loading both the preloaded kernel and any
221 * modules; whether preloaded or dynamicly loaded.
222 */
223static int
224link_elf_link_common_finish(linker_file_t lf)
225{
226#ifdef GDB
227    elf_file_t ef = (elf_file_t)lf;
228    char *newfilename;
229#endif
230    int error;
231
232    /* Notify MD code that a module is being loaded. */
233    error = elf_cpu_load_file(lf);
234    if (error)
235	return (error);
236
237#ifdef GDB
238    GDB_STATE(RT_ADD);
239    ef->gdb.l_addr = lf->address;
240    newfilename = malloc(strlen(lf->filename) + 1, M_LINKER, M_WAITOK);
241    strcpy(newfilename, lf->filename);
242    ef->gdb.l_name = newfilename;
243    ef->gdb.l_ld = ef->dynamic;
244    link_elf_add_gdb(&ef->gdb);
245    GDB_STATE(RT_CONSISTENT);
246#endif
247
248    return (0);
249}
250
251static void
252link_elf_init(void* arg)
253{
254    Elf_Dyn	*dp;
255    caddr_t	modptr, baseptr, sizeptr;
256    elf_file_t	ef;
257    char	*modname;
258
259    linker_add_class(&link_elf_class);
260
261    dp = (Elf_Dyn*) &_DYNAMIC;
262    modname = NULL;
263    modptr = preload_search_by_type("elf" __XSTRING(__ELF_WORD_SIZE) " kernel");
264    if (modptr == NULL)
265	modptr = preload_search_by_type("elf kernel");
266    if (modptr)
267	modname = (char *)preload_search_info(modptr, MODINFO_NAME);
268    if (modname == NULL)
269	modname = "kernel";
270    linker_kernel_file = linker_make_file(modname, &link_elf_class);
271    if (linker_kernel_file == NULL)
272	panic("link_elf_init: Can't create linker structures for kernel");
273
274    ef = (elf_file_t) linker_kernel_file;
275    ef->preloaded = 1;
276    ef->address = 0;
277#ifdef SPARSE_MAPPING
278    ef->object = 0;
279#endif
280    ef->dynamic = dp;
281
282    if (dp)
283	parse_dynamic(ef);
284    linker_kernel_file->address = (caddr_t) KERNBASE;
285    linker_kernel_file->size = -(intptr_t)linker_kernel_file->address;
286
287    if (modptr) {
288	ef->modptr = modptr;
289	baseptr = preload_search_info(modptr, MODINFO_ADDR);
290	if (baseptr)
291	    linker_kernel_file->address = *(caddr_t *)baseptr;
292	sizeptr = preload_search_info(modptr, MODINFO_SIZE);
293	if (sizeptr)
294	    linker_kernel_file->size = *(size_t *)sizeptr;
295    }
296    (void)link_elf_preload_parse_symbols(ef);
297
298#ifdef GDB
299    r_debug.r_map = NULL;
300    r_debug.r_brk = r_debug_state;
301    r_debug.r_state = RT_CONSISTENT;
302#endif
303
304    (void)link_elf_link_common_finish(linker_kernel_file);
305}
306
307SYSINIT(link_elf, SI_SUB_KLD, SI_ORDER_SECOND, link_elf_init, 0);
308
309static int
310link_elf_preload_parse_symbols(elf_file_t ef)
311{
312    caddr_t	pointer;
313    caddr_t	ssym, esym, base;
314    caddr_t	strtab;
315    int		strcnt;
316    Elf_Sym*	symtab;
317    int		symcnt;
318
319    if (ef->modptr == NULL)
320	return 0;
321    pointer = preload_search_info(ef->modptr, MODINFO_METADATA|MODINFOMD_SSYM);
322    if (pointer == NULL)
323	return 0;
324    ssym = *(caddr_t *)pointer;
325    pointer = preload_search_info(ef->modptr, MODINFO_METADATA|MODINFOMD_ESYM);
326    if (pointer == NULL)
327	return 0;
328    esym = *(caddr_t *)pointer;
329
330    base = ssym;
331
332    symcnt = *(long *)base;
333    base += sizeof(long);
334    symtab = (Elf_Sym *)base;
335    base += roundup(symcnt, sizeof(long));
336
337    if (base > esym || base < ssym) {
338	printf("Symbols are corrupt!\n");
339	return EINVAL;
340    }
341
342    strcnt = *(long *)base;
343    base += sizeof(long);
344    strtab = base;
345    base += roundup(strcnt, sizeof(long));
346
347    if (base > esym || base < ssym) {
348	printf("Symbols are corrupt!\n");
349	return EINVAL;
350    }
351
352    ef->ddbsymtab = symtab;
353    ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);
354    ef->ddbstrtab = strtab;
355    ef->ddbstrcnt = strcnt;
356
357    return 0;
358}
359
360static int
361parse_dynamic(elf_file_t ef)
362{
363    Elf_Dyn *dp;
364    int plttype = DT_REL;
365
366    for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {
367	switch (dp->d_tag) {
368	case DT_HASH:
369	{
370	    /* From src/libexec/rtld-elf/rtld.c */
371	    const Elf_Hashelt *hashtab = (const Elf_Hashelt *)
372		(ef->address + dp->d_un.d_ptr);
373	    ef->nbuckets = hashtab[0];
374	    ef->nchains = hashtab[1];
375	    ef->buckets = hashtab + 2;
376	    ef->chains = ef->buckets + ef->nbuckets;
377	    break;
378	}
379	case DT_STRTAB:
380	    ef->strtab = (caddr_t) (ef->address + dp->d_un.d_ptr);
381	    break;
382	case DT_STRSZ:
383	    ef->strsz = dp->d_un.d_val;
384	    break;
385	case DT_SYMTAB:
386	    ef->symtab = (Elf_Sym*) (ef->address + dp->d_un.d_ptr);
387	    break;
388	case DT_SYMENT:
389	    if (dp->d_un.d_val != sizeof(Elf_Sym))
390		return ENOEXEC;
391	    break;
392	case DT_PLTGOT:
393	    ef->got = (Elf_Addr *) (ef->address + dp->d_un.d_ptr);
394	    break;
395	case DT_REL:
396	    ef->rel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr);
397	    break;
398	case DT_RELSZ:
399	    ef->relsize = dp->d_un.d_val;
400	    break;
401	case DT_RELENT:
402	    if (dp->d_un.d_val != sizeof(Elf_Rel))
403		return ENOEXEC;
404	    break;
405	case DT_JMPREL:
406	    ef->pltrel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr);
407	    break;
408	case DT_PLTRELSZ:
409	    ef->pltrelsize = dp->d_un.d_val;
410	    break;
411	case DT_RELA:
412	    ef->rela = (const Elf_Rela *) (ef->address + dp->d_un.d_ptr);
413	    break;
414	case DT_RELASZ:
415	    ef->relasize = dp->d_un.d_val;
416	    break;
417	case DT_RELAENT:
418	    if (dp->d_un.d_val != sizeof(Elf_Rela))
419		return ENOEXEC;
420	    break;
421	case DT_PLTREL:
422	    plttype = dp->d_un.d_val;
423	    if (plttype != DT_REL && plttype != DT_RELA)
424		return ENOEXEC;
425	    break;
426#ifdef GDB
427	case DT_DEBUG:
428	    dp->d_un.d_ptr = (Elf_Addr) &r_debug;
429	    break;
430#endif
431	}
432    }
433
434    if (plttype == DT_RELA) {
435	ef->pltrela = (const Elf_Rela *) ef->pltrel;
436	ef->pltrel = NULL;
437	ef->pltrelasize = ef->pltrelsize;
438	ef->pltrelsize = 0;
439    }
440
441    ef->ddbsymtab = ef->symtab;
442    ef->ddbsymcnt = ef->nchains;
443    ef->ddbstrtab = ef->strtab;
444    ef->ddbstrcnt = ef->strsz;
445
446    return 0;
447}
448
449static int
450link_elf_link_preload(linker_class_t cls,
451		      const char* filename, linker_file_t *result)
452{
453    caddr_t		modptr, baseptr, sizeptr, dynptr;
454    char		*type;
455    elf_file_t		ef;
456    linker_file_t	lf;
457    int			error;
458    vm_offset_t		dp;
459
460    /* Look to see if we have the file preloaded */
461    modptr = preload_search_by_name(filename);
462    if (modptr == NULL)
463	return ENOENT;
464
465    type = (char *)preload_search_info(modptr, MODINFO_TYPE);
466    baseptr = preload_search_info(modptr, MODINFO_ADDR);
467    sizeptr = preload_search_info(modptr, MODINFO_SIZE);
468    dynptr = preload_search_info(modptr, MODINFO_METADATA|MODINFOMD_DYNAMIC);
469    if (type == NULL ||
470	(strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE) " module") != 0 &&
471	 strcmp(type, "elf module") != 0))
472	return (EFTYPE);
473    if (baseptr == NULL || sizeptr == NULL || dynptr == NULL)
474	return (EINVAL);
475
476    lf = linker_make_file(filename, &link_elf_class);
477    if (lf == NULL) {
478	return ENOMEM;
479    }
480
481    ef = (elf_file_t) lf;
482    ef->preloaded = 1;
483    ef->modptr = modptr;
484    ef->address = *(caddr_t *)baseptr;
485#ifdef SPARSE_MAPPING
486    ef->object = 0;
487#endif
488    dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr;
489    ef->dynamic = (Elf_Dyn *)dp;
490    lf->address = ef->address;
491    lf->size = *(size_t *)sizeptr;
492
493    error = parse_dynamic(ef);
494    if (error) {
495	linker_file_unload(lf);
496	return error;
497    }
498    link_elf_reloc_local(lf);
499    *result = lf;
500    return (0);
501}
502
503static int
504link_elf_link_preload_finish(linker_file_t lf)
505{
506    elf_file_t		ef;
507    int error;
508
509    ef = (elf_file_t) lf;
510#if 0	/* this will be more trouble than it's worth for now */
511    for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {
512	if (dp->d_tag != DT_NEEDED)
513	    continue;
514	modname = ef->strtab + dp->d_un.d_val;
515	error = linker_load_module(modname, lf);
516	if (error)
517	    goto out;
518    }
519#endif
520    error = relocate_file(ef);
521    if (error)
522	return error;
523    (void)link_elf_preload_parse_symbols(ef);
524
525    return (link_elf_link_common_finish(lf));
526}
527
528static int
529link_elf_load_file(linker_class_t cls, const char* filename,
530	linker_file_t* result)
531{
532    struct nameidata nd;
533    struct thread* td = curthread;	/* XXX */
534    Elf_Ehdr *hdr;
535    caddr_t firstpage;
536    int nbytes, i;
537    Elf_Phdr *phdr;
538    Elf_Phdr *phlimit;
539    Elf_Phdr *segs[2];
540    int nsegs;
541    Elf_Phdr *phdyn;
542    Elf_Phdr *phphdr;
543    caddr_t mapbase;
544    size_t mapsize;
545    Elf_Off base_offset;
546    Elf_Addr base_vaddr;
547    Elf_Addr base_vlimit;
548    int error = 0;
549    int resid, flags;
550    elf_file_t ef;
551    linker_file_t lf;
552    Elf_Shdr *shdr;
553    int symtabindex;
554    int symstrindex;
555    int symcnt;
556    int strcnt;
557
558    GIANT_REQUIRED;
559
560    shdr = NULL;
561    lf = NULL;
562
563    NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td);
564    flags = FREAD;
565    error = vn_open(&nd, &flags, 0, -1);
566    if (error)
567	return error;
568    NDFREE(&nd, NDF_ONLY_PNBUF);
569#ifdef MAC
570    error = mac_check_kld_load(curthread->td_ucred, nd.ni_vp);
571    if (error) {
572	firstpage = NULL;
573	goto out;
574    }
575#endif
576
577    /*
578     * Read the elf header from the file.
579     */
580    firstpage = malloc(PAGE_SIZE, M_LINKER, M_WAITOK);
581    if (firstpage == NULL) {
582	error = ENOMEM;
583	goto out;
584    }
585    hdr = (Elf_Ehdr *)firstpage;
586    error = vn_rdwr(UIO_READ, nd.ni_vp, firstpage, PAGE_SIZE, 0,
587		    UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
588		    &resid, td);
589    nbytes = PAGE_SIZE - resid;
590    if (error)
591	goto out;
592
593    if (!IS_ELF(*hdr)) {
594	error = ENOEXEC;
595	goto out;
596    }
597
598    if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS
599      || hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
600	link_elf_error("Unsupported file layout");
601	error = ENOEXEC;
602	goto out;
603    }
604    if (hdr->e_ident[EI_VERSION] != EV_CURRENT
605      || hdr->e_version != EV_CURRENT) {
606	link_elf_error("Unsupported file version");
607	error = ENOEXEC;
608	goto out;
609    }
610    if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) {
611	link_elf_error("Unsupported file type");
612	error = ENOEXEC;
613	goto out;
614    }
615    if (hdr->e_machine != ELF_TARG_MACH) {
616	link_elf_error("Unsupported machine");
617	error = ENOEXEC;
618	goto out;
619    }
620
621    /*
622     * We rely on the program header being in the first page.  This is
623     * not strictly required by the ABI specification, but it seems to
624     * always true in practice.  And, it simplifies things considerably.
625     */
626    if (!((hdr->e_phentsize == sizeof(Elf_Phdr)) &&
627	  (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= PAGE_SIZE) &&
628	  (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= nbytes)))
629	link_elf_error("Unreadable program headers");
630
631    /*
632     * Scan the program header entries, and save key information.
633     *
634     * We rely on there being exactly two load segments, text and data,
635     * in that order.
636     */
637    phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff);
638    phlimit = phdr + hdr->e_phnum;
639    nsegs = 0;
640    phdyn = NULL;
641    phphdr = NULL;
642    while (phdr < phlimit) {
643	switch (phdr->p_type) {
644
645	case PT_LOAD:
646	    if (nsegs == 2) {
647		link_elf_error("Too many sections");
648		error = ENOEXEC;
649		goto out;
650	    }
651	    /*
652	     * XXX: We just trust they come in right order ??
653	     */
654	    segs[nsegs] = phdr;
655	    ++nsegs;
656	    break;
657
658	case PT_PHDR:
659	    phphdr = phdr;
660	    break;
661
662	case PT_DYNAMIC:
663	    phdyn = phdr;
664	    break;
665
666	case PT_INTERP:
667	    link_elf_error("Unsupported file type");
668	    error = ENOEXEC;
669	    goto out;
670	}
671
672	++phdr;
673    }
674    if (phdyn == NULL) {
675	link_elf_error("Object is not dynamically-linked");
676	error = ENOEXEC;
677	goto out;
678    }
679    if (nsegs != 2) {
680	link_elf_error("Too few sections");
681	error = ENOEXEC;
682	goto out;
683    }
684
685    /*
686     * Allocate the entire address space of the object, to stake out our
687     * contiguous region, and to establish the base address for relocation.
688     */
689    base_offset = trunc_page(segs[0]->p_offset);
690    base_vaddr = trunc_page(segs[0]->p_vaddr);
691    base_vlimit = round_page(segs[1]->p_vaddr + segs[1]->p_memsz);
692    mapsize = base_vlimit - base_vaddr;
693
694    lf = linker_make_file(filename, &link_elf_class);
695    if (!lf) {
696	error = ENOMEM;
697	goto out;
698    }
699
700    ef = (elf_file_t) lf;
701#ifdef SPARSE_MAPPING
702    ef->object = vm_object_allocate(OBJT_DEFAULT, mapsize >> PAGE_SHIFT);
703    if (ef->object == NULL) {
704	error = ENOMEM;
705	goto out;
706    }
707    vm_object_reference(ef->object);
708    ef->address = (caddr_t) vm_map_min(kernel_map);
709    error = vm_map_find(kernel_map, ef->object, 0,
710			(vm_offset_t *) &ef->address,
711			mapsize, 1,
712			VM_PROT_ALL, VM_PROT_ALL, 0);
713    if (error) {
714	vm_object_deallocate(ef->object);
715	ef->object = 0;
716	goto out;
717    }
718#else
719    ef->address = malloc(mapsize, M_LINKER, M_WAITOK);
720    if (!ef->address) {
721	error = ENOMEM;
722	goto out;
723    }
724#endif
725    mapbase = ef->address;
726
727    /*
728     * Read the text and data sections and zero the bss.
729     */
730    for (i = 0; i < 2; i++) {
731	caddr_t segbase = mapbase + segs[i]->p_vaddr - base_vaddr;
732	error = vn_rdwr(UIO_READ, nd.ni_vp,
733			segbase, segs[i]->p_filesz, segs[i]->p_offset,
734			UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
735			&resid, td);
736	if (error) {
737	    goto out;
738	}
739	bzero(segbase + segs[i]->p_filesz,
740	      segs[i]->p_memsz - segs[i]->p_filesz);
741
742#ifdef SPARSE_MAPPING
743	/*
744	 * Wire down the pages
745	 */
746	vm_map_wire(kernel_map,
747		    (vm_offset_t) segbase,
748		    (vm_offset_t) segbase + segs[i]->p_memsz,
749		    VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES);
750#endif
751    }
752
753#ifdef GPROF
754    /* Update profiling information with the new text segment. */
755    kmupetext((uintfptr_t)(mapbase + segs[0]->p_vaddr - base_vaddr +
756	segs[0]->p_memsz));
757#endif
758
759    ef->dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr);
760
761    lf->address = ef->address;
762    lf->size = mapsize;
763
764    error = parse_dynamic(ef);
765    if (error)
766	goto out;
767    link_elf_reloc_local(lf);
768
769    error = linker_load_dependencies(lf);
770    if (error)
771	goto out;
772#if 0	/* this will be more trouble than it's worth for now */
773    for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {
774	if (dp->d_tag != DT_NEEDED)
775	    continue;
776	modname = ef->strtab + dp->d_un.d_val;
777	error = linker_load_module(modname, lf);
778	if (error)
779	    goto out;
780    }
781#endif
782    error = relocate_file(ef);
783    if (error)
784	goto out;
785
786    /* Try and load the symbol table if it's present.  (you can strip it!) */
787    nbytes = hdr->e_shnum * hdr->e_shentsize;
788    if (nbytes == 0 || hdr->e_shoff == 0)
789	goto nosyms;
790    shdr = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO);
791    if (shdr == NULL) {
792	error = ENOMEM;
793	goto out;
794    }
795    error = vn_rdwr(UIO_READ, nd.ni_vp,
796		    (caddr_t)shdr, nbytes, hdr->e_shoff,
797		    UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
798		    &resid, td);
799    if (error)
800	goto out;
801    symtabindex = -1;
802    symstrindex = -1;
803    for (i = 0; i < hdr->e_shnum; i++) {
804	if (shdr[i].sh_type == SHT_SYMTAB) {
805	    symtabindex = i;
806	    symstrindex = shdr[i].sh_link;
807	}
808    }
809    if (symtabindex < 0 || symstrindex < 0)
810	goto nosyms;
811
812    symcnt = shdr[symtabindex].sh_size;
813    ef->symbase = malloc(symcnt, M_LINKER, M_WAITOK);
814    strcnt = shdr[symstrindex].sh_size;
815    ef->strbase = malloc(strcnt, M_LINKER, M_WAITOK);
816
817    if (ef->symbase == NULL || ef->strbase == NULL) {
818	error = ENOMEM;
819	goto out;
820    }
821    error = vn_rdwr(UIO_READ, nd.ni_vp,
822		    ef->symbase, symcnt, shdr[symtabindex].sh_offset,
823		    UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
824		    &resid, td);
825    if (error)
826	goto out;
827    error = vn_rdwr(UIO_READ, nd.ni_vp,
828		    ef->strbase, strcnt, shdr[symstrindex].sh_offset,
829		    UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
830		    &resid, td);
831    if (error)
832	goto out;
833
834    ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);
835    ef->ddbsymtab = (const Elf_Sym *)ef->symbase;
836    ef->ddbstrcnt = strcnt;
837    ef->ddbstrtab = ef->strbase;
838
839    error = link_elf_link_common_finish(lf);
840    if (error)
841	goto out;
842
843nosyms:
844
845    *result = lf;
846
847out:
848    if (error && lf)
849	linker_file_unload(lf);
850    if (shdr)
851	free(shdr, M_LINKER);
852    if (firstpage)
853	free(firstpage, M_LINKER);
854    VOP_UNLOCK(nd.ni_vp, 0, td);
855    vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
856
857    return error;
858}
859
860static void
861link_elf_unload_file(linker_file_t file)
862{
863    elf_file_t ef = (elf_file_t) file;
864
865#ifdef GDB
866    if (ef->gdb.l_ld) {
867	GDB_STATE(RT_DELETE);
868	free((void *)(uintptr_t)ef->gdb.l_name, M_LINKER);
869	link_elf_delete_gdb(&ef->gdb);
870	GDB_STATE(RT_CONSISTENT);
871    }
872#endif
873
874    /* Notify MD code that a module is being unloaded. */
875    elf_cpu_unload_file(file);
876
877    if (ef->preloaded) {
878	link_elf_unload_preload(file);
879	return;
880    }
881
882#ifdef SPARSE_MAPPING
883    if (ef->object) {
884	vm_map_remove(kernel_map, (vm_offset_t) ef->address,
885		      (vm_offset_t) ef->address
886		      + (ef->object->size << PAGE_SHIFT));
887	vm_object_deallocate(ef->object);
888    }
889#else
890    if (ef->address)
891	free(ef->address, M_LINKER);
892#endif
893    if (ef->symbase)
894	free(ef->symbase, M_LINKER);
895    if (ef->strbase)
896	free(ef->strbase, M_LINKER);
897}
898
899static void
900link_elf_unload_preload(linker_file_t file)
901{
902    if (file->filename)
903	preload_delete_name(file->filename);
904}
905
906static const char *
907symbol_name(elf_file_t ef, Elf_Word r_info)
908{
909    const Elf_Sym *ref;
910
911    if (ELF_R_SYM(r_info)) {
912	ref = ef->symtab + ELF_R_SYM(r_info);
913	return ef->strtab + ref->st_name;
914    } else
915	return NULL;
916}
917
918static int
919relocate_file(elf_file_t ef)
920{
921    const Elf_Rel *rellim;
922    const Elf_Rel *rel;
923    const Elf_Rela *relalim;
924    const Elf_Rela *rela;
925    const char *symname;
926
927    /* Perform relocations without addend if there are any: */
928    rel = ef->rel;
929    if (rel) {
930	rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize);
931	while (rel < rellim) {
932	    if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rel, ELF_RELOC_REL,
933			  elf_lookup)) {
934		symname = symbol_name(ef, rel->r_info);
935		printf("link_elf: symbol %s undefined\n", symname);
936		return ENOENT;
937	    }
938	    rel++;
939	}
940    }
941
942    /* Perform relocations with addend if there are any: */
943    rela = ef->rela;
944    if (rela) {
945	relalim = (const Elf_Rela *)((const char *)ef->rela + ef->relasize);
946	while (rela < relalim) {
947	    if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rela, ELF_RELOC_RELA,
948			  elf_lookup)) {
949		symname = symbol_name(ef, rela->r_info);
950		printf("link_elf: symbol %s undefined\n", symname);
951		return ENOENT;
952	    }
953	    rela++;
954	}
955    }
956
957    /* Perform PLT relocations without addend if there are any: */
958    rel = ef->pltrel;
959    if (rel) {
960	rellim = (const Elf_Rel *)((const char *)ef->pltrel + ef->pltrelsize);
961	while (rel < rellim) {
962	    if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rel, ELF_RELOC_REL,
963			  elf_lookup)) {
964		symname = symbol_name(ef, rel->r_info);
965		printf("link_elf: symbol %s undefined\n", symname);
966		return ENOENT;
967	    }
968	    rel++;
969	}
970    }
971
972    /* Perform relocations with addend if there are any: */
973    rela = ef->pltrela;
974    if (rela) {
975	relalim = (const Elf_Rela *)((const char *)ef->pltrela + ef->pltrelasize);
976	while (rela < relalim) {
977	    if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rela, ELF_RELOC_RELA,
978			  elf_lookup)) {
979		symname = symbol_name(ef, rela->r_info);
980		printf("link_elf: symbol %s undefined\n", symname);
981		return ENOENT;
982	    }
983	    rela++;
984	}
985    }
986
987    return 0;
988}
989
990/*
991 * Hash function for symbol table lookup.  Don't even think about changing
992 * this.  It is specified by the System V ABI.
993 */
994static unsigned long
995elf_hash(const char *name)
996{
997    const unsigned char *p = (const unsigned char *) name;
998    unsigned long h = 0;
999    unsigned long g;
1000
1001    while (*p != '\0') {
1002	h = (h << 4) + *p++;
1003	if ((g = h & 0xf0000000) != 0)
1004	    h ^= g >> 24;
1005	h &= ~g;
1006    }
1007    return h;
1008}
1009
1010static int
1011link_elf_lookup_symbol(linker_file_t lf, const char* name, c_linker_sym_t* sym)
1012{
1013    elf_file_t ef = (elf_file_t) lf;
1014    unsigned long symnum;
1015    const Elf_Sym* symp;
1016    const char *strp;
1017    unsigned long hash;
1018    int i;
1019
1020    /* First, search hashed global symbols */
1021    hash = elf_hash(name);
1022    symnum = ef->buckets[hash % ef->nbuckets];
1023
1024    while (symnum != STN_UNDEF) {
1025	if (symnum >= ef->nchains) {
1026	    printf("link_elf_lookup_symbol: corrupt symbol table\n");
1027	    return ENOENT;
1028	}
1029
1030	symp = ef->symtab + symnum;
1031	if (symp->st_name == 0) {
1032	    printf("link_elf_lookup_symbol: corrupt symbol table\n");
1033	    return ENOENT;
1034	}
1035
1036	strp = ef->strtab + symp->st_name;
1037
1038	if (strcmp(name, strp) == 0) {
1039	    if (symp->st_shndx != SHN_UNDEF ||
1040		(symp->st_value != 0 &&
1041		 ELF_ST_TYPE(symp->st_info) == STT_FUNC)) {
1042		*sym = (c_linker_sym_t) symp;
1043		return 0;
1044	    } else
1045		return ENOENT;
1046	}
1047
1048	symnum = ef->chains[symnum];
1049    }
1050
1051    /* If we have not found it, look at the full table (if loaded) */
1052    if (ef->symtab == ef->ddbsymtab)
1053	return ENOENT;
1054
1055    /* Exhaustive search */
1056    for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
1057	strp = ef->ddbstrtab + symp->st_name;
1058	if (strcmp(name, strp) == 0) {
1059	    if (symp->st_shndx != SHN_UNDEF ||
1060		(symp->st_value != 0 &&
1061		 ELF_ST_TYPE(symp->st_info) == STT_FUNC)) {
1062		*sym = (c_linker_sym_t) symp;
1063		return 0;
1064	    } else
1065		return ENOENT;
1066	}
1067    }
1068
1069    return ENOENT;
1070}
1071
1072static int
1073link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, linker_symval_t* symval)
1074{
1075	elf_file_t ef = (elf_file_t) lf;
1076	const Elf_Sym* es = (const Elf_Sym*) sym;
1077
1078	if (es >= ef->symtab && es < (ef->symtab + ef->nchains)) {
1079	    symval->name = ef->strtab + es->st_name;
1080	    symval->value = (caddr_t) ef->address + es->st_value;
1081	    symval->size = es->st_size;
1082	    return 0;
1083	}
1084	if (ef->symtab == ef->ddbsymtab)
1085	    return ENOENT;
1086	if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {
1087	    symval->name = ef->ddbstrtab + es->st_name;
1088	    symval->value = (caddr_t) ef->address + es->st_value;
1089	    symval->size = es->st_size;
1090	    return 0;
1091	}
1092	return ENOENT;
1093}
1094
1095static int
1096link_elf_search_symbol(linker_file_t lf, caddr_t value,
1097		       c_linker_sym_t* sym, long* diffp)
1098{
1099	elf_file_t ef = (elf_file_t) lf;
1100	u_long off = (uintptr_t) (void *) value;
1101	u_long diff = off;
1102	u_long st_value;
1103	const Elf_Sym* es;
1104	const Elf_Sym* best = 0;
1105	int i;
1106
1107	for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) {
1108		if (es->st_name == 0)
1109			continue;
1110		st_value = es->st_value + (uintptr_t) (void *) ef->address;
1111		if (off >= st_value) {
1112			if (off - st_value < diff) {
1113				diff = off - st_value;
1114				best = es;
1115				if (diff == 0)
1116					break;
1117			} else if (off - st_value == diff) {
1118				best = es;
1119			}
1120		}
1121	}
1122	if (best == 0)
1123		*diffp = off;
1124	else
1125		*diffp = diff;
1126	*sym = (c_linker_sym_t) best;
1127
1128	return 0;
1129}
1130
1131/*
1132 * Look up a linker set on an ELF system.
1133 */
1134static int
1135link_elf_lookup_set(linker_file_t lf, const char *name,
1136		    void ***startp, void ***stopp, int *countp)
1137{
1138	c_linker_sym_t sym;
1139	linker_symval_t symval;
1140	char *setsym;
1141	void **start, **stop;
1142	int len, error = 0, count;
1143
1144	len = strlen(name) + sizeof("__start_set_"); /* sizeof includes \0 */
1145	setsym = malloc(len, M_LINKER, M_WAITOK);
1146	if (setsym == NULL)
1147		return ENOMEM;
1148
1149	/* get address of first entry */
1150	snprintf(setsym, len, "%s%s", "__start_set_", name);
1151	error = link_elf_lookup_symbol(lf, setsym, &sym);
1152	if (error)
1153		goto out;
1154	link_elf_symbol_values(lf, sym, &symval);
1155	if (symval.value == 0) {
1156		error = ESRCH;
1157		goto out;
1158	}
1159	start = (void **)symval.value;
1160
1161	/* get address of last entry */
1162	snprintf(setsym, len, "%s%s", "__stop_set_", name);
1163	error = link_elf_lookup_symbol(lf, setsym, &sym);
1164	if (error)
1165		goto out;
1166	link_elf_symbol_values(lf, sym, &symval);
1167	if (symval.value == 0) {
1168		error = ESRCH;
1169		goto out;
1170	}
1171	stop = (void **)symval.value;
1172
1173	/* and the number of entries */
1174	count = stop - start;
1175
1176	/* and copy out */
1177	if (startp)
1178		*startp = start;
1179	if (stopp)
1180		*stopp = stop;
1181	if (countp)
1182		*countp = count;
1183
1184out:
1185	free(setsym, M_LINKER);
1186	return error;
1187}
1188
1189static int
1190link_elf_each_function_name(linker_file_t file,
1191  int (*callback)(const char *, void *), void *opaque) {
1192    elf_file_t ef = (elf_file_t)file;
1193    const Elf_Sym* symp;
1194    int i, error;
1195
1196    /* Exhaustive search */
1197    for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
1198	if (symp->st_value != 0 &&
1199	    ELF_ST_TYPE(symp->st_info) == STT_FUNC) {
1200		error = callback(ef->ddbstrtab + symp->st_name, opaque);
1201		if (error)
1202		    return (error);
1203	}
1204    }
1205    return (0);
1206}
1207
1208#ifdef __ia64__
1209/*
1210 * Each KLD has its own GP. The GP value for each load module is given by
1211 * DT_PLTGOT on ia64. We need GP to construct function descriptors, but
1212 * don't have direct access to the ELF file structure. The link_elf_get_gp()
1213 * function returns the GP given a pointer to a generic linker file struct.
1214 */
1215Elf_Addr
1216link_elf_get_gp(linker_file_t lf)
1217{
1218	elf_file_t ef = (elf_file_t)lf;
1219	return (Elf_Addr)ef->got;
1220}
1221#endif
1222
1223const Elf_Sym *
1224elf_get_sym(linker_file_t lf, Elf_Word symidx)
1225{
1226	elf_file_t ef = (elf_file_t)lf;
1227
1228	if (symidx >= ef->nchains)
1229		return (NULL);
1230	return (ef->symtab + symidx);
1231}
1232
1233const char *
1234elf_get_symname(linker_file_t lf, Elf_Word symidx)
1235{
1236	elf_file_t ef = (elf_file_t)lf;
1237	const Elf_Sym *sym;
1238
1239	if (symidx >= ef->nchains)
1240		return (NULL);
1241	sym = ef->symtab + symidx;
1242	return (ef->strtab + sym->st_name);
1243}
1244
1245/*
1246 * Symbol lookup function that can be used when the symbol index is known (ie
1247 * in relocations). It uses the symbol index instead of doing a fully fledged
1248 * hash table based lookup when such is valid. For example for local symbols.
1249 * This is not only more efficient, it's also more correct. It's not always
1250 * the case that the symbol can be found through the hash table.
1251 */
1252static Elf_Addr
1253elf_lookup(linker_file_t lf, Elf_Word symidx, int deps)
1254{
1255	elf_file_t ef = (elf_file_t)lf;
1256	const Elf_Sym *sym;
1257	const char *symbol;
1258
1259	/* Don't even try to lookup the symbol if the index is bogus. */
1260	if (symidx >= ef->nchains)
1261		return (0);
1262
1263	sym = ef->symtab + symidx;
1264
1265	/*
1266	 * Don't do a full lookup when the symbol is local. It may even
1267	 * fail because it may not be found through the hash table.
1268	 */
1269	if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) {
1270		/* Force lookup failure when we have an insanity. */
1271		if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0)
1272			return (0);
1273		return ((Elf_Addr)ef->address + sym->st_value);
1274	}
1275
1276	/*
1277	 * XXX we can avoid doing a hash table based lookup for global
1278	 * symbols as well. This however is not always valid, so we'll
1279	 * just do it the hard way for now. Performance tweaks can
1280	 * always be added.
1281	 */
1282
1283	symbol = ef->strtab + sym->st_name;
1284
1285	/* Force a lookup failure if the symbol name is bogus. */
1286	if (*symbol == 0)
1287		return (0);
1288
1289	return ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps));
1290}
1291
1292static void
1293link_elf_reloc_local(linker_file_t lf)
1294{
1295    const Elf_Rel *rellim;
1296    const Elf_Rel *rel;
1297    const Elf_Rela *relalim;
1298    const Elf_Rela *rela;
1299    elf_file_t ef = (elf_file_t)lf;
1300
1301    /* Perform relocations without addend if there are any: */
1302    if ((rel = ef->rel) != NULL) {
1303	rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize);
1304	while (rel < rellim) {
1305	    elf_reloc_local(lf, (Elf_Addr)ef->address, rel, ELF_RELOC_REL,
1306			    elf_lookup);
1307	    rel++;
1308	}
1309    }
1310
1311    /* Perform relocations with addend if there are any: */
1312    if ((rela = ef->rela) != NULL) {
1313	relalim = (const Elf_Rela *)((const char *)ef->rela + ef->relasize);
1314	while (rela < relalim) {
1315	    elf_reloc_local(lf, (Elf_Addr)ef->address, rela, ELF_RELOC_RELA,
1316			    elf_lookup);
1317	    rela++;
1318	}
1319    }
1320}
1321