196
197static int
198link_elf_link_preload(linker_class_t cls, const char *filename,
199 linker_file_t *result)
200{
201 Elf_Ehdr *hdr;
202 Elf_Shdr *shdr;
203 Elf_Sym *es;
204 void *modptr, *baseptr, *sizeptr;
205 char *type;
206 elf_file_t ef;
207 linker_file_t lf;
208 Elf_Addr off;
209 int error, i, j, pb, ra, rl, shstrindex, symstrindex, symtabindex;
210
211 /* Look to see if we have the file preloaded */
212 modptr = preload_search_by_name(filename);
213 if (modptr == NULL)
214 return ENOENT;
215
216 type = (char *)preload_search_info(modptr, MODINFO_TYPE);
217 baseptr = preload_search_info(modptr, MODINFO_ADDR);
218 sizeptr = preload_search_info(modptr, MODINFO_SIZE);
219 hdr = (Elf_Ehdr *)preload_search_info(modptr, MODINFO_METADATA |
220 MODINFOMD_ELFHDR);
221 shdr = (Elf_Shdr *)preload_search_info(modptr, MODINFO_METADATA |
222 MODINFOMD_SHDR);
223 if (type == NULL || (strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE)
224 " obj module") != 0 &&
225 strcmp(type, "elf obj module") != 0)) {
226 return (EFTYPE);
227 }
228 if (baseptr == NULL || sizeptr == NULL || hdr == NULL ||
229 shdr == NULL)
230 return (EINVAL);
231
232 lf = linker_make_file(filename, &link_elf_class);
233 if (lf == NULL)
234 return (ENOMEM);
235
236 ef = (elf_file_t)lf;
237 ef->preloaded = 1;
238 ef->address = *(caddr_t *)baseptr;
239 lf->address = *(caddr_t *)baseptr;
240 lf->size = *(size_t *)sizeptr;
241
242 if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS ||
243 hdr->e_ident[EI_DATA] != ELF_TARG_DATA ||
244 hdr->e_ident[EI_VERSION] != EV_CURRENT ||
245 hdr->e_version != EV_CURRENT ||
246 hdr->e_type != ET_REL ||
247 hdr->e_machine != ELF_TARG_MACH) {
248 error = EFTYPE;
249 goto out;
250 }
251 ef->e_shdr = shdr;
252
253 /* Scan the section header for information and table sizing. */
254 symtabindex = -1;
255 symstrindex = -1;
256 for (i = 0; i < hdr->e_shnum; i++) {
257 switch (shdr[i].sh_type) {
258 case SHT_PROGBITS:
259 case SHT_NOBITS:
260#ifdef __amd64__
261 case SHT_X86_64_UNWIND:
262#endif
263 /* Ignore sections not loaded by the loader. */
264 if (shdr[i].sh_addr == 0)
265 break;
266 ef->nprogtab++;
267 break;
268 case SHT_SYMTAB:
269 symtabindex = i;
270 symstrindex = shdr[i].sh_link;
271 break;
272 case SHT_REL:
273 /*
274 * Ignore relocation tables for sections not
275 * loaded by the loader.
276 */
277 if (shdr[shdr[i].sh_info].sh_addr == 0)
278 break;
279 ef->nreltab++;
280 break;
281 case SHT_RELA:
282 if (shdr[shdr[i].sh_info].sh_addr == 0)
283 break;
284 ef->nrelatab++;
285 break;
286 }
287 }
288
289 shstrindex = hdr->e_shstrndx;
290 if (ef->nprogtab == 0 || symstrindex < 0 ||
291 symstrindex >= hdr->e_shnum ||
292 shdr[symstrindex].sh_type != SHT_STRTAB || shstrindex == 0 ||
293 shstrindex >= hdr->e_shnum ||
294 shdr[shstrindex].sh_type != SHT_STRTAB) {
295 printf("%s: bad/missing section headers\n", filename);
296 error = ENOEXEC;
297 goto out;
298 }
299
300 /* Allocate space for tracking the load chunks */
301 if (ef->nprogtab != 0)
302 ef->progtab = malloc(ef->nprogtab * sizeof(*ef->progtab),
303 M_LINKER, M_WAITOK | M_ZERO);
304 if (ef->nreltab != 0)
305 ef->reltab = malloc(ef->nreltab * sizeof(*ef->reltab),
306 M_LINKER, M_WAITOK | M_ZERO);
307 if (ef->nrelatab != 0)
308 ef->relatab = malloc(ef->nrelatab * sizeof(*ef->relatab),
309 M_LINKER, M_WAITOK | M_ZERO);
310 if ((ef->nprogtab != 0 && ef->progtab == NULL) ||
311 (ef->nreltab != 0 && ef->reltab == NULL) ||
312 (ef->nrelatab != 0 && ef->relatab == NULL)) {
313 error = ENOMEM;
314 goto out;
315 }
316
317 /* XXX, relocate the sh_addr fields saved by the loader. */
318 off = 0;
319 for (i = 0; i < hdr->e_shnum; i++) {
320 if (shdr[i].sh_addr != 0 && (off == 0 || shdr[i].sh_addr < off))
321 off = shdr[i].sh_addr;
322 }
323 for (i = 0; i < hdr->e_shnum; i++) {
324 if (shdr[i].sh_addr != 0)
325 shdr[i].sh_addr = shdr[i].sh_addr - off +
326 (Elf_Addr)ef->address;
327 }
328
329 ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym);
330 ef->ddbsymtab = (Elf_Sym *)shdr[symtabindex].sh_addr;
331 ef->ddbstrcnt = shdr[symstrindex].sh_size;
332 ef->ddbstrtab = (char *)shdr[symstrindex].sh_addr;
333 ef->shstrcnt = shdr[shstrindex].sh_size;
334 ef->shstrtab = (char *)shdr[shstrindex].sh_addr;
335
336 /* Now fill out progtab and the relocation tables. */
337 pb = 0;
338 rl = 0;
339 ra = 0;
340 for (i = 0; i < hdr->e_shnum; i++) {
341 switch (shdr[i].sh_type) {
342 case SHT_PROGBITS:
343 case SHT_NOBITS:
344#ifdef __amd64__
345 case SHT_X86_64_UNWIND:
346#endif
347 if (shdr[i].sh_addr == 0)
348 break;
349 ef->progtab[pb].addr = (void *)shdr[i].sh_addr;
350 if (shdr[i].sh_type == SHT_PROGBITS)
351 ef->progtab[pb].name = "<<PROGBITS>>";
352#ifdef __amd64__
353 else if (shdr[i].sh_type == SHT_X86_64_UNWIND)
354 ef->progtab[pb].name = "<<UNWIND>>";
355#endif
356 else
357 ef->progtab[pb].name = "<<NOBITS>>";
358 ef->progtab[pb].size = shdr[i].sh_size;
359 ef->progtab[pb].sec = i;
360 if (ef->shstrtab && shdr[i].sh_name != 0)
361 ef->progtab[pb].name =
362 ef->shstrtab + shdr[i].sh_name;
363 if (ef->progtab[pb].name != NULL &&
364 !strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) {
365 void *dpcpu;
366
367 dpcpu = dpcpu_alloc(shdr[i].sh_size);
368 if (dpcpu == NULL) {
369 printf("%s: pcpu module space is out "
370 "of space; cannot allocate %#jx "
371 "for %s\n", __func__,
372 (uintmax_t)shdr[i].sh_size,
373 filename);
374 error = ENOSPC;
375 goto out;
376 }
377 memcpy(dpcpu, ef->progtab[pb].addr,
378 ef->progtab[pb].size);
379 dpcpu_copy(dpcpu, shdr[i].sh_size);
380 ef->progtab[pb].addr = dpcpu;
381#ifdef VIMAGE
382 } else if (ef->progtab[pb].name != NULL &&
383 !strcmp(ef->progtab[pb].name, VNET_SETNAME)) {
384 void *vnet_data;
385
386 vnet_data = vnet_data_alloc(shdr[i].sh_size);
387 if (vnet_data == NULL) {
388 printf("%s: vnet module space is out "
389 "of space; cannot allocate %#jx "
390 "for %s\n", __func__,
391 (uintmax_t)shdr[i].sh_size,
392 filename);
393 error = ENOSPC;
394 goto out;
395 }
396 memcpy(vnet_data, ef->progtab[pb].addr,
397 ef->progtab[pb].size);
398 vnet_data_copy(vnet_data, shdr[i].sh_size);
399 ef->progtab[pb].addr = vnet_data;
400#endif
401 } else if (ef->progtab[pb].name != NULL &&
402 !strcmp(ef->progtab[pb].name, ".ctors")) {
403 lf->ctors_addr = ef->progtab[pb].addr;
404 lf->ctors_size = shdr[i].sh_size;
405 }
406
407 /* Update all symbol values with the offset. */
408 for (j = 0; j < ef->ddbsymcnt; j++) {
409 es = &ef->ddbsymtab[j];
410 if (es->st_shndx != i)
411 continue;
412 es->st_value += (Elf_Addr)ef->progtab[pb].addr;
413 }
414 pb++;
415 break;
416 case SHT_REL:
417 if (shdr[shdr[i].sh_info].sh_addr == 0)
418 break;
419 ef->reltab[rl].rel = (Elf_Rel *)shdr[i].sh_addr;
420 ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel);
421 ef->reltab[rl].sec = shdr[i].sh_info;
422 rl++;
423 break;
424 case SHT_RELA:
425 if (shdr[shdr[i].sh_info].sh_addr == 0)
426 break;
427 ef->relatab[ra].rela = (Elf_Rela *)shdr[i].sh_addr;
428 ef->relatab[ra].nrela =
429 shdr[i].sh_size / sizeof(Elf_Rela);
430 ef->relatab[ra].sec = shdr[i].sh_info;
431 ra++;
432 break;
433 }
434 }
435 if (pb != ef->nprogtab) {
436 printf("%s: lost progbits\n", filename);
437 error = ENOEXEC;
438 goto out;
439 }
440 if (rl != ef->nreltab) {
441 printf("%s: lost reltab\n", filename);
442 error = ENOEXEC;
443 goto out;
444 }
445 if (ra != ef->nrelatab) {
446 printf("%s: lost relatab\n", filename);
447 error = ENOEXEC;
448 goto out;
449 }
450
451 /* Local intra-module relocations */
452 error = link_elf_reloc_local(lf, false);
453 if (error != 0)
454 goto out;
455 *result = lf;
456 return (0);
457
458out:
459 /* preload not done this way */
460 linker_file_unload(lf, LINKER_UNLOAD_FORCE);
461 return (error);
462}
463
464static void
465link_elf_invoke_ctors(caddr_t addr, size_t size)
466{
467 void (**ctor)(void);
468 size_t i, cnt;
469
470 if (addr == NULL || size == 0)
471 return;
472 cnt = size / sizeof(*ctor);
473 ctor = (void *)addr;
474 for (i = 0; i < cnt; i++) {
475 if (ctor[i] != NULL)
476 (*ctor[i])();
477 }
478}
479
480static int
481link_elf_link_preload_finish(linker_file_t lf)
482{
483 elf_file_t ef;
484 int error;
485
486 ef = (elf_file_t)lf;
487 error = relocate_file(ef);
488 if (error)
489 return (error);
490
491 /* Notify MD code that a module is being loaded. */
492 error = elf_cpu_load_file(lf);
493 if (error)
494 return (error);
495
496#if defined(__i386__) || defined(__amd64__)
497 /* Now ifuncs. */
498 error = link_elf_reloc_local(lf, true);
499 if (error != 0)
500 return (error);
501#endif
502
503 /* Invoke .ctors */
504 link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size);
505 return (0);
506}
507
508static int
509link_elf_load_file(linker_class_t cls, const char *filename,
510 linker_file_t *result)
511{
512 struct nameidata nd;
513 struct thread *td = curthread; /* XXX */
514 Elf_Ehdr *hdr;
515 Elf_Shdr *shdr;
516 Elf_Sym *es;
517 int nbytes, i, j;
518 vm_offset_t mapbase;
519 size_t mapsize;
520 int error = 0;
521 ssize_t resid;
522 int flags;
523 elf_file_t ef;
524 linker_file_t lf;
525 int symtabindex;
526 int symstrindex;
527 int shstrindex;
528 int nsym;
529 int pb, rl, ra;
530 int alignmask;
531
532 shdr = NULL;
533 lf = NULL;
534 mapsize = 0;
535 hdr = NULL;
536
537 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td);
538 flags = FREAD;
539 error = vn_open(&nd, &flags, 0, NULL);
540 if (error)
541 return error;
542 NDFREE(&nd, NDF_ONLY_PNBUF);
543 if (nd.ni_vp->v_type != VREG) {
544 error = ENOEXEC;
545 goto out;
546 }
547#ifdef MAC
548 error = mac_kld_check_load(td->td_ucred, nd.ni_vp);
549 if (error) {
550 goto out;
551 }
552#endif
553
554 /* Read the elf header from the file. */
555 hdr = malloc(sizeof(*hdr), M_LINKER, M_WAITOK);
556 error = vn_rdwr(UIO_READ, nd.ni_vp, (void *)hdr, sizeof(*hdr), 0,
557 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
558 &resid, td);
559 if (error)
560 goto out;
561 if (resid != 0){
562 error = ENOEXEC;
563 goto out;
564 }
565
566 if (!IS_ELF(*hdr)) {
567 error = ENOEXEC;
568 goto out;
569 }
570
571 if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS
572 || hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
573 link_elf_error(filename, "Unsupported file layout");
574 error = ENOEXEC;
575 goto out;
576 }
577 if (hdr->e_ident[EI_VERSION] != EV_CURRENT
578 || hdr->e_version != EV_CURRENT) {
579 link_elf_error(filename, "Unsupported file version");
580 error = ENOEXEC;
581 goto out;
582 }
583 if (hdr->e_type != ET_REL) {
584 error = ENOSYS;
585 goto out;
586 }
587 if (hdr->e_machine != ELF_TARG_MACH) {
588 link_elf_error(filename, "Unsupported machine");
589 error = ENOEXEC;
590 goto out;
591 }
592
593 lf = linker_make_file(filename, &link_elf_class);
594 if (!lf) {
595 error = ENOMEM;
596 goto out;
597 }
598 ef = (elf_file_t) lf;
599 ef->nprogtab = 0;
600 ef->e_shdr = 0;
601 ef->nreltab = 0;
602 ef->nrelatab = 0;
603
604 /* Allocate and read in the section header */
605 nbytes = hdr->e_shnum * hdr->e_shentsize;
606 if (nbytes == 0 || hdr->e_shoff == 0 ||
607 hdr->e_shentsize != sizeof(Elf_Shdr)) {
608 error = ENOEXEC;
609 goto out;
610 }
611 shdr = malloc(nbytes, M_LINKER, M_WAITOK);
612 ef->e_shdr = shdr;
613 error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shdr, nbytes, hdr->e_shoff,
614 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td);
615 if (error)
616 goto out;
617 if (resid) {
618 error = ENOEXEC;
619 goto out;
620 }
621
622 /* Scan the section header for information and table sizing. */
623 nsym = 0;
624 symtabindex = -1;
625 symstrindex = -1;
626 for (i = 0; i < hdr->e_shnum; i++) {
627 if (shdr[i].sh_size == 0)
628 continue;
629 switch (shdr[i].sh_type) {
630 case SHT_PROGBITS:
631 case SHT_NOBITS:
632#ifdef __amd64__
633 case SHT_X86_64_UNWIND:
634#endif
635 if ((shdr[i].sh_flags & SHF_ALLOC) == 0)
636 break;
637 ef->nprogtab++;
638 break;
639 case SHT_SYMTAB:
640 nsym++;
641 symtabindex = i;
642 symstrindex = shdr[i].sh_link;
643 break;
644 case SHT_REL:
645 /*
646 * Ignore relocation tables for unallocated
647 * sections.
648 */
649 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0)
650 break;
651 ef->nreltab++;
652 break;
653 case SHT_RELA:
654 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0)
655 break;
656 ef->nrelatab++;
657 break;
658 case SHT_STRTAB:
659 break;
660 }
661 }
662 if (ef->nprogtab == 0) {
663 link_elf_error(filename, "file has no contents");
664 error = ENOEXEC;
665 goto out;
666 }
667 if (nsym != 1) {
668 /* Only allow one symbol table for now */
669 link_elf_error(filename, "file has no valid symbol table");
670 error = ENOEXEC;
671 goto out;
672 }
673 if (symstrindex < 0 || symstrindex > hdr->e_shnum ||
674 shdr[symstrindex].sh_type != SHT_STRTAB) {
675 link_elf_error(filename, "file has invalid symbol strings");
676 error = ENOEXEC;
677 goto out;
678 }
679
680 /* Allocate space for tracking the load chunks */
681 if (ef->nprogtab != 0)
682 ef->progtab = malloc(ef->nprogtab * sizeof(*ef->progtab),
683 M_LINKER, M_WAITOK | M_ZERO);
684 if (ef->nreltab != 0)
685 ef->reltab = malloc(ef->nreltab * sizeof(*ef->reltab),
686 M_LINKER, M_WAITOK | M_ZERO);
687 if (ef->nrelatab != 0)
688 ef->relatab = malloc(ef->nrelatab * sizeof(*ef->relatab),
689 M_LINKER, M_WAITOK | M_ZERO);
690
691 if (symtabindex == -1) {
692 link_elf_error(filename, "lost symbol table index");
693 error = ENOEXEC;
694 goto out;
695 }
696 /* Allocate space for and load the symbol table */
697 ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym);
698 ef->ddbsymtab = malloc(shdr[symtabindex].sh_size, M_LINKER, M_WAITOK);
699 error = vn_rdwr(UIO_READ, nd.ni_vp, (void *)ef->ddbsymtab,
700 shdr[symtabindex].sh_size, shdr[symtabindex].sh_offset,
701 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
702 &resid, td);
703 if (error)
704 goto out;
705 if (resid != 0){
706 error = EINVAL;
707 goto out;
708 }
709
710 /* Allocate space for and load the symbol strings */
711 ef->ddbstrcnt = shdr[symstrindex].sh_size;
712 ef->ddbstrtab = malloc(shdr[symstrindex].sh_size, M_LINKER, M_WAITOK);
713 error = vn_rdwr(UIO_READ, nd.ni_vp, ef->ddbstrtab,
714 shdr[symstrindex].sh_size, shdr[symstrindex].sh_offset,
715 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
716 &resid, td);
717 if (error)
718 goto out;
719 if (resid != 0){
720 error = EINVAL;
721 goto out;
722 }
723
724 /* Do we have a string table for the section names? */
725 shstrindex = -1;
726 if (hdr->e_shstrndx != 0 &&
727 shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) {
728 shstrindex = hdr->e_shstrndx;
729 ef->shstrcnt = shdr[shstrindex].sh_size;
730 ef->shstrtab = malloc(shdr[shstrindex].sh_size, M_LINKER,
731 M_WAITOK);
732 error = vn_rdwr(UIO_READ, nd.ni_vp, ef->shstrtab,
733 shdr[shstrindex].sh_size, shdr[shstrindex].sh_offset,
734 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
735 &resid, td);
736 if (error)
737 goto out;
738 if (resid != 0){
739 error = EINVAL;
740 goto out;
741 }
742 }
743
744 /* Size up code/data(progbits) and bss(nobits). */
745 alignmask = 0;
746 for (i = 0; i < hdr->e_shnum; i++) {
747 if (shdr[i].sh_size == 0)
748 continue;
749 switch (shdr[i].sh_type) {
750 case SHT_PROGBITS:
751 case SHT_NOBITS:
752#ifdef __amd64__
753 case SHT_X86_64_UNWIND:
754#endif
755 if ((shdr[i].sh_flags & SHF_ALLOC) == 0)
756 break;
757 alignmask = shdr[i].sh_addralign - 1;
758 mapsize += alignmask;
759 mapsize &= ~alignmask;
760 mapsize += shdr[i].sh_size;
761 break;
762 }
763 }
764
765 /*
766 * We know how much space we need for the text/data/bss/etc.
767 * This stuff needs to be in a single chunk so that profiling etc
768 * can get the bounds and gdb can associate offsets with modules
769 */
770 ef->object = vm_object_allocate(OBJT_DEFAULT,
771 round_page(mapsize) >> PAGE_SHIFT);
772 if (ef->object == NULL) {
773 error = ENOMEM;
774 goto out;
775 }
776 ef->address = (caddr_t) vm_map_min(kernel_map);
777
778 /*
779 * In order to satisfy amd64's architectural requirements on the
780 * location of code and data in the kernel's address space, request a
781 * mapping that is above the kernel.
782 */
783#ifdef __amd64__
784 mapbase = KERNBASE;
785#else
786 mapbase = VM_MIN_KERNEL_ADDRESS;
787#endif
788 error = vm_map_find(kernel_map, ef->object, 0, &mapbase,
789 round_page(mapsize), 0, VMFS_OPTIMAL_SPACE, VM_PROT_ALL,
790 VM_PROT_ALL, 0);
791 if (error) {
792 vm_object_deallocate(ef->object);
793 ef->object = 0;
794 goto out;
795 }
796
797 /* Wire the pages */
798 error = vm_map_wire(kernel_map, mapbase,
799 mapbase + round_page(mapsize),
800 VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES);
801 if (error != KERN_SUCCESS) {
802 error = ENOMEM;
803 goto out;
804 }
805
806 /* Inform the kld system about the situation */
807 lf->address = ef->address = (caddr_t)mapbase;
808 lf->size = mapsize;
809
810 /*
811 * Now load code/data(progbits), zero bss(nobits), allocate space for
812 * and load relocs
813 */
814 pb = 0;
815 rl = 0;
816 ra = 0;
817 alignmask = 0;
818 for (i = 0; i < hdr->e_shnum; i++) {
819 if (shdr[i].sh_size == 0)
820 continue;
821 switch (shdr[i].sh_type) {
822 case SHT_PROGBITS:
823 case SHT_NOBITS:
824#ifdef __amd64__
825 case SHT_X86_64_UNWIND:
826#endif
827 if ((shdr[i].sh_flags & SHF_ALLOC) == 0)
828 break;
829 alignmask = shdr[i].sh_addralign - 1;
830 mapbase += alignmask;
831 mapbase &= ~alignmask;
832 if (ef->shstrtab != NULL && shdr[i].sh_name != 0) {
833 ef->progtab[pb].name =
834 ef->shstrtab + shdr[i].sh_name;
835 if (!strcmp(ef->progtab[pb].name, ".ctors")) {
836 lf->ctors_addr = (caddr_t)mapbase;
837 lf->ctors_size = shdr[i].sh_size;
838 }
839 } else if (shdr[i].sh_type == SHT_PROGBITS)
840 ef->progtab[pb].name = "<<PROGBITS>>";
841#ifdef __amd64__
842 else if (shdr[i].sh_type == SHT_X86_64_UNWIND)
843 ef->progtab[pb].name = "<<UNWIND>>";
844#endif
845 else
846 ef->progtab[pb].name = "<<NOBITS>>";
847 if (ef->progtab[pb].name != NULL &&
848 !strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) {
849 ef->progtab[pb].addr =
850 dpcpu_alloc(shdr[i].sh_size);
851 if (ef->progtab[pb].addr == NULL) {
852 printf("%s: pcpu module space is out "
853 "of space; cannot allocate %#jx "
854 "for %s\n", __func__,
855 (uintmax_t)shdr[i].sh_size,
856 filename);
857 }
858 }
859#ifdef VIMAGE
860 else if (ef->progtab[pb].name != NULL &&
861 !strcmp(ef->progtab[pb].name, VNET_SETNAME)) {
862 ef->progtab[pb].addr =
863 vnet_data_alloc(shdr[i].sh_size);
864 if (ef->progtab[pb].addr == NULL) {
865 printf("%s: vnet module space is out "
866 "of space; cannot allocate %#jx "
867 "for %s\n", __func__,
868 (uintmax_t)shdr[i].sh_size,
869 filename);
870 }
871 }
872#endif
873 else
874 ef->progtab[pb].addr =
875 (void *)(uintptr_t)mapbase;
876 if (ef->progtab[pb].addr == NULL) {
877 error = ENOSPC;
878 goto out;
879 }
880 ef->progtab[pb].size = shdr[i].sh_size;
881 ef->progtab[pb].sec = i;
882 if (shdr[i].sh_type == SHT_PROGBITS
883#ifdef __amd64__
884 || shdr[i].sh_type == SHT_X86_64_UNWIND
885#endif
886 ) {
887 error = vn_rdwr(UIO_READ, nd.ni_vp,
888 ef->progtab[pb].addr,
889 shdr[i].sh_size, shdr[i].sh_offset,
890 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred,
891 NOCRED, &resid, td);
892 if (error)
893 goto out;
894 if (resid != 0){
895 error = EINVAL;
896 goto out;
897 }
898 /* Initialize the per-cpu or vnet area. */
899 if (ef->progtab[pb].addr != (void *)mapbase &&
900 !strcmp(ef->progtab[pb].name, DPCPU_SETNAME))
901 dpcpu_copy(ef->progtab[pb].addr,
902 shdr[i].sh_size);
903#ifdef VIMAGE
904 else if (ef->progtab[pb].addr !=
905 (void *)mapbase &&
906 !strcmp(ef->progtab[pb].name, VNET_SETNAME))
907 vnet_data_copy(ef->progtab[pb].addr,
908 shdr[i].sh_size);
909#endif
910 } else
911 bzero(ef->progtab[pb].addr, shdr[i].sh_size);
912
913 /* Update all symbol values with the offset. */
914 for (j = 0; j < ef->ddbsymcnt; j++) {
915 es = &ef->ddbsymtab[j];
916 if (es->st_shndx != i)
917 continue;
918 es->st_value += (Elf_Addr)ef->progtab[pb].addr;
919 }
920 mapbase += shdr[i].sh_size;
921 pb++;
922 break;
923 case SHT_REL:
924 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0)
925 break;
926 ef->reltab[rl].rel = malloc(shdr[i].sh_size, M_LINKER,
927 M_WAITOK);
928 ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel);
929 ef->reltab[rl].sec = shdr[i].sh_info;
930 error = vn_rdwr(UIO_READ, nd.ni_vp,
931 (void *)ef->reltab[rl].rel,
932 shdr[i].sh_size, shdr[i].sh_offset,
933 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
934 &resid, td);
935 if (error)
936 goto out;
937 if (resid != 0){
938 error = EINVAL;
939 goto out;
940 }
941 rl++;
942 break;
943 case SHT_RELA:
944 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0)
945 break;
946 ef->relatab[ra].rela = malloc(shdr[i].sh_size, M_LINKER,
947 M_WAITOK);
948 ef->relatab[ra].nrela =
949 shdr[i].sh_size / sizeof(Elf_Rela);
950 ef->relatab[ra].sec = shdr[i].sh_info;
951 error = vn_rdwr(UIO_READ, nd.ni_vp,
952 (void *)ef->relatab[ra].rela,
953 shdr[i].sh_size, shdr[i].sh_offset,
954 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
955 &resid, td);
956 if (error)
957 goto out;
958 if (resid != 0){
959 error = EINVAL;
960 goto out;
961 }
962 ra++;
963 break;
964 }
965 }
966 if (pb != ef->nprogtab) {
967 link_elf_error(filename, "lost progbits");
968 error = ENOEXEC;
969 goto out;
970 }
971 if (rl != ef->nreltab) {
972 link_elf_error(filename, "lost reltab");
973 error = ENOEXEC;
974 goto out;
975 }
976 if (ra != ef->nrelatab) {
977 link_elf_error(filename, "lost relatab");
978 error = ENOEXEC;
979 goto out;
980 }
981 if (mapbase != (vm_offset_t)ef->address + mapsize) {
982 printf(
983 "%s: mapbase 0x%lx != address %p + mapsize 0x%lx (0x%lx)\n",
984 filename != NULL ? filename : "<none>",
985 (u_long)mapbase, ef->address, (u_long)mapsize,
986 (u_long)(vm_offset_t)ef->address + mapsize);
987 error = ENOMEM;
988 goto out;
989 }
990
991 /* Local intra-module relocations */
992 error = link_elf_reloc_local(lf, false);
993 if (error != 0)
994 goto out;
995
996 /* Pull in dependencies */
997 VOP_UNLOCK(nd.ni_vp, 0);
998 error = linker_load_dependencies(lf);
999 vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY);
1000 if (error)
1001 goto out;
1002
1003 /* External relocations */
1004 error = relocate_file(ef);
1005 if (error)
1006 goto out;
1007
1008 /* Notify MD code that a module is being loaded. */
1009 error = elf_cpu_load_file(lf);
1010 if (error)
1011 goto out;
1012
1013#if defined(__i386__) || defined(__amd64__)
1014 /* Now ifuncs. */
1015 error = link_elf_reloc_local(lf, true);
1016 if (error != 0)
1017 goto out;
1018#endif
1019
1020 /* Invoke .ctors */
1021 link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size);
1022
1023 *result = lf;
1024
1025out:
1026 VOP_UNLOCK(nd.ni_vp, 0);
1027 vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
1028 if (error && lf)
1029 linker_file_unload(lf, LINKER_UNLOAD_FORCE);
1030 free(hdr, M_LINKER);
1031
1032 return error;
1033}
1034
1035static void
1036link_elf_unload_file(linker_file_t file)
1037{
1038 elf_file_t ef = (elf_file_t) file;
1039 int i;
1040
1041 /* Notify MD code that a module is being unloaded. */
1042 elf_cpu_unload_file(file);
1043
1044 if (ef->progtab) {
1045 for (i = 0; i < ef->nprogtab; i++) {
1046 if (ef->progtab[i].size == 0)
1047 continue;
1048 if (ef->progtab[i].name == NULL)
1049 continue;
1050 if (!strcmp(ef->progtab[i].name, DPCPU_SETNAME))
1051 dpcpu_free(ef->progtab[i].addr,
1052 ef->progtab[i].size);
1053#ifdef VIMAGE
1054 else if (!strcmp(ef->progtab[i].name, VNET_SETNAME))
1055 vnet_data_free(ef->progtab[i].addr,
1056 ef->progtab[i].size);
1057#endif
1058 }
1059 }
1060 if (ef->preloaded) {
1061 free(ef->reltab, M_LINKER);
1062 free(ef->relatab, M_LINKER);
1063 free(ef->progtab, M_LINKER);
1064 free(ef->ctftab, M_LINKER);
1065 free(ef->ctfoff, M_LINKER);
1066 free(ef->typoff, M_LINKER);
1067 if (file->pathname != NULL)
1068 preload_delete_name(file->pathname);
1069 return;
1070 }
1071
1072 for (i = 0; i < ef->nreltab; i++)
1073 free(ef->reltab[i].rel, M_LINKER);
1074 for (i = 0; i < ef->nrelatab; i++)
1075 free(ef->relatab[i].rela, M_LINKER);
1076 free(ef->reltab, M_LINKER);
1077 free(ef->relatab, M_LINKER);
1078 free(ef->progtab, M_LINKER);
1079
1080 if (ef->object) {
1081 vm_map_remove(kernel_map, (vm_offset_t) ef->address,
1082 (vm_offset_t) ef->address +
1083 (ef->object->size << PAGE_SHIFT));
1084 }
1085 free(ef->e_shdr, M_LINKER);
1086 free(ef->ddbsymtab, M_LINKER);
1087 free(ef->ddbstrtab, M_LINKER);
1088 free(ef->shstrtab, M_LINKER);
1089 free(ef->ctftab, M_LINKER);
1090 free(ef->ctfoff, M_LINKER);
1091 free(ef->typoff, M_LINKER);
1092}
1093
1094static const char *
1095symbol_name(elf_file_t ef, Elf_Size r_info)
1096{
1097 const Elf_Sym *ref;
1098
1099 if (ELF_R_SYM(r_info)) {
1100 ref = ef->ddbsymtab + ELF_R_SYM(r_info);
1101 return ef->ddbstrtab + ref->st_name;
1102 } else
1103 return NULL;
1104}
1105
1106static Elf_Addr
1107findbase(elf_file_t ef, int sec)
1108{
1109 int i;
1110 Elf_Addr base = 0;
1111
1112 for (i = 0; i < ef->nprogtab; i++) {
1113 if (sec == ef->progtab[i].sec) {
1114 base = (Elf_Addr)ef->progtab[i].addr;
1115 break;
1116 }
1117 }
1118 return base;
1119}
1120
1121static int
1122relocate_file(elf_file_t ef)
1123{
1124 const Elf_Rel *rellim;
1125 const Elf_Rel *rel;
1126 const Elf_Rela *relalim;
1127 const Elf_Rela *rela;
1128 const char *symname;
1129 const Elf_Sym *sym;
1130 int i;
1131 Elf_Size symidx;
1132 Elf_Addr base;
1133
1134
1135 /* Perform relocations without addend if there are any: */
1136 for (i = 0; i < ef->nreltab; i++) {
1137 rel = ef->reltab[i].rel;
1138 if (rel == NULL) {
1139 link_elf_error(ef->lf.filename, "lost a reltab!");
1140 return (ENOEXEC);
1141 }
1142 rellim = rel + ef->reltab[i].nrel;
1143 base = findbase(ef, ef->reltab[i].sec);
1144 if (base == 0) {
1145 link_elf_error(ef->lf.filename, "lost base for reltab");
1146 return (ENOEXEC);
1147 }
1148 for ( ; rel < rellim; rel++) {
1149 symidx = ELF_R_SYM(rel->r_info);
1150 if (symidx >= ef->ddbsymcnt)
1151 continue;
1152 sym = ef->ddbsymtab + symidx;
1153 /* Local relocs are already done */
1154 if (ELF_ST_BIND(sym->st_info) == STB_LOCAL)
1155 continue;
1156 if (elf_reloc(&ef->lf, base, rel, ELF_RELOC_REL,
1157 elf_obj_lookup)) {
1158 symname = symbol_name(ef, rel->r_info);
1159 printf("link_elf_obj: symbol %s undefined\n",
1160 symname);
1161 return (ENOENT);
1162 }
1163 }
1164 }
1165
1166 /* Perform relocations with addend if there are any: */
1167 for (i = 0; i < ef->nrelatab; i++) {
1168 rela = ef->relatab[i].rela;
1169 if (rela == NULL) {
1170 link_elf_error(ef->lf.filename, "lost a relatab!");
1171 return (ENOEXEC);
1172 }
1173 relalim = rela + ef->relatab[i].nrela;
1174 base = findbase(ef, ef->relatab[i].sec);
1175 if (base == 0) {
1176 link_elf_error(ef->lf.filename,
1177 "lost base for relatab");
1178 return (ENOEXEC);
1179 }
1180 for ( ; rela < relalim; rela++) {
1181 symidx = ELF_R_SYM(rela->r_info);
1182 if (symidx >= ef->ddbsymcnt)
1183 continue;
1184 sym = ef->ddbsymtab + symidx;
1185 /* Local relocs are already done */
1186 if (ELF_ST_BIND(sym->st_info) == STB_LOCAL)
1187 continue;
1188 if (elf_reloc(&ef->lf, base, rela, ELF_RELOC_RELA,
1189 elf_obj_lookup)) {
1190 symname = symbol_name(ef, rela->r_info);
1191 printf("link_elf_obj: symbol %s undefined\n",
1192 symname);
1193 return (ENOENT);
1194 }
1195 }
1196 }
1197
1198 /*
1199 * Only clean SHN_FBSD_CACHED for successful return. If we
1200 * modified symbol table for the object but found an
1201 * unresolved symbol, there is no reason to roll back.
1202 */
1203 elf_obj_cleanup_globals_cache(ef);
1204
1205 return (0);
1206}
1207
1208static int
1209link_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t *sym)
1210{
1211 elf_file_t ef = (elf_file_t) lf;
1212 const Elf_Sym *symp;
1213 const char *strp;
1214 int i;
1215
1216 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
1217 strp = ef->ddbstrtab + symp->st_name;
1218 if (symp->st_shndx != SHN_UNDEF && strcmp(name, strp) == 0) {
1219 *sym = (c_linker_sym_t) symp;
1220 return 0;
1221 }
1222 }
1223 return ENOENT;
1224}
1225
1226static int
1227link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym,
1228 linker_symval_t *symval)
1229{
1230 elf_file_t ef;
1231 const Elf_Sym *es;
1232 caddr_t val;
1233
1234 ef = (elf_file_t) lf;
1235 es = (const Elf_Sym*) sym;
1236 val = (caddr_t)es->st_value;
1237 if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {
1238 symval->name = ef->ddbstrtab + es->st_name;
1239 val = (caddr_t)es->st_value;
1240 if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC)
1241 val = ((caddr_t (*)(void))val)();
1242 symval->value = val;
1243 symval->size = es->st_size;
1244 return 0;
1245 }
1246 return ENOENT;
1247}
1248
1249static int
1250link_elf_search_symbol(linker_file_t lf, caddr_t value,
1251 c_linker_sym_t *sym, long *diffp)
1252{
1253 elf_file_t ef = (elf_file_t) lf;
1254 u_long off = (uintptr_t) (void *) value;
1255 u_long diff = off;
1256 u_long st_value;
1257 const Elf_Sym *es;
1258 const Elf_Sym *best = NULL;
1259 int i;
1260
1261 for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) {
1262 if (es->st_name == 0)
1263 continue;
1264 st_value = es->st_value;
1265 if (off >= st_value) {
1266 if (off - st_value < diff) {
1267 diff = off - st_value;
1268 best = es;
1269 if (diff == 0)
1270 break;
1271 } else if (off - st_value == diff) {
1272 best = es;
1273 }
1274 }
1275 }
1276 if (best == NULL)
1277 *diffp = off;
1278 else
1279 *diffp = diff;
1280 *sym = (c_linker_sym_t) best;
1281
1282 return 0;
1283}
1284
1285/*
1286 * Look up a linker set on an ELF system.
1287 */
1288static int
1289link_elf_lookup_set(linker_file_t lf, const char *name,
1290 void ***startp, void ***stopp, int *countp)
1291{
1292 elf_file_t ef = (elf_file_t)lf;
1293 void **start, **stop;
1294 int i, count;
1295
1296 /* Relative to section number */
1297 for (i = 0; i < ef->nprogtab; i++) {
1298 if ((strncmp(ef->progtab[i].name, "set_", 4) == 0) &&
1299 strcmp(ef->progtab[i].name + 4, name) == 0) {
1300 start = (void **)ef->progtab[i].addr;
1301 stop = (void **)((char *)ef->progtab[i].addr +
1302 ef->progtab[i].size);
1303 count = stop - start;
1304 if (startp)
1305 *startp = start;
1306 if (stopp)
1307 *stopp = stop;
1308 if (countp)
1309 *countp = count;
1310 return (0);
1311 }
1312 }
1313 return (ESRCH);
1314}
1315
1316static int
1317link_elf_each_function_name(linker_file_t file,
1318 int (*callback)(const char *, void *), void *opaque)
1319{
1320 elf_file_t ef = (elf_file_t)file;
1321 const Elf_Sym *symp;
1322 int i, error;
1323
1324 /* Exhaustive search */
1325 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
1326 if (symp->st_value != 0 &&
1327 (ELF_ST_TYPE(symp->st_info) == STT_FUNC ||
1328 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) {
1329 error = callback(ef->ddbstrtab + symp->st_name, opaque);
1330 if (error)
1331 return (error);
1332 }
1333 }
1334 return (0);
1335}
1336
1337static int
1338link_elf_each_function_nameval(linker_file_t file,
1339 linker_function_nameval_callback_t callback, void *opaque)
1340{
1341 linker_symval_t symval;
1342 elf_file_t ef = (elf_file_t)file;
1343 const Elf_Sym* symp;
1344 int i, error;
1345
1346 /* Exhaustive search */
1347 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
1348 if (symp->st_value != 0 &&
1349 (ELF_ST_TYPE(symp->st_info) == STT_FUNC ||
1350 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) {
1351 error = link_elf_symbol_values(file,
1352 (c_linker_sym_t)symp, &symval);
1353 if (error)
1354 return (error);
1355 error = callback(file, i, &symval, opaque);
1356 if (error)
1357 return (error);
1358 }
1359 }
1360 return (0);
1361}
1362
1363static void
1364elf_obj_cleanup_globals_cache(elf_file_t ef)
1365{
1366 Elf_Sym *sym;
1367 Elf_Size i;
1368
1369 for (i = 0; i < ef->ddbsymcnt; i++) {
1370 sym = ef->ddbsymtab + i;
1371 if (sym->st_shndx == SHN_FBSD_CACHED) {
1372 sym->st_shndx = SHN_UNDEF;
1373 sym->st_value = 0;
1374 }
1375 }
1376}
1377
1378/*
1379 * Symbol lookup function that can be used when the symbol index is known (ie
1380 * in relocations). It uses the symbol index instead of doing a fully fledged
1381 * hash table based lookup when such is valid. For example for local symbols.
1382 * This is not only more efficient, it's also more correct. It's not always
1383 * the case that the symbol can be found through the hash table.
1384 */
1385static int
1386elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res)
1387{
1388 elf_file_t ef = (elf_file_t)lf;
1389 Elf_Sym *sym;
1390 const char *symbol;
1391 Elf_Addr res1;
1392
1393 /* Don't even try to lookup the symbol if the index is bogus. */
1394 if (symidx >= ef->ddbsymcnt) {
1395 *res = 0;
1396 return (EINVAL);
1397 }
1398
1399 sym = ef->ddbsymtab + symidx;
1400
1401 /* Quick answer if there is a definition included. */
1402 if (sym->st_shndx != SHN_UNDEF) {
1403 res1 = (Elf_Addr)sym->st_value;
1404 if (ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC)
1405 res1 = ((Elf_Addr (*)(void))res1)();
1406 *res = res1;
1407 return (0);
1408 }
1409
1410 /* If we get here, then it is undefined and needs a lookup. */
1411 switch (ELF_ST_BIND(sym->st_info)) {
1412 case STB_LOCAL:
1413 /* Local, but undefined? huh? */
1414 *res = 0;
1415 return (EINVAL);
1416
1417 case STB_GLOBAL:
1418 case STB_WEAK:
1419 /* Relative to Data or Function name */
1420 symbol = ef->ddbstrtab + sym->st_name;
1421
1422 /* Force a lookup failure if the symbol name is bogus. */
1423 if (*symbol == 0) {
1424 *res = 0;
1425 return (EINVAL);
1426 }
1427 res1 = (Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps);
1428
1429 /*
1430 * Cache global lookups during module relocation. The failure
1431 * case is particularly expensive for callers, who must scan
1432 * through the entire globals table doing strcmp(). Cache to
1433 * avoid doing such work repeatedly.
1434 *
1435 * After relocation is complete, undefined globals will be
1436 * restored to SHN_UNDEF in elf_obj_cleanup_globals_cache(),
1437 * above.
1438 */
1439 if (res1 != 0) {
1440 sym->st_shndx = SHN_FBSD_CACHED;
1441 sym->st_value = res1;
1442 *res = res1;
1443 return (0);
1444 } else if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
1445 sym->st_value = 0;
1446 *res = 0;
1447 return (0);
1448 }
1449 return (EINVAL);
1450
1451 default:
1452 return (EINVAL);
1453 }
1454}
1455
1456static void
1457link_elf_fix_link_set(elf_file_t ef)
1458{
1459 static const char startn[] = "__start_";
1460 static const char stopn[] = "__stop_";
1461 Elf_Sym *sym;
1462 const char *sym_name, *linkset_name;
1463 Elf_Addr startp, stopp;
1464 Elf_Size symidx;
1465 int start, i;
1466
1467 startp = stopp = 0;
1468 for (symidx = 1 /* zero entry is special */;
1469 symidx < ef->ddbsymcnt; symidx++) {
1470 sym = ef->ddbsymtab + symidx;
1471 if (sym->st_shndx != SHN_UNDEF)
1472 continue;
1473
1474 sym_name = ef->ddbstrtab + sym->st_name;
1475 if (strncmp(sym_name, startn, sizeof(startn) - 1) == 0) {
1476 start = 1;
1477 linkset_name = sym_name + sizeof(startn) - 1;
1478 }
1479 else if (strncmp(sym_name, stopn, sizeof(stopn) - 1) == 0) {
1480 start = 0;
1481 linkset_name = sym_name + sizeof(stopn) - 1;
1482 }
1483 else
1484 continue;
1485
1486 for (i = 0; i < ef->nprogtab; i++) {
1487 if (strcmp(ef->progtab[i].name, linkset_name) == 0) {
1488 startp = (Elf_Addr)ef->progtab[i].addr;
1489 stopp = (Elf_Addr)(startp + ef->progtab[i].size);
1490 break;
1491 }
1492 }
1493 if (i == ef->nprogtab)
1494 continue;
1495
1496 sym->st_value = start ? startp : stopp;
1497 sym->st_shndx = i;
1498 }
1499}
1500
1501static int
1502link_elf_reloc_local(linker_file_t lf, bool ifuncs)
1503{
1504 elf_file_t ef = (elf_file_t)lf;
1505 const Elf_Rel *rellim;
1506 const Elf_Rel *rel;
1507 const Elf_Rela *relalim;
1508 const Elf_Rela *rela;
1509 const Elf_Sym *sym;
1510 Elf_Addr base;
1511 int i;
1512 Elf_Size symidx;
1513
1514 link_elf_fix_link_set(ef);
1515
1516 /* Perform relocations without addend if there are any: */
1517 for (i = 0; i < ef->nreltab; i++) {
1518 rel = ef->reltab[i].rel;
1519 if (rel == NULL) {
1520 link_elf_error(ef->lf.filename, "lost a reltab");
1521 return (ENOEXEC);
1522 }
1523 rellim = rel + ef->reltab[i].nrel;
1524 base = findbase(ef, ef->reltab[i].sec);
1525 if (base == 0) {
1526 link_elf_error(ef->lf.filename, "lost base for reltab");
1527 return (ENOEXEC);
1528 }
1529 for ( ; rel < rellim; rel++) {
1530 symidx = ELF_R_SYM(rel->r_info);
1531 if (symidx >= ef->ddbsymcnt)
1532 continue;
1533 sym = ef->ddbsymtab + symidx;
1534 /* Only do local relocs */
1535 if (ELF_ST_BIND(sym->st_info) != STB_LOCAL)
1536 continue;
1537 if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC ||
1538 elf_is_ifunc_reloc(rel->r_info)) == ifuncs)
1539 elf_reloc_local(lf, base, rel, ELF_RELOC_REL,
1540 elf_obj_lookup);
1541 }
1542 }
1543
1544 /* Perform relocations with addend if there are any: */
1545 for (i = 0; i < ef->nrelatab; i++) {
1546 rela = ef->relatab[i].rela;
1547 if (rela == NULL) {
1548 link_elf_error(ef->lf.filename, "lost a relatab!");
1549 return (ENOEXEC);
1550 }
1551 relalim = rela + ef->relatab[i].nrela;
1552 base = findbase(ef, ef->relatab[i].sec);
1553 if (base == 0) {
1554 link_elf_error(ef->lf.filename, "lost base for reltab");
1555 return (ENOEXEC);
1556 }
1557 for ( ; rela < relalim; rela++) {
1558 symidx = ELF_R_SYM(rela->r_info);
1559 if (symidx >= ef->ddbsymcnt)
1560 continue;
1561 sym = ef->ddbsymtab + symidx;
1562 /* Only do local relocs */
1563 if (ELF_ST_BIND(sym->st_info) != STB_LOCAL)
1564 continue;
1565 if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC ||
1566 elf_is_ifunc_reloc(rela->r_info)) == ifuncs)
1567 elf_reloc_local(lf, base, rela, ELF_RELOC_RELA,
1568 elf_obj_lookup);
1569 }
1570 }
1571 return (0);
1572}
1573
1574static long
1575link_elf_symtab_get(linker_file_t lf, const Elf_Sym **symtab)
1576{
1577 elf_file_t ef = (elf_file_t)lf;
1578
1579 *symtab = ef->ddbsymtab;
1580
1581 if (*symtab == NULL)
1582 return (0);
1583
1584 return (ef->ddbsymcnt);
1585}
1586
1587static long
1588link_elf_strtab_get(linker_file_t lf, caddr_t *strtab)
1589{
1590 elf_file_t ef = (elf_file_t)lf;
1591
1592 *strtab = ef->ddbstrtab;
1593
1594 if (*strtab == NULL)
1595 return (0);
1596
1597 return (ef->ddbstrcnt);
1598}
| 196
197static int
198link_elf_link_preload(linker_class_t cls, const char *filename,
199 linker_file_t *result)
200{
201 Elf_Ehdr *hdr;
202 Elf_Shdr *shdr;
203 Elf_Sym *es;
204 void *modptr, *baseptr, *sizeptr;
205 char *type;
206 elf_file_t ef;
207 linker_file_t lf;
208 Elf_Addr off;
209 int error, i, j, pb, ra, rl, shstrindex, symstrindex, symtabindex;
210
211 /* Look to see if we have the file preloaded */
212 modptr = preload_search_by_name(filename);
213 if (modptr == NULL)
214 return ENOENT;
215
216 type = (char *)preload_search_info(modptr, MODINFO_TYPE);
217 baseptr = preload_search_info(modptr, MODINFO_ADDR);
218 sizeptr = preload_search_info(modptr, MODINFO_SIZE);
219 hdr = (Elf_Ehdr *)preload_search_info(modptr, MODINFO_METADATA |
220 MODINFOMD_ELFHDR);
221 shdr = (Elf_Shdr *)preload_search_info(modptr, MODINFO_METADATA |
222 MODINFOMD_SHDR);
223 if (type == NULL || (strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE)
224 " obj module") != 0 &&
225 strcmp(type, "elf obj module") != 0)) {
226 return (EFTYPE);
227 }
228 if (baseptr == NULL || sizeptr == NULL || hdr == NULL ||
229 shdr == NULL)
230 return (EINVAL);
231
232 lf = linker_make_file(filename, &link_elf_class);
233 if (lf == NULL)
234 return (ENOMEM);
235
236 ef = (elf_file_t)lf;
237 ef->preloaded = 1;
238 ef->address = *(caddr_t *)baseptr;
239 lf->address = *(caddr_t *)baseptr;
240 lf->size = *(size_t *)sizeptr;
241
242 if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS ||
243 hdr->e_ident[EI_DATA] != ELF_TARG_DATA ||
244 hdr->e_ident[EI_VERSION] != EV_CURRENT ||
245 hdr->e_version != EV_CURRENT ||
246 hdr->e_type != ET_REL ||
247 hdr->e_machine != ELF_TARG_MACH) {
248 error = EFTYPE;
249 goto out;
250 }
251 ef->e_shdr = shdr;
252
253 /* Scan the section header for information and table sizing. */
254 symtabindex = -1;
255 symstrindex = -1;
256 for (i = 0; i < hdr->e_shnum; i++) {
257 switch (shdr[i].sh_type) {
258 case SHT_PROGBITS:
259 case SHT_NOBITS:
260#ifdef __amd64__
261 case SHT_X86_64_UNWIND:
262#endif
263 /* Ignore sections not loaded by the loader. */
264 if (shdr[i].sh_addr == 0)
265 break;
266 ef->nprogtab++;
267 break;
268 case SHT_SYMTAB:
269 symtabindex = i;
270 symstrindex = shdr[i].sh_link;
271 break;
272 case SHT_REL:
273 /*
274 * Ignore relocation tables for sections not
275 * loaded by the loader.
276 */
277 if (shdr[shdr[i].sh_info].sh_addr == 0)
278 break;
279 ef->nreltab++;
280 break;
281 case SHT_RELA:
282 if (shdr[shdr[i].sh_info].sh_addr == 0)
283 break;
284 ef->nrelatab++;
285 break;
286 }
287 }
288
289 shstrindex = hdr->e_shstrndx;
290 if (ef->nprogtab == 0 || symstrindex < 0 ||
291 symstrindex >= hdr->e_shnum ||
292 shdr[symstrindex].sh_type != SHT_STRTAB || shstrindex == 0 ||
293 shstrindex >= hdr->e_shnum ||
294 shdr[shstrindex].sh_type != SHT_STRTAB) {
295 printf("%s: bad/missing section headers\n", filename);
296 error = ENOEXEC;
297 goto out;
298 }
299
300 /* Allocate space for tracking the load chunks */
301 if (ef->nprogtab != 0)
302 ef->progtab = malloc(ef->nprogtab * sizeof(*ef->progtab),
303 M_LINKER, M_WAITOK | M_ZERO);
304 if (ef->nreltab != 0)
305 ef->reltab = malloc(ef->nreltab * sizeof(*ef->reltab),
306 M_LINKER, M_WAITOK | M_ZERO);
307 if (ef->nrelatab != 0)
308 ef->relatab = malloc(ef->nrelatab * sizeof(*ef->relatab),
309 M_LINKER, M_WAITOK | M_ZERO);
310 if ((ef->nprogtab != 0 && ef->progtab == NULL) ||
311 (ef->nreltab != 0 && ef->reltab == NULL) ||
312 (ef->nrelatab != 0 && ef->relatab == NULL)) {
313 error = ENOMEM;
314 goto out;
315 }
316
317 /* XXX, relocate the sh_addr fields saved by the loader. */
318 off = 0;
319 for (i = 0; i < hdr->e_shnum; i++) {
320 if (shdr[i].sh_addr != 0 && (off == 0 || shdr[i].sh_addr < off))
321 off = shdr[i].sh_addr;
322 }
323 for (i = 0; i < hdr->e_shnum; i++) {
324 if (shdr[i].sh_addr != 0)
325 shdr[i].sh_addr = shdr[i].sh_addr - off +
326 (Elf_Addr)ef->address;
327 }
328
329 ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym);
330 ef->ddbsymtab = (Elf_Sym *)shdr[symtabindex].sh_addr;
331 ef->ddbstrcnt = shdr[symstrindex].sh_size;
332 ef->ddbstrtab = (char *)shdr[symstrindex].sh_addr;
333 ef->shstrcnt = shdr[shstrindex].sh_size;
334 ef->shstrtab = (char *)shdr[shstrindex].sh_addr;
335
336 /* Now fill out progtab and the relocation tables. */
337 pb = 0;
338 rl = 0;
339 ra = 0;
340 for (i = 0; i < hdr->e_shnum; i++) {
341 switch (shdr[i].sh_type) {
342 case SHT_PROGBITS:
343 case SHT_NOBITS:
344#ifdef __amd64__
345 case SHT_X86_64_UNWIND:
346#endif
347 if (shdr[i].sh_addr == 0)
348 break;
349 ef->progtab[pb].addr = (void *)shdr[i].sh_addr;
350 if (shdr[i].sh_type == SHT_PROGBITS)
351 ef->progtab[pb].name = "<<PROGBITS>>";
352#ifdef __amd64__
353 else if (shdr[i].sh_type == SHT_X86_64_UNWIND)
354 ef->progtab[pb].name = "<<UNWIND>>";
355#endif
356 else
357 ef->progtab[pb].name = "<<NOBITS>>";
358 ef->progtab[pb].size = shdr[i].sh_size;
359 ef->progtab[pb].sec = i;
360 if (ef->shstrtab && shdr[i].sh_name != 0)
361 ef->progtab[pb].name =
362 ef->shstrtab + shdr[i].sh_name;
363 if (ef->progtab[pb].name != NULL &&
364 !strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) {
365 void *dpcpu;
366
367 dpcpu = dpcpu_alloc(shdr[i].sh_size);
368 if (dpcpu == NULL) {
369 printf("%s: pcpu module space is out "
370 "of space; cannot allocate %#jx "
371 "for %s\n", __func__,
372 (uintmax_t)shdr[i].sh_size,
373 filename);
374 error = ENOSPC;
375 goto out;
376 }
377 memcpy(dpcpu, ef->progtab[pb].addr,
378 ef->progtab[pb].size);
379 dpcpu_copy(dpcpu, shdr[i].sh_size);
380 ef->progtab[pb].addr = dpcpu;
381#ifdef VIMAGE
382 } else if (ef->progtab[pb].name != NULL &&
383 !strcmp(ef->progtab[pb].name, VNET_SETNAME)) {
384 void *vnet_data;
385
386 vnet_data = vnet_data_alloc(shdr[i].sh_size);
387 if (vnet_data == NULL) {
388 printf("%s: vnet module space is out "
389 "of space; cannot allocate %#jx "
390 "for %s\n", __func__,
391 (uintmax_t)shdr[i].sh_size,
392 filename);
393 error = ENOSPC;
394 goto out;
395 }
396 memcpy(vnet_data, ef->progtab[pb].addr,
397 ef->progtab[pb].size);
398 vnet_data_copy(vnet_data, shdr[i].sh_size);
399 ef->progtab[pb].addr = vnet_data;
400#endif
401 } else if (ef->progtab[pb].name != NULL &&
402 !strcmp(ef->progtab[pb].name, ".ctors")) {
403 lf->ctors_addr = ef->progtab[pb].addr;
404 lf->ctors_size = shdr[i].sh_size;
405 }
406
407 /* Update all symbol values with the offset. */
408 for (j = 0; j < ef->ddbsymcnt; j++) {
409 es = &ef->ddbsymtab[j];
410 if (es->st_shndx != i)
411 continue;
412 es->st_value += (Elf_Addr)ef->progtab[pb].addr;
413 }
414 pb++;
415 break;
416 case SHT_REL:
417 if (shdr[shdr[i].sh_info].sh_addr == 0)
418 break;
419 ef->reltab[rl].rel = (Elf_Rel *)shdr[i].sh_addr;
420 ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel);
421 ef->reltab[rl].sec = shdr[i].sh_info;
422 rl++;
423 break;
424 case SHT_RELA:
425 if (shdr[shdr[i].sh_info].sh_addr == 0)
426 break;
427 ef->relatab[ra].rela = (Elf_Rela *)shdr[i].sh_addr;
428 ef->relatab[ra].nrela =
429 shdr[i].sh_size / sizeof(Elf_Rela);
430 ef->relatab[ra].sec = shdr[i].sh_info;
431 ra++;
432 break;
433 }
434 }
435 if (pb != ef->nprogtab) {
436 printf("%s: lost progbits\n", filename);
437 error = ENOEXEC;
438 goto out;
439 }
440 if (rl != ef->nreltab) {
441 printf("%s: lost reltab\n", filename);
442 error = ENOEXEC;
443 goto out;
444 }
445 if (ra != ef->nrelatab) {
446 printf("%s: lost relatab\n", filename);
447 error = ENOEXEC;
448 goto out;
449 }
450
451 /* Local intra-module relocations */
452 error = link_elf_reloc_local(lf, false);
453 if (error != 0)
454 goto out;
455 *result = lf;
456 return (0);
457
458out:
459 /* preload not done this way */
460 linker_file_unload(lf, LINKER_UNLOAD_FORCE);
461 return (error);
462}
463
464static void
465link_elf_invoke_ctors(caddr_t addr, size_t size)
466{
467 void (**ctor)(void);
468 size_t i, cnt;
469
470 if (addr == NULL || size == 0)
471 return;
472 cnt = size / sizeof(*ctor);
473 ctor = (void *)addr;
474 for (i = 0; i < cnt; i++) {
475 if (ctor[i] != NULL)
476 (*ctor[i])();
477 }
478}
479
480static int
481link_elf_link_preload_finish(linker_file_t lf)
482{
483 elf_file_t ef;
484 int error;
485
486 ef = (elf_file_t)lf;
487 error = relocate_file(ef);
488 if (error)
489 return (error);
490
491 /* Notify MD code that a module is being loaded. */
492 error = elf_cpu_load_file(lf);
493 if (error)
494 return (error);
495
496#if defined(__i386__) || defined(__amd64__)
497 /* Now ifuncs. */
498 error = link_elf_reloc_local(lf, true);
499 if (error != 0)
500 return (error);
501#endif
502
503 /* Invoke .ctors */
504 link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size);
505 return (0);
506}
507
508static int
509link_elf_load_file(linker_class_t cls, const char *filename,
510 linker_file_t *result)
511{
512 struct nameidata nd;
513 struct thread *td = curthread; /* XXX */
514 Elf_Ehdr *hdr;
515 Elf_Shdr *shdr;
516 Elf_Sym *es;
517 int nbytes, i, j;
518 vm_offset_t mapbase;
519 size_t mapsize;
520 int error = 0;
521 ssize_t resid;
522 int flags;
523 elf_file_t ef;
524 linker_file_t lf;
525 int symtabindex;
526 int symstrindex;
527 int shstrindex;
528 int nsym;
529 int pb, rl, ra;
530 int alignmask;
531
532 shdr = NULL;
533 lf = NULL;
534 mapsize = 0;
535 hdr = NULL;
536
537 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td);
538 flags = FREAD;
539 error = vn_open(&nd, &flags, 0, NULL);
540 if (error)
541 return error;
542 NDFREE(&nd, NDF_ONLY_PNBUF);
543 if (nd.ni_vp->v_type != VREG) {
544 error = ENOEXEC;
545 goto out;
546 }
547#ifdef MAC
548 error = mac_kld_check_load(td->td_ucred, nd.ni_vp);
549 if (error) {
550 goto out;
551 }
552#endif
553
554 /* Read the elf header from the file. */
555 hdr = malloc(sizeof(*hdr), M_LINKER, M_WAITOK);
556 error = vn_rdwr(UIO_READ, nd.ni_vp, (void *)hdr, sizeof(*hdr), 0,
557 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
558 &resid, td);
559 if (error)
560 goto out;
561 if (resid != 0){
562 error = ENOEXEC;
563 goto out;
564 }
565
566 if (!IS_ELF(*hdr)) {
567 error = ENOEXEC;
568 goto out;
569 }
570
571 if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS
572 || hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
573 link_elf_error(filename, "Unsupported file layout");
574 error = ENOEXEC;
575 goto out;
576 }
577 if (hdr->e_ident[EI_VERSION] != EV_CURRENT
578 || hdr->e_version != EV_CURRENT) {
579 link_elf_error(filename, "Unsupported file version");
580 error = ENOEXEC;
581 goto out;
582 }
583 if (hdr->e_type != ET_REL) {
584 error = ENOSYS;
585 goto out;
586 }
587 if (hdr->e_machine != ELF_TARG_MACH) {
588 link_elf_error(filename, "Unsupported machine");
589 error = ENOEXEC;
590 goto out;
591 }
592
593 lf = linker_make_file(filename, &link_elf_class);
594 if (!lf) {
595 error = ENOMEM;
596 goto out;
597 }
598 ef = (elf_file_t) lf;
599 ef->nprogtab = 0;
600 ef->e_shdr = 0;
601 ef->nreltab = 0;
602 ef->nrelatab = 0;
603
604 /* Allocate and read in the section header */
605 nbytes = hdr->e_shnum * hdr->e_shentsize;
606 if (nbytes == 0 || hdr->e_shoff == 0 ||
607 hdr->e_shentsize != sizeof(Elf_Shdr)) {
608 error = ENOEXEC;
609 goto out;
610 }
611 shdr = malloc(nbytes, M_LINKER, M_WAITOK);
612 ef->e_shdr = shdr;
613 error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shdr, nbytes, hdr->e_shoff,
614 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td);
615 if (error)
616 goto out;
617 if (resid) {
618 error = ENOEXEC;
619 goto out;
620 }
621
622 /* Scan the section header for information and table sizing. */
623 nsym = 0;
624 symtabindex = -1;
625 symstrindex = -1;
626 for (i = 0; i < hdr->e_shnum; i++) {
627 if (shdr[i].sh_size == 0)
628 continue;
629 switch (shdr[i].sh_type) {
630 case SHT_PROGBITS:
631 case SHT_NOBITS:
632#ifdef __amd64__
633 case SHT_X86_64_UNWIND:
634#endif
635 if ((shdr[i].sh_flags & SHF_ALLOC) == 0)
636 break;
637 ef->nprogtab++;
638 break;
639 case SHT_SYMTAB:
640 nsym++;
641 symtabindex = i;
642 symstrindex = shdr[i].sh_link;
643 break;
644 case SHT_REL:
645 /*
646 * Ignore relocation tables for unallocated
647 * sections.
648 */
649 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0)
650 break;
651 ef->nreltab++;
652 break;
653 case SHT_RELA:
654 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0)
655 break;
656 ef->nrelatab++;
657 break;
658 case SHT_STRTAB:
659 break;
660 }
661 }
662 if (ef->nprogtab == 0) {
663 link_elf_error(filename, "file has no contents");
664 error = ENOEXEC;
665 goto out;
666 }
667 if (nsym != 1) {
668 /* Only allow one symbol table for now */
669 link_elf_error(filename, "file has no valid symbol table");
670 error = ENOEXEC;
671 goto out;
672 }
673 if (symstrindex < 0 || symstrindex > hdr->e_shnum ||
674 shdr[symstrindex].sh_type != SHT_STRTAB) {
675 link_elf_error(filename, "file has invalid symbol strings");
676 error = ENOEXEC;
677 goto out;
678 }
679
680 /* Allocate space for tracking the load chunks */
681 if (ef->nprogtab != 0)
682 ef->progtab = malloc(ef->nprogtab * sizeof(*ef->progtab),
683 M_LINKER, M_WAITOK | M_ZERO);
684 if (ef->nreltab != 0)
685 ef->reltab = malloc(ef->nreltab * sizeof(*ef->reltab),
686 M_LINKER, M_WAITOK | M_ZERO);
687 if (ef->nrelatab != 0)
688 ef->relatab = malloc(ef->nrelatab * sizeof(*ef->relatab),
689 M_LINKER, M_WAITOK | M_ZERO);
690
691 if (symtabindex == -1) {
692 link_elf_error(filename, "lost symbol table index");
693 error = ENOEXEC;
694 goto out;
695 }
696 /* Allocate space for and load the symbol table */
697 ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym);
698 ef->ddbsymtab = malloc(shdr[symtabindex].sh_size, M_LINKER, M_WAITOK);
699 error = vn_rdwr(UIO_READ, nd.ni_vp, (void *)ef->ddbsymtab,
700 shdr[symtabindex].sh_size, shdr[symtabindex].sh_offset,
701 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
702 &resid, td);
703 if (error)
704 goto out;
705 if (resid != 0){
706 error = EINVAL;
707 goto out;
708 }
709
710 /* Allocate space for and load the symbol strings */
711 ef->ddbstrcnt = shdr[symstrindex].sh_size;
712 ef->ddbstrtab = malloc(shdr[symstrindex].sh_size, M_LINKER, M_WAITOK);
713 error = vn_rdwr(UIO_READ, nd.ni_vp, ef->ddbstrtab,
714 shdr[symstrindex].sh_size, shdr[symstrindex].sh_offset,
715 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
716 &resid, td);
717 if (error)
718 goto out;
719 if (resid != 0){
720 error = EINVAL;
721 goto out;
722 }
723
724 /* Do we have a string table for the section names? */
725 shstrindex = -1;
726 if (hdr->e_shstrndx != 0 &&
727 shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) {
728 shstrindex = hdr->e_shstrndx;
729 ef->shstrcnt = shdr[shstrindex].sh_size;
730 ef->shstrtab = malloc(shdr[shstrindex].sh_size, M_LINKER,
731 M_WAITOK);
732 error = vn_rdwr(UIO_READ, nd.ni_vp, ef->shstrtab,
733 shdr[shstrindex].sh_size, shdr[shstrindex].sh_offset,
734 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
735 &resid, td);
736 if (error)
737 goto out;
738 if (resid != 0){
739 error = EINVAL;
740 goto out;
741 }
742 }
743
744 /* Size up code/data(progbits) and bss(nobits). */
745 alignmask = 0;
746 for (i = 0; i < hdr->e_shnum; i++) {
747 if (shdr[i].sh_size == 0)
748 continue;
749 switch (shdr[i].sh_type) {
750 case SHT_PROGBITS:
751 case SHT_NOBITS:
752#ifdef __amd64__
753 case SHT_X86_64_UNWIND:
754#endif
755 if ((shdr[i].sh_flags & SHF_ALLOC) == 0)
756 break;
757 alignmask = shdr[i].sh_addralign - 1;
758 mapsize += alignmask;
759 mapsize &= ~alignmask;
760 mapsize += shdr[i].sh_size;
761 break;
762 }
763 }
764
765 /*
766 * We know how much space we need for the text/data/bss/etc.
767 * This stuff needs to be in a single chunk so that profiling etc
768 * can get the bounds and gdb can associate offsets with modules
769 */
770 ef->object = vm_object_allocate(OBJT_DEFAULT,
771 round_page(mapsize) >> PAGE_SHIFT);
772 if (ef->object == NULL) {
773 error = ENOMEM;
774 goto out;
775 }
776 ef->address = (caddr_t) vm_map_min(kernel_map);
777
778 /*
779 * In order to satisfy amd64's architectural requirements on the
780 * location of code and data in the kernel's address space, request a
781 * mapping that is above the kernel.
782 */
783#ifdef __amd64__
784 mapbase = KERNBASE;
785#else
786 mapbase = VM_MIN_KERNEL_ADDRESS;
787#endif
788 error = vm_map_find(kernel_map, ef->object, 0, &mapbase,
789 round_page(mapsize), 0, VMFS_OPTIMAL_SPACE, VM_PROT_ALL,
790 VM_PROT_ALL, 0);
791 if (error) {
792 vm_object_deallocate(ef->object);
793 ef->object = 0;
794 goto out;
795 }
796
797 /* Wire the pages */
798 error = vm_map_wire(kernel_map, mapbase,
799 mapbase + round_page(mapsize),
800 VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES);
801 if (error != KERN_SUCCESS) {
802 error = ENOMEM;
803 goto out;
804 }
805
806 /* Inform the kld system about the situation */
807 lf->address = ef->address = (caddr_t)mapbase;
808 lf->size = mapsize;
809
810 /*
811 * Now load code/data(progbits), zero bss(nobits), allocate space for
812 * and load relocs
813 */
814 pb = 0;
815 rl = 0;
816 ra = 0;
817 alignmask = 0;
818 for (i = 0; i < hdr->e_shnum; i++) {
819 if (shdr[i].sh_size == 0)
820 continue;
821 switch (shdr[i].sh_type) {
822 case SHT_PROGBITS:
823 case SHT_NOBITS:
824#ifdef __amd64__
825 case SHT_X86_64_UNWIND:
826#endif
827 if ((shdr[i].sh_flags & SHF_ALLOC) == 0)
828 break;
829 alignmask = shdr[i].sh_addralign - 1;
830 mapbase += alignmask;
831 mapbase &= ~alignmask;
832 if (ef->shstrtab != NULL && shdr[i].sh_name != 0) {
833 ef->progtab[pb].name =
834 ef->shstrtab + shdr[i].sh_name;
835 if (!strcmp(ef->progtab[pb].name, ".ctors")) {
836 lf->ctors_addr = (caddr_t)mapbase;
837 lf->ctors_size = shdr[i].sh_size;
838 }
839 } else if (shdr[i].sh_type == SHT_PROGBITS)
840 ef->progtab[pb].name = "<<PROGBITS>>";
841#ifdef __amd64__
842 else if (shdr[i].sh_type == SHT_X86_64_UNWIND)
843 ef->progtab[pb].name = "<<UNWIND>>";
844#endif
845 else
846 ef->progtab[pb].name = "<<NOBITS>>";
847 if (ef->progtab[pb].name != NULL &&
848 !strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) {
849 ef->progtab[pb].addr =
850 dpcpu_alloc(shdr[i].sh_size);
851 if (ef->progtab[pb].addr == NULL) {
852 printf("%s: pcpu module space is out "
853 "of space; cannot allocate %#jx "
854 "for %s\n", __func__,
855 (uintmax_t)shdr[i].sh_size,
856 filename);
857 }
858 }
859#ifdef VIMAGE
860 else if (ef->progtab[pb].name != NULL &&
861 !strcmp(ef->progtab[pb].name, VNET_SETNAME)) {
862 ef->progtab[pb].addr =
863 vnet_data_alloc(shdr[i].sh_size);
864 if (ef->progtab[pb].addr == NULL) {
865 printf("%s: vnet module space is out "
866 "of space; cannot allocate %#jx "
867 "for %s\n", __func__,
868 (uintmax_t)shdr[i].sh_size,
869 filename);
870 }
871 }
872#endif
873 else
874 ef->progtab[pb].addr =
875 (void *)(uintptr_t)mapbase;
876 if (ef->progtab[pb].addr == NULL) {
877 error = ENOSPC;
878 goto out;
879 }
880 ef->progtab[pb].size = shdr[i].sh_size;
881 ef->progtab[pb].sec = i;
882 if (shdr[i].sh_type == SHT_PROGBITS
883#ifdef __amd64__
884 || shdr[i].sh_type == SHT_X86_64_UNWIND
885#endif
886 ) {
887 error = vn_rdwr(UIO_READ, nd.ni_vp,
888 ef->progtab[pb].addr,
889 shdr[i].sh_size, shdr[i].sh_offset,
890 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred,
891 NOCRED, &resid, td);
892 if (error)
893 goto out;
894 if (resid != 0){
895 error = EINVAL;
896 goto out;
897 }
898 /* Initialize the per-cpu or vnet area. */
899 if (ef->progtab[pb].addr != (void *)mapbase &&
900 !strcmp(ef->progtab[pb].name, DPCPU_SETNAME))
901 dpcpu_copy(ef->progtab[pb].addr,
902 shdr[i].sh_size);
903#ifdef VIMAGE
904 else if (ef->progtab[pb].addr !=
905 (void *)mapbase &&
906 !strcmp(ef->progtab[pb].name, VNET_SETNAME))
907 vnet_data_copy(ef->progtab[pb].addr,
908 shdr[i].sh_size);
909#endif
910 } else
911 bzero(ef->progtab[pb].addr, shdr[i].sh_size);
912
913 /* Update all symbol values with the offset. */
914 for (j = 0; j < ef->ddbsymcnt; j++) {
915 es = &ef->ddbsymtab[j];
916 if (es->st_shndx != i)
917 continue;
918 es->st_value += (Elf_Addr)ef->progtab[pb].addr;
919 }
920 mapbase += shdr[i].sh_size;
921 pb++;
922 break;
923 case SHT_REL:
924 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0)
925 break;
926 ef->reltab[rl].rel = malloc(shdr[i].sh_size, M_LINKER,
927 M_WAITOK);
928 ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel);
929 ef->reltab[rl].sec = shdr[i].sh_info;
930 error = vn_rdwr(UIO_READ, nd.ni_vp,
931 (void *)ef->reltab[rl].rel,
932 shdr[i].sh_size, shdr[i].sh_offset,
933 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
934 &resid, td);
935 if (error)
936 goto out;
937 if (resid != 0){
938 error = EINVAL;
939 goto out;
940 }
941 rl++;
942 break;
943 case SHT_RELA:
944 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0)
945 break;
946 ef->relatab[ra].rela = malloc(shdr[i].sh_size, M_LINKER,
947 M_WAITOK);
948 ef->relatab[ra].nrela =
949 shdr[i].sh_size / sizeof(Elf_Rela);
950 ef->relatab[ra].sec = shdr[i].sh_info;
951 error = vn_rdwr(UIO_READ, nd.ni_vp,
952 (void *)ef->relatab[ra].rela,
953 shdr[i].sh_size, shdr[i].sh_offset,
954 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
955 &resid, td);
956 if (error)
957 goto out;
958 if (resid != 0){
959 error = EINVAL;
960 goto out;
961 }
962 ra++;
963 break;
964 }
965 }
966 if (pb != ef->nprogtab) {
967 link_elf_error(filename, "lost progbits");
968 error = ENOEXEC;
969 goto out;
970 }
971 if (rl != ef->nreltab) {
972 link_elf_error(filename, "lost reltab");
973 error = ENOEXEC;
974 goto out;
975 }
976 if (ra != ef->nrelatab) {
977 link_elf_error(filename, "lost relatab");
978 error = ENOEXEC;
979 goto out;
980 }
981 if (mapbase != (vm_offset_t)ef->address + mapsize) {
982 printf(
983 "%s: mapbase 0x%lx != address %p + mapsize 0x%lx (0x%lx)\n",
984 filename != NULL ? filename : "<none>",
985 (u_long)mapbase, ef->address, (u_long)mapsize,
986 (u_long)(vm_offset_t)ef->address + mapsize);
987 error = ENOMEM;
988 goto out;
989 }
990
991 /* Local intra-module relocations */
992 error = link_elf_reloc_local(lf, false);
993 if (error != 0)
994 goto out;
995
996 /* Pull in dependencies */
997 VOP_UNLOCK(nd.ni_vp, 0);
998 error = linker_load_dependencies(lf);
999 vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY);
1000 if (error)
1001 goto out;
1002
1003 /* External relocations */
1004 error = relocate_file(ef);
1005 if (error)
1006 goto out;
1007
1008 /* Notify MD code that a module is being loaded. */
1009 error = elf_cpu_load_file(lf);
1010 if (error)
1011 goto out;
1012
1013#if defined(__i386__) || defined(__amd64__)
1014 /* Now ifuncs. */
1015 error = link_elf_reloc_local(lf, true);
1016 if (error != 0)
1017 goto out;
1018#endif
1019
1020 /* Invoke .ctors */
1021 link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size);
1022
1023 *result = lf;
1024
1025out:
1026 VOP_UNLOCK(nd.ni_vp, 0);
1027 vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
1028 if (error && lf)
1029 linker_file_unload(lf, LINKER_UNLOAD_FORCE);
1030 free(hdr, M_LINKER);
1031
1032 return error;
1033}
1034
1035static void
1036link_elf_unload_file(linker_file_t file)
1037{
1038 elf_file_t ef = (elf_file_t) file;
1039 int i;
1040
1041 /* Notify MD code that a module is being unloaded. */
1042 elf_cpu_unload_file(file);
1043
1044 if (ef->progtab) {
1045 for (i = 0; i < ef->nprogtab; i++) {
1046 if (ef->progtab[i].size == 0)
1047 continue;
1048 if (ef->progtab[i].name == NULL)
1049 continue;
1050 if (!strcmp(ef->progtab[i].name, DPCPU_SETNAME))
1051 dpcpu_free(ef->progtab[i].addr,
1052 ef->progtab[i].size);
1053#ifdef VIMAGE
1054 else if (!strcmp(ef->progtab[i].name, VNET_SETNAME))
1055 vnet_data_free(ef->progtab[i].addr,
1056 ef->progtab[i].size);
1057#endif
1058 }
1059 }
1060 if (ef->preloaded) {
1061 free(ef->reltab, M_LINKER);
1062 free(ef->relatab, M_LINKER);
1063 free(ef->progtab, M_LINKER);
1064 free(ef->ctftab, M_LINKER);
1065 free(ef->ctfoff, M_LINKER);
1066 free(ef->typoff, M_LINKER);
1067 if (file->pathname != NULL)
1068 preload_delete_name(file->pathname);
1069 return;
1070 }
1071
1072 for (i = 0; i < ef->nreltab; i++)
1073 free(ef->reltab[i].rel, M_LINKER);
1074 for (i = 0; i < ef->nrelatab; i++)
1075 free(ef->relatab[i].rela, M_LINKER);
1076 free(ef->reltab, M_LINKER);
1077 free(ef->relatab, M_LINKER);
1078 free(ef->progtab, M_LINKER);
1079
1080 if (ef->object) {
1081 vm_map_remove(kernel_map, (vm_offset_t) ef->address,
1082 (vm_offset_t) ef->address +
1083 (ef->object->size << PAGE_SHIFT));
1084 }
1085 free(ef->e_shdr, M_LINKER);
1086 free(ef->ddbsymtab, M_LINKER);
1087 free(ef->ddbstrtab, M_LINKER);
1088 free(ef->shstrtab, M_LINKER);
1089 free(ef->ctftab, M_LINKER);
1090 free(ef->ctfoff, M_LINKER);
1091 free(ef->typoff, M_LINKER);
1092}
1093
1094static const char *
1095symbol_name(elf_file_t ef, Elf_Size r_info)
1096{
1097 const Elf_Sym *ref;
1098
1099 if (ELF_R_SYM(r_info)) {
1100 ref = ef->ddbsymtab + ELF_R_SYM(r_info);
1101 return ef->ddbstrtab + ref->st_name;
1102 } else
1103 return NULL;
1104}
1105
1106static Elf_Addr
1107findbase(elf_file_t ef, int sec)
1108{
1109 int i;
1110 Elf_Addr base = 0;
1111
1112 for (i = 0; i < ef->nprogtab; i++) {
1113 if (sec == ef->progtab[i].sec) {
1114 base = (Elf_Addr)ef->progtab[i].addr;
1115 break;
1116 }
1117 }
1118 return base;
1119}
1120
1121static int
1122relocate_file(elf_file_t ef)
1123{
1124 const Elf_Rel *rellim;
1125 const Elf_Rel *rel;
1126 const Elf_Rela *relalim;
1127 const Elf_Rela *rela;
1128 const char *symname;
1129 const Elf_Sym *sym;
1130 int i;
1131 Elf_Size symidx;
1132 Elf_Addr base;
1133
1134
1135 /* Perform relocations without addend if there are any: */
1136 for (i = 0; i < ef->nreltab; i++) {
1137 rel = ef->reltab[i].rel;
1138 if (rel == NULL) {
1139 link_elf_error(ef->lf.filename, "lost a reltab!");
1140 return (ENOEXEC);
1141 }
1142 rellim = rel + ef->reltab[i].nrel;
1143 base = findbase(ef, ef->reltab[i].sec);
1144 if (base == 0) {
1145 link_elf_error(ef->lf.filename, "lost base for reltab");
1146 return (ENOEXEC);
1147 }
1148 for ( ; rel < rellim; rel++) {
1149 symidx = ELF_R_SYM(rel->r_info);
1150 if (symidx >= ef->ddbsymcnt)
1151 continue;
1152 sym = ef->ddbsymtab + symidx;
1153 /* Local relocs are already done */
1154 if (ELF_ST_BIND(sym->st_info) == STB_LOCAL)
1155 continue;
1156 if (elf_reloc(&ef->lf, base, rel, ELF_RELOC_REL,
1157 elf_obj_lookup)) {
1158 symname = symbol_name(ef, rel->r_info);
1159 printf("link_elf_obj: symbol %s undefined\n",
1160 symname);
1161 return (ENOENT);
1162 }
1163 }
1164 }
1165
1166 /* Perform relocations with addend if there are any: */
1167 for (i = 0; i < ef->nrelatab; i++) {
1168 rela = ef->relatab[i].rela;
1169 if (rela == NULL) {
1170 link_elf_error(ef->lf.filename, "lost a relatab!");
1171 return (ENOEXEC);
1172 }
1173 relalim = rela + ef->relatab[i].nrela;
1174 base = findbase(ef, ef->relatab[i].sec);
1175 if (base == 0) {
1176 link_elf_error(ef->lf.filename,
1177 "lost base for relatab");
1178 return (ENOEXEC);
1179 }
1180 for ( ; rela < relalim; rela++) {
1181 symidx = ELF_R_SYM(rela->r_info);
1182 if (symidx >= ef->ddbsymcnt)
1183 continue;
1184 sym = ef->ddbsymtab + symidx;
1185 /* Local relocs are already done */
1186 if (ELF_ST_BIND(sym->st_info) == STB_LOCAL)
1187 continue;
1188 if (elf_reloc(&ef->lf, base, rela, ELF_RELOC_RELA,
1189 elf_obj_lookup)) {
1190 symname = symbol_name(ef, rela->r_info);
1191 printf("link_elf_obj: symbol %s undefined\n",
1192 symname);
1193 return (ENOENT);
1194 }
1195 }
1196 }
1197
1198 /*
1199 * Only clean SHN_FBSD_CACHED for successful return. If we
1200 * modified symbol table for the object but found an
1201 * unresolved symbol, there is no reason to roll back.
1202 */
1203 elf_obj_cleanup_globals_cache(ef);
1204
1205 return (0);
1206}
1207
1208static int
1209link_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t *sym)
1210{
1211 elf_file_t ef = (elf_file_t) lf;
1212 const Elf_Sym *symp;
1213 const char *strp;
1214 int i;
1215
1216 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
1217 strp = ef->ddbstrtab + symp->st_name;
1218 if (symp->st_shndx != SHN_UNDEF && strcmp(name, strp) == 0) {
1219 *sym = (c_linker_sym_t) symp;
1220 return 0;
1221 }
1222 }
1223 return ENOENT;
1224}
1225
1226static int
1227link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym,
1228 linker_symval_t *symval)
1229{
1230 elf_file_t ef;
1231 const Elf_Sym *es;
1232 caddr_t val;
1233
1234 ef = (elf_file_t) lf;
1235 es = (const Elf_Sym*) sym;
1236 val = (caddr_t)es->st_value;
1237 if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {
1238 symval->name = ef->ddbstrtab + es->st_name;
1239 val = (caddr_t)es->st_value;
1240 if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC)
1241 val = ((caddr_t (*)(void))val)();
1242 symval->value = val;
1243 symval->size = es->st_size;
1244 return 0;
1245 }
1246 return ENOENT;
1247}
1248
1249static int
1250link_elf_search_symbol(linker_file_t lf, caddr_t value,
1251 c_linker_sym_t *sym, long *diffp)
1252{
1253 elf_file_t ef = (elf_file_t) lf;
1254 u_long off = (uintptr_t) (void *) value;
1255 u_long diff = off;
1256 u_long st_value;
1257 const Elf_Sym *es;
1258 const Elf_Sym *best = NULL;
1259 int i;
1260
1261 for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) {
1262 if (es->st_name == 0)
1263 continue;
1264 st_value = es->st_value;
1265 if (off >= st_value) {
1266 if (off - st_value < diff) {
1267 diff = off - st_value;
1268 best = es;
1269 if (diff == 0)
1270 break;
1271 } else if (off - st_value == diff) {
1272 best = es;
1273 }
1274 }
1275 }
1276 if (best == NULL)
1277 *diffp = off;
1278 else
1279 *diffp = diff;
1280 *sym = (c_linker_sym_t) best;
1281
1282 return 0;
1283}
1284
1285/*
1286 * Look up a linker set on an ELF system.
1287 */
1288static int
1289link_elf_lookup_set(linker_file_t lf, const char *name,
1290 void ***startp, void ***stopp, int *countp)
1291{
1292 elf_file_t ef = (elf_file_t)lf;
1293 void **start, **stop;
1294 int i, count;
1295
1296 /* Relative to section number */
1297 for (i = 0; i < ef->nprogtab; i++) {
1298 if ((strncmp(ef->progtab[i].name, "set_", 4) == 0) &&
1299 strcmp(ef->progtab[i].name + 4, name) == 0) {
1300 start = (void **)ef->progtab[i].addr;
1301 stop = (void **)((char *)ef->progtab[i].addr +
1302 ef->progtab[i].size);
1303 count = stop - start;
1304 if (startp)
1305 *startp = start;
1306 if (stopp)
1307 *stopp = stop;
1308 if (countp)
1309 *countp = count;
1310 return (0);
1311 }
1312 }
1313 return (ESRCH);
1314}
1315
1316static int
1317link_elf_each_function_name(linker_file_t file,
1318 int (*callback)(const char *, void *), void *opaque)
1319{
1320 elf_file_t ef = (elf_file_t)file;
1321 const Elf_Sym *symp;
1322 int i, error;
1323
1324 /* Exhaustive search */
1325 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
1326 if (symp->st_value != 0 &&
1327 (ELF_ST_TYPE(symp->st_info) == STT_FUNC ||
1328 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) {
1329 error = callback(ef->ddbstrtab + symp->st_name, opaque);
1330 if (error)
1331 return (error);
1332 }
1333 }
1334 return (0);
1335}
1336
1337static int
1338link_elf_each_function_nameval(linker_file_t file,
1339 linker_function_nameval_callback_t callback, void *opaque)
1340{
1341 linker_symval_t symval;
1342 elf_file_t ef = (elf_file_t)file;
1343 const Elf_Sym* symp;
1344 int i, error;
1345
1346 /* Exhaustive search */
1347 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
1348 if (symp->st_value != 0 &&
1349 (ELF_ST_TYPE(symp->st_info) == STT_FUNC ||
1350 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) {
1351 error = link_elf_symbol_values(file,
1352 (c_linker_sym_t)symp, &symval);
1353 if (error)
1354 return (error);
1355 error = callback(file, i, &symval, opaque);
1356 if (error)
1357 return (error);
1358 }
1359 }
1360 return (0);
1361}
1362
1363static void
1364elf_obj_cleanup_globals_cache(elf_file_t ef)
1365{
1366 Elf_Sym *sym;
1367 Elf_Size i;
1368
1369 for (i = 0; i < ef->ddbsymcnt; i++) {
1370 sym = ef->ddbsymtab + i;
1371 if (sym->st_shndx == SHN_FBSD_CACHED) {
1372 sym->st_shndx = SHN_UNDEF;
1373 sym->st_value = 0;
1374 }
1375 }
1376}
1377
1378/*
1379 * Symbol lookup function that can be used when the symbol index is known (ie
1380 * in relocations). It uses the symbol index instead of doing a fully fledged
1381 * hash table based lookup when such is valid. For example for local symbols.
1382 * This is not only more efficient, it's also more correct. It's not always
1383 * the case that the symbol can be found through the hash table.
1384 */
1385static int
1386elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res)
1387{
1388 elf_file_t ef = (elf_file_t)lf;
1389 Elf_Sym *sym;
1390 const char *symbol;
1391 Elf_Addr res1;
1392
1393 /* Don't even try to lookup the symbol if the index is bogus. */
1394 if (symidx >= ef->ddbsymcnt) {
1395 *res = 0;
1396 return (EINVAL);
1397 }
1398
1399 sym = ef->ddbsymtab + symidx;
1400
1401 /* Quick answer if there is a definition included. */
1402 if (sym->st_shndx != SHN_UNDEF) {
1403 res1 = (Elf_Addr)sym->st_value;
1404 if (ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC)
1405 res1 = ((Elf_Addr (*)(void))res1)();
1406 *res = res1;
1407 return (0);
1408 }
1409
1410 /* If we get here, then it is undefined and needs a lookup. */
1411 switch (ELF_ST_BIND(sym->st_info)) {
1412 case STB_LOCAL:
1413 /* Local, but undefined? huh? */
1414 *res = 0;
1415 return (EINVAL);
1416
1417 case STB_GLOBAL:
1418 case STB_WEAK:
1419 /* Relative to Data or Function name */
1420 symbol = ef->ddbstrtab + sym->st_name;
1421
1422 /* Force a lookup failure if the symbol name is bogus. */
1423 if (*symbol == 0) {
1424 *res = 0;
1425 return (EINVAL);
1426 }
1427 res1 = (Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps);
1428
1429 /*
1430 * Cache global lookups during module relocation. The failure
1431 * case is particularly expensive for callers, who must scan
1432 * through the entire globals table doing strcmp(). Cache to
1433 * avoid doing such work repeatedly.
1434 *
1435 * After relocation is complete, undefined globals will be
1436 * restored to SHN_UNDEF in elf_obj_cleanup_globals_cache(),
1437 * above.
1438 */
1439 if (res1 != 0) {
1440 sym->st_shndx = SHN_FBSD_CACHED;
1441 sym->st_value = res1;
1442 *res = res1;
1443 return (0);
1444 } else if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
1445 sym->st_value = 0;
1446 *res = 0;
1447 return (0);
1448 }
1449 return (EINVAL);
1450
1451 default:
1452 return (EINVAL);
1453 }
1454}
1455
1456static void
1457link_elf_fix_link_set(elf_file_t ef)
1458{
1459 static const char startn[] = "__start_";
1460 static const char stopn[] = "__stop_";
1461 Elf_Sym *sym;
1462 const char *sym_name, *linkset_name;
1463 Elf_Addr startp, stopp;
1464 Elf_Size symidx;
1465 int start, i;
1466
1467 startp = stopp = 0;
1468 for (symidx = 1 /* zero entry is special */;
1469 symidx < ef->ddbsymcnt; symidx++) {
1470 sym = ef->ddbsymtab + symidx;
1471 if (sym->st_shndx != SHN_UNDEF)
1472 continue;
1473
1474 sym_name = ef->ddbstrtab + sym->st_name;
1475 if (strncmp(sym_name, startn, sizeof(startn) - 1) == 0) {
1476 start = 1;
1477 linkset_name = sym_name + sizeof(startn) - 1;
1478 }
1479 else if (strncmp(sym_name, stopn, sizeof(stopn) - 1) == 0) {
1480 start = 0;
1481 linkset_name = sym_name + sizeof(stopn) - 1;
1482 }
1483 else
1484 continue;
1485
1486 for (i = 0; i < ef->nprogtab; i++) {
1487 if (strcmp(ef->progtab[i].name, linkset_name) == 0) {
1488 startp = (Elf_Addr)ef->progtab[i].addr;
1489 stopp = (Elf_Addr)(startp + ef->progtab[i].size);
1490 break;
1491 }
1492 }
1493 if (i == ef->nprogtab)
1494 continue;
1495
1496 sym->st_value = start ? startp : stopp;
1497 sym->st_shndx = i;
1498 }
1499}
1500
1501static int
1502link_elf_reloc_local(linker_file_t lf, bool ifuncs)
1503{
1504 elf_file_t ef = (elf_file_t)lf;
1505 const Elf_Rel *rellim;
1506 const Elf_Rel *rel;
1507 const Elf_Rela *relalim;
1508 const Elf_Rela *rela;
1509 const Elf_Sym *sym;
1510 Elf_Addr base;
1511 int i;
1512 Elf_Size symidx;
1513
1514 link_elf_fix_link_set(ef);
1515
1516 /* Perform relocations without addend if there are any: */
1517 for (i = 0; i < ef->nreltab; i++) {
1518 rel = ef->reltab[i].rel;
1519 if (rel == NULL) {
1520 link_elf_error(ef->lf.filename, "lost a reltab");
1521 return (ENOEXEC);
1522 }
1523 rellim = rel + ef->reltab[i].nrel;
1524 base = findbase(ef, ef->reltab[i].sec);
1525 if (base == 0) {
1526 link_elf_error(ef->lf.filename, "lost base for reltab");
1527 return (ENOEXEC);
1528 }
1529 for ( ; rel < rellim; rel++) {
1530 symidx = ELF_R_SYM(rel->r_info);
1531 if (symidx >= ef->ddbsymcnt)
1532 continue;
1533 sym = ef->ddbsymtab + symidx;
1534 /* Only do local relocs */
1535 if (ELF_ST_BIND(sym->st_info) != STB_LOCAL)
1536 continue;
1537 if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC ||
1538 elf_is_ifunc_reloc(rel->r_info)) == ifuncs)
1539 elf_reloc_local(lf, base, rel, ELF_RELOC_REL,
1540 elf_obj_lookup);
1541 }
1542 }
1543
1544 /* Perform relocations with addend if there are any: */
1545 for (i = 0; i < ef->nrelatab; i++) {
1546 rela = ef->relatab[i].rela;
1547 if (rela == NULL) {
1548 link_elf_error(ef->lf.filename, "lost a relatab!");
1549 return (ENOEXEC);
1550 }
1551 relalim = rela + ef->relatab[i].nrela;
1552 base = findbase(ef, ef->relatab[i].sec);
1553 if (base == 0) {
1554 link_elf_error(ef->lf.filename, "lost base for reltab");
1555 return (ENOEXEC);
1556 }
1557 for ( ; rela < relalim; rela++) {
1558 symidx = ELF_R_SYM(rela->r_info);
1559 if (symidx >= ef->ddbsymcnt)
1560 continue;
1561 sym = ef->ddbsymtab + symidx;
1562 /* Only do local relocs */
1563 if (ELF_ST_BIND(sym->st_info) != STB_LOCAL)
1564 continue;
1565 if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC ||
1566 elf_is_ifunc_reloc(rela->r_info)) == ifuncs)
1567 elf_reloc_local(lf, base, rela, ELF_RELOC_RELA,
1568 elf_obj_lookup);
1569 }
1570 }
1571 return (0);
1572}
1573
1574static long
1575link_elf_symtab_get(linker_file_t lf, const Elf_Sym **symtab)
1576{
1577 elf_file_t ef = (elf_file_t)lf;
1578
1579 *symtab = ef->ddbsymtab;
1580
1581 if (*symtab == NULL)
1582 return (0);
1583
1584 return (ef->ddbsymcnt);
1585}
1586
1587static long
1588link_elf_strtab_get(linker_file_t lf, caddr_t *strtab)
1589{
1590 elf_file_t ef = (elf_file_t)lf;
1591
1592 *strtab = ef->ddbstrtab;
1593
1594 if (*strtab == NULL)
1595 return (0);
1596
1597 return (ef->ddbstrcnt);
1598}
|