1/* 2 * IA-64-specific support for kernel module loader. 3 * 4 * Copyright (C) 2003 Hewlett-Packard Co 5 * David Mosberger-Tang <davidm@hpl.hp.com> 6 * 7 * Loosely based on patch by Rusty Russell. 8 */ 9 10/* relocs tested so far: 11 12 DIR64LSB 13 FPTR64LSB 14 GPREL22 15 LDXMOV 16 LDXMOV 17 LTOFF22 18 LTOFF22X 19 LTOFF22X 20 LTOFF_FPTR22 21 PCREL21B (for br.call only; br.cond is not supported out of modules!) 22 PCREL60B (for brl.cond only; brl.call is not supported for modules!) 23 PCREL64LSB 24 SECREL32LSB 25 SEGREL64LSB 26 */ 27 28 29#include <linux/kernel.h> 30#include <linux/sched.h> 31#include <linux/elf.h> 32#include <linux/moduleloader.h> 33#include <linux/string.h> 34#include <linux/vmalloc.h> 35 36#include <asm/patch.h> 37#include <asm/unaligned.h> 38 39#define ARCH_MODULE_DEBUG 0 40 41#if ARCH_MODULE_DEBUG 42# define DEBUGP printk 43# define inline 44#else 45# define DEBUGP(fmt , a...) 46#endif 47 48#ifdef CONFIG_ITANIUM 49# define USE_BRL 0 50#else 51# define USE_BRL 1 52#endif 53 54#define MAX_LTOFF ((uint64_t) (1 << 22)) /* max. allowable linkage-table offset */ 55 56/* Define some relocation helper macros/types: */ 57 58#define FORMAT_SHIFT 0 59#define FORMAT_BITS 3 60#define FORMAT_MASK ((1 << FORMAT_BITS) - 1) 61#define VALUE_SHIFT 3 62#define VALUE_BITS 5 63#define VALUE_MASK ((1 << VALUE_BITS) - 1) 64 65enum reloc_target_format { 66 /* direct encoded formats: */ 67 RF_NONE = 0, 68 RF_INSN14 = 1, 69 RF_INSN22 = 2, 70 RF_INSN64 = 3, 71 RF_32MSB = 4, 72 RF_32LSB = 5, 73 RF_64MSB = 6, 74 RF_64LSB = 7, 75 76 /* formats that cannot be directly decoded: */ 77 RF_INSN60, 78 RF_INSN21B, /* imm21 form 1 */ 79 RF_INSN21M, /* imm21 form 2 */ 80 RF_INSN21F /* imm21 form 3 */ 81}; 82 83enum reloc_value_formula { 84 RV_DIRECT = 4, /* S + A */ 85 RV_GPREL = 5, /* @gprel(S + A) */ 86 RV_LTREL = 6, /* @ltoff(S + A) */ 87 RV_PLTREL = 7, /* @pltoff(S + A) */ 88 RV_FPTR = 8, /* @fptr(S + A) */ 89 RV_PCREL = 9, /* S + A - P */ 90 RV_LTREL_FPTR = 10, /* @ltoff(@fptr(S + A)) */ 91 RV_SEGREL = 11, /* @segrel(S + A) */ 92 RV_SECREL = 12, /* @secrel(S + A) */ 93 RV_BDREL = 13, /* BD + A */ 94 RV_LTV = 14, /* S + A (like RV_DIRECT, except frozen at static link-time) */ 95 RV_PCREL2 = 15, /* S + A - P */ 96 RV_SPECIAL = 16, /* various (see below) */ 97 RV_RSVD17 = 17, 98 RV_TPREL = 18, /* @tprel(S + A) */ 99 RV_LTREL_TPREL = 19, /* @ltoff(@tprel(S + A)) */ 100 RV_DTPMOD = 20, /* @dtpmod(S + A) */ 101 RV_LTREL_DTPMOD = 21, /* @ltoff(@dtpmod(S + A)) */ 102 RV_DTPREL = 22, /* @dtprel(S + A) */ 103 RV_LTREL_DTPREL = 23, /* @ltoff(@dtprel(S + A)) */ 104 RV_RSVD24 = 24, 105 RV_RSVD25 = 25, 106 RV_RSVD26 = 26, 107 RV_RSVD27 = 27 108 /* 28-31 reserved for implementation-specific purposes. */ 109}; 110 111#define N(reloc) [R_IA64_##reloc] = #reloc 112 113static const char *reloc_name[256] = { 114 N(NONE), N(IMM14), N(IMM22), N(IMM64), 115 N(DIR32MSB), N(DIR32LSB), N(DIR64MSB), N(DIR64LSB), 116 N(GPREL22), N(GPREL64I), N(GPREL32MSB), N(GPREL32LSB), 117 N(GPREL64MSB), N(GPREL64LSB), N(LTOFF22), N(LTOFF64I), 118 N(PLTOFF22), N(PLTOFF64I), N(PLTOFF64MSB), N(PLTOFF64LSB), 119 N(FPTR64I), N(FPTR32MSB), N(FPTR32LSB), N(FPTR64MSB), 120 N(FPTR64LSB), N(PCREL60B), N(PCREL21B), N(PCREL21M), 121 N(PCREL21F), N(PCREL32MSB), N(PCREL32LSB), N(PCREL64MSB), 122 N(PCREL64LSB), N(LTOFF_FPTR22), N(LTOFF_FPTR64I), N(LTOFF_FPTR32MSB), 123 N(LTOFF_FPTR32LSB), N(LTOFF_FPTR64MSB), N(LTOFF_FPTR64LSB), N(SEGREL32MSB), 124 N(SEGREL32LSB), N(SEGREL64MSB), N(SEGREL64LSB), N(SECREL32MSB), 125 N(SECREL32LSB), N(SECREL64MSB), N(SECREL64LSB), N(REL32MSB), 126 N(REL32LSB), N(REL64MSB), N(REL64LSB), N(LTV32MSB), 127 N(LTV32LSB), N(LTV64MSB), N(LTV64LSB), N(PCREL21BI), 128 N(PCREL22), N(PCREL64I), N(IPLTMSB), N(IPLTLSB), 129 N(COPY), N(LTOFF22X), N(LDXMOV), N(TPREL14), 130 N(TPREL22), N(TPREL64I), N(TPREL64MSB), N(TPREL64LSB), 131 N(LTOFF_TPREL22), N(DTPMOD64MSB), N(DTPMOD64LSB), N(LTOFF_DTPMOD22), 132 N(DTPREL14), N(DTPREL22), N(DTPREL64I), N(DTPREL32MSB), 133 N(DTPREL32LSB), N(DTPREL64MSB), N(DTPREL64LSB), N(LTOFF_DTPREL22) 134}; 135 136#undef N 137 138struct got_entry { 139 uint64_t val; 140}; 141 142struct fdesc { 143 uint64_t ip; 144 uint64_t gp; 145}; 146 147/* Opaque struct for insns, to protect against derefs. */ 148struct insn; 149 150static inline uint64_t 151bundle (const struct insn *insn) 152{ 153 return (uint64_t) insn & ~0xfUL; 154} 155 156static inline int 157slot (const struct insn *insn) 158{ 159 return (uint64_t) insn & 0x3; 160} 161 162static int 163apply_imm64 (struct module *mod, struct insn *insn, uint64_t val) 164{ 165 if (slot(insn) != 2) { 166 printk(KERN_ERR "%s: invalid slot number %d for IMM64\n", 167 mod->name, slot(insn)); 168 return 0; 169 } 170 ia64_patch_imm64((u64) insn, val); 171 return 1; 172} 173 174static int 175apply_imm60 (struct module *mod, struct insn *insn, uint64_t val) 176{ 177 if (slot(insn) != 2) { 178 printk(KERN_ERR "%s: invalid slot number %d for IMM60\n", 179 mod->name, slot(insn)); 180 return 0; 181 } 182 if (val + ((uint64_t) 1 << 59) >= (1UL << 60)) { 183 printk(KERN_ERR "%s: value %ld out of IMM60 range\n", mod->name, (int64_t) val); 184 return 0; 185 } 186 ia64_patch_imm60((u64) insn, val); 187 return 1; 188} 189 190static int 191apply_imm22 (struct module *mod, struct insn *insn, uint64_t val) 192{ 193 if (val + (1 << 21) >= (1 << 22)) { 194 printk(KERN_ERR "%s: value %li out of IMM22 range\n", mod->name, (int64_t)val); 195 return 0; 196 } 197 ia64_patch((u64) insn, 0x01fffcfe000UL, ( ((val & 0x200000UL) << 15) /* bit 21 -> 36 */ 198 | ((val & 0x1f0000UL) << 6) /* bit 16 -> 22 */ 199 | ((val & 0x00ff80UL) << 20) /* bit 7 -> 27 */ 200 | ((val & 0x00007fUL) << 13) /* bit 0 -> 13 */)); 201 return 1; 202} 203 204static int 205apply_imm21b (struct module *mod, struct insn *insn, uint64_t val) 206{ 207 if (val + (1 << 20) >= (1 << 21)) { 208 printk(KERN_ERR "%s: value %li out of IMM21b range\n", mod->name, (int64_t)val); 209 return 0; 210 } 211 ia64_patch((u64) insn, 0x11ffffe000UL, ( ((val & 0x100000UL) << 16) /* bit 20 -> 36 */ 212 | ((val & 0x0fffffUL) << 13) /* bit 0 -> 13 */)); 213 return 1; 214} 215 216#if USE_BRL 217 218struct plt_entry { 219 /* Three instruction bundles in PLT. */ 220 unsigned char bundle[2][16]; 221}; 222 223static const struct plt_entry ia64_plt_template = { 224 { 225 { 226 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 227 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /* movl gp=TARGET_GP */ 228 0x00, 0x00, 0x00, 0x60 229 }, 230 { 231 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 232 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* brl.many gp=TARGET_GP */ 233 0x08, 0x00, 0x00, 0xc0 234 } 235 } 236}; 237 238static int 239patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp) 240{ 241 if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_gp) 242 && apply_imm60(mod, (struct insn *) (plt->bundle[1] + 2), 243 (target_ip - (int64_t) plt->bundle[1]) / 16)) 244 return 1; 245 return 0; 246} 247 248unsigned long 249plt_target (struct plt_entry *plt) 250{ 251 uint64_t b0, b1, *b = (uint64_t *) plt->bundle[1]; 252 long off; 253 254 b0 = b[0]; b1 = b[1]; 255 off = ( ((b1 & 0x00fffff000000000UL) >> 36) /* imm20b -> bit 0 */ 256 | ((b0 >> 48) << 20) | ((b1 & 0x7fffffUL) << 36) /* imm39 -> bit 20 */ 257 | ((b1 & 0x0800000000000000UL) << 0)); /* i -> bit 59 */ 258 return (long) plt->bundle[1] + 16*off; 259} 260 261#else /* !USE_BRL */ 262 263struct plt_entry { 264 /* Three instruction bundles in PLT. */ 265 unsigned char bundle[3][16]; 266}; 267 268static const struct plt_entry ia64_plt_template = { 269 { 270 { 271 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 272 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* movl r16=TARGET_IP */ 273 0x02, 0x00, 0x00, 0x60 274 }, 275 { 276 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */ 277 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /* movl gp=TARGET_GP */ 278 0x00, 0x00, 0x00, 0x60 279 }, 280 { 281 0x11, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MIB] nop.m 0 */ 282 0x60, 0x80, 0x04, 0x80, 0x03, 0x00, /* mov b6=r16 */ 283 0x60, 0x00, 0x80, 0x00 /* br.few b6 */ 284 } 285 } 286}; 287 288static int 289patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp) 290{ 291 if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_ip) 292 && apply_imm64(mod, (struct insn *) (plt->bundle[1] + 2), target_gp)) 293 return 1; 294 return 0; 295} 296 297unsigned long 298plt_target (struct plt_entry *plt) 299{ 300 uint64_t b0, b1, *b = (uint64_t *) plt->bundle[0]; 301 302 b0 = b[0]; b1 = b[1]; 303 return ( ((b1 & 0x000007f000000000) >> 36) /* imm7b -> bit 0 */ 304 | ((b1 & 0x07fc000000000000) >> 43) /* imm9d -> bit 7 */ 305 | ((b1 & 0x0003e00000000000) >> 29) /* imm5c -> bit 16 */ 306 | ((b1 & 0x0000100000000000) >> 23) /* ic -> bit 21 */ 307 | ((b0 >> 46) << 22) | ((b1 & 0x7fffff) << 40) /* imm41 -> bit 22 */ 308 | ((b1 & 0x0800000000000000) << 4)); /* i -> bit 63 */ 309} 310 311#endif /* !USE_BRL */ 312 313void * 314module_alloc (unsigned long size) 315{ 316 if (!size) 317 return NULL; 318 return vmalloc(size); 319} 320 321void 322module_free (struct module *mod, void *module_region) 323{ 324 if (mod->arch.init_unw_table && module_region == mod->module_init) { 325 unw_remove_unwind_table(mod->arch.init_unw_table); 326 mod->arch.init_unw_table = NULL; 327 } 328 vfree(module_region); 329} 330 331/* Have we already seen one of these relocations? */ 332static int 333duplicate_reloc (const Elf64_Rela *rela, unsigned int num) 334{ 335 unsigned int i; 336 337 for (i = 0; i < num; i++) { 338 if (rela[i].r_info == rela[num].r_info && rela[i].r_addend == rela[num].r_addend) 339 return 1; 340 } 341 return 0; 342} 343 344/* Count how many GOT entries we may need */ 345static unsigned int 346count_gots (const Elf64_Rela *rela, unsigned int num) 347{ 348 unsigned int i, ret = 0; 349 350 /* Sure, this is order(n^2), but it's usually short, and not 351 time critical */ 352 for (i = 0; i < num; i++) { 353 switch (ELF64_R_TYPE(rela[i].r_info)) { 354 case R_IA64_LTOFF22: 355 case R_IA64_LTOFF22X: 356 case R_IA64_LTOFF64I: 357 case R_IA64_LTOFF_FPTR22: 358 case R_IA64_LTOFF_FPTR64I: 359 case R_IA64_LTOFF_FPTR32MSB: 360 case R_IA64_LTOFF_FPTR32LSB: 361 case R_IA64_LTOFF_FPTR64MSB: 362 case R_IA64_LTOFF_FPTR64LSB: 363 if (!duplicate_reloc(rela, i)) 364 ret++; 365 break; 366 } 367 } 368 return ret; 369} 370 371/* Count how many PLT entries we may need */ 372static unsigned int 373count_plts (const Elf64_Rela *rela, unsigned int num) 374{ 375 unsigned int i, ret = 0; 376 377 /* Sure, this is order(n^2), but it's usually short, and not 378 time critical */ 379 for (i = 0; i < num; i++) { 380 switch (ELF64_R_TYPE(rela[i].r_info)) { 381 case R_IA64_PCREL21B: 382 case R_IA64_PLTOFF22: 383 case R_IA64_PLTOFF64I: 384 case R_IA64_PLTOFF64MSB: 385 case R_IA64_PLTOFF64LSB: 386 case R_IA64_IPLTMSB: 387 case R_IA64_IPLTLSB: 388 if (!duplicate_reloc(rela, i)) 389 ret++; 390 break; 391 } 392 } 393 return ret; 394} 395 396/* We need to create an function-descriptors for any internal function 397 which is referenced. */ 398static unsigned int 399count_fdescs (const Elf64_Rela *rela, unsigned int num) 400{ 401 unsigned int i, ret = 0; 402 403 /* Sure, this is order(n^2), but it's usually short, and not time critical. */ 404 for (i = 0; i < num; i++) { 405 switch (ELF64_R_TYPE(rela[i].r_info)) { 406 case R_IA64_FPTR64I: 407 case R_IA64_FPTR32LSB: 408 case R_IA64_FPTR32MSB: 409 case R_IA64_FPTR64LSB: 410 case R_IA64_FPTR64MSB: 411 case R_IA64_LTOFF_FPTR22: 412 case R_IA64_LTOFF_FPTR32LSB: 413 case R_IA64_LTOFF_FPTR32MSB: 414 case R_IA64_LTOFF_FPTR64I: 415 case R_IA64_LTOFF_FPTR64LSB: 416 case R_IA64_LTOFF_FPTR64MSB: 417 case R_IA64_IPLTMSB: 418 case R_IA64_IPLTLSB: 419 /* 420 * Jumps to static functions sometimes go straight to their 421 * offset. Of course, that may not be possible if the jump is 422 * from init -> core or vice. versa, so we need to generate an 423 * FDESC (and PLT etc) for that. 424 */ 425 case R_IA64_PCREL21B: 426 if (!duplicate_reloc(rela, i)) 427 ret++; 428 break; 429 } 430 } 431 return ret; 432} 433 434int 435module_frob_arch_sections (Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, char *secstrings, 436 struct module *mod) 437{ 438 unsigned long core_plts = 0, init_plts = 0, gots = 0, fdescs = 0; 439 Elf64_Shdr *s, *sechdrs_end = sechdrs + ehdr->e_shnum; 440 441 /* 442 * To store the PLTs and function-descriptors, we expand the .text section for 443 * core module-code and the .init.text section for initialization code. 444 */ 445 for (s = sechdrs; s < sechdrs_end; ++s) 446 if (strcmp(".core.plt", secstrings + s->sh_name) == 0) 447 mod->arch.core_plt = s; 448 else if (strcmp(".init.plt", secstrings + s->sh_name) == 0) 449 mod->arch.init_plt = s; 450 else if (strcmp(".got", secstrings + s->sh_name) == 0) 451 mod->arch.got = s; 452 else if (strcmp(".opd", secstrings + s->sh_name) == 0) 453 mod->arch.opd = s; 454 else if (strcmp(".IA_64.unwind", secstrings + s->sh_name) == 0) 455 mod->arch.unwind = s; 456 457 if (!mod->arch.core_plt || !mod->arch.init_plt || !mod->arch.got || !mod->arch.opd) { 458 printk(KERN_ERR "%s: sections missing\n", mod->name); 459 return -ENOEXEC; 460 } 461 462 /* GOT and PLTs can occur in any relocated section... */ 463 for (s = sechdrs + 1; s < sechdrs_end; ++s) { 464 const Elf64_Rela *rels = (void *)ehdr + s->sh_offset; 465 unsigned long numrels = s->sh_size/sizeof(Elf64_Rela); 466 467 if (s->sh_type != SHT_RELA) 468 continue; 469 470 gots += count_gots(rels, numrels); 471 fdescs += count_fdescs(rels, numrels); 472 if (strstr(secstrings + s->sh_name, ".init")) 473 init_plts += count_plts(rels, numrels); 474 else 475 core_plts += count_plts(rels, numrels); 476 } 477 478 mod->arch.core_plt->sh_type = SHT_NOBITS; 479 mod->arch.core_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC; 480 mod->arch.core_plt->sh_addralign = 16; 481 mod->arch.core_plt->sh_size = core_plts * sizeof(struct plt_entry); 482 mod->arch.init_plt->sh_type = SHT_NOBITS; 483 mod->arch.init_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC; 484 mod->arch.init_plt->sh_addralign = 16; 485 mod->arch.init_plt->sh_size = init_plts * sizeof(struct plt_entry); 486 mod->arch.got->sh_type = SHT_NOBITS; 487 mod->arch.got->sh_flags = ARCH_SHF_SMALL | SHF_ALLOC; 488 mod->arch.got->sh_addralign = 8; 489 mod->arch.got->sh_size = gots * sizeof(struct got_entry); 490 mod->arch.opd->sh_type = SHT_NOBITS; 491 mod->arch.opd->sh_flags = SHF_ALLOC; 492 mod->arch.opd->sh_addralign = 8; 493 mod->arch.opd->sh_size = fdescs * sizeof(struct fdesc); 494 DEBUGP("%s: core.plt=%lx, init.plt=%lx, got=%lx, fdesc=%lx\n", 495 __FUNCTION__, mod->arch.core_plt->sh_size, mod->arch.init_plt->sh_size, 496 mod->arch.got->sh_size, mod->arch.opd->sh_size); 497 return 0; 498} 499 500static inline int 501in_init (const struct module *mod, uint64_t addr) 502{ 503 return addr - (uint64_t) mod->module_init < mod->init_size; 504} 505 506static inline int 507in_core (const struct module *mod, uint64_t addr) 508{ 509 return addr - (uint64_t) mod->module_core < mod->core_size; 510} 511 512static inline int 513is_internal (const struct module *mod, uint64_t value) 514{ 515 return in_init(mod, value) || in_core(mod, value); 516} 517 518/* 519 * Get gp-relative offset for the linkage-table entry of VALUE. 520 */ 521static uint64_t 522get_ltoff (struct module *mod, uint64_t value, int *okp) 523{ 524 struct got_entry *got, *e; 525 526 if (!*okp) 527 return 0; 528 529 got = (void *) mod->arch.got->sh_addr; 530 for (e = got; e < got + mod->arch.next_got_entry; ++e) 531 if (e->val == value) 532 goto found; 533 534 /* Not enough GOT entries? */ 535 if (e >= (struct got_entry *) (mod->arch.got->sh_addr + mod->arch.got->sh_size)) 536 BUG(); 537 538 e->val = value; 539 ++mod->arch.next_got_entry; 540 found: 541 return (uint64_t) e - mod->arch.gp; 542} 543 544static inline int 545gp_addressable (struct module *mod, uint64_t value) 546{ 547 return value - mod->arch.gp + MAX_LTOFF/2 < MAX_LTOFF; 548} 549 550/* Get PC-relative PLT entry for this value. Returns 0 on failure. */ 551static uint64_t 552get_plt (struct module *mod, const struct insn *insn, uint64_t value, int *okp) 553{ 554 struct plt_entry *plt, *plt_end; 555 uint64_t target_ip, target_gp; 556 557 if (!*okp) 558 return 0; 559 560 if (in_init(mod, (uint64_t) insn)) { 561 plt = (void *) mod->arch.init_plt->sh_addr; 562 plt_end = (void *) plt + mod->arch.init_plt->sh_size; 563 } else { 564 plt = (void *) mod->arch.core_plt->sh_addr; 565 plt_end = (void *) plt + mod->arch.core_plt->sh_size; 566 } 567 568 /* "value" is a pointer to a function-descriptor; fetch the target ip/gp from it: */ 569 target_ip = ((uint64_t *) value)[0]; 570 target_gp = ((uint64_t *) value)[1]; 571 572 /* Look for existing PLT entry. */ 573 while (plt->bundle[0][0]) { 574 if (plt_target(plt) == target_ip) 575 goto found; 576 if (++plt >= plt_end) 577 BUG(); 578 } 579 *plt = ia64_plt_template; 580 if (!patch_plt(mod, plt, target_ip, target_gp)) { 581 *okp = 0; 582 return 0; 583 } 584#if ARCH_MODULE_DEBUG 585 if (plt_target(plt) != target_ip) { 586 printk("%s: mistargeted PLT: wanted %lx, got %lx\n", 587 __FUNCTION__, target_ip, plt_target(plt)); 588 *okp = 0; 589 return 0; 590 } 591#endif 592 found: 593 return (uint64_t) plt; 594} 595 596/* Get function descriptor for VALUE. */ 597static uint64_t 598get_fdesc (struct module *mod, uint64_t value, int *okp) 599{ 600 struct fdesc *fdesc = (void *) mod->arch.opd->sh_addr; 601 602 if (!*okp) 603 return 0; 604 605 if (!value) { 606 printk(KERN_ERR "%s: fdesc for zero requested!\n", mod->name); 607 return 0; 608 } 609 610 if (!is_internal(mod, value)) 611 /* 612 * If it's not a module-local entry-point, "value" already points to a 613 * function-descriptor. 614 */ 615 return value; 616 617 /* Look for existing function descriptor. */ 618 while (fdesc->ip) { 619 if (fdesc->ip == value) 620 return (uint64_t)fdesc; 621 if ((uint64_t) ++fdesc >= mod->arch.opd->sh_addr + mod->arch.opd->sh_size) 622 BUG(); 623 } 624 625 /* Create new one */ 626 fdesc->ip = value; 627 fdesc->gp = mod->arch.gp; 628 return (uint64_t) fdesc; 629} 630 631static inline int 632do_reloc (struct module *mod, uint8_t r_type, Elf64_Sym *sym, uint64_t addend, 633 Elf64_Shdr *sec, void *location) 634{ 635 enum reloc_target_format format = (r_type >> FORMAT_SHIFT) & FORMAT_MASK; 636 enum reloc_value_formula formula = (r_type >> VALUE_SHIFT) & VALUE_MASK; 637 uint64_t val; 638 int ok = 1; 639 640 val = sym->st_value + addend; 641 642 switch (formula) { 643 case RV_SEGREL: /* segment base is arbitrarily chosen to be 0 for kernel modules */ 644 case RV_DIRECT: 645 break; 646 647 case RV_GPREL: val -= mod->arch.gp; break; 648 case RV_LTREL: val = get_ltoff(mod, val, &ok); break; 649 case RV_PLTREL: val = get_plt(mod, location, val, &ok); break; 650 case RV_FPTR: val = get_fdesc(mod, val, &ok); break; 651 case RV_SECREL: val -= sec->sh_addr; break; 652 case RV_LTREL_FPTR: val = get_ltoff(mod, get_fdesc(mod, val, &ok), &ok); break; 653 654 case RV_PCREL: 655 switch (r_type) { 656 case R_IA64_PCREL21B: 657 if ((in_init(mod, val) && in_core(mod, (uint64_t)location)) || 658 (in_core(mod, val) && in_init(mod, (uint64_t)location))) { 659 /* 660 * Init section may have been allocated far away from core, 661 * if the branch won't reach, then allocate a plt for it. 662 */ 663 uint64_t delta = ((int64_t)val - (int64_t)location) / 16; 664 if (delta + (1 << 20) >= (1 << 21)) { 665 val = get_fdesc(mod, val, &ok); 666 val = get_plt(mod, location, val, &ok); 667 } 668 } else if (!is_internal(mod, val)) 669 val = get_plt(mod, location, val, &ok); 670 /* FALL THROUGH */ 671 default: 672 val -= bundle(location); 673 break; 674 675 case R_IA64_PCREL32MSB: 676 case R_IA64_PCREL32LSB: 677 case R_IA64_PCREL64MSB: 678 case R_IA64_PCREL64LSB: 679 val -= (uint64_t) location; 680 break; 681 682 } 683 switch (r_type) { 684 case R_IA64_PCREL60B: format = RF_INSN60; break; 685 case R_IA64_PCREL21B: format = RF_INSN21B; break; 686 case R_IA64_PCREL21M: format = RF_INSN21M; break; 687 case R_IA64_PCREL21F: format = RF_INSN21F; break; 688 default: break; 689 } 690 break; 691 692 case RV_BDREL: 693 val -= (uint64_t) (in_init(mod, val) ? mod->module_init : mod->module_core); 694 break; 695 696 case RV_LTV: 697 /* can link-time value relocs happen here? */ 698 BUG(); 699 break; 700 701 case RV_PCREL2: 702 if (r_type == R_IA64_PCREL21BI) { 703 if (!is_internal(mod, val)) { 704 printk(KERN_ERR "%s: %s reloc against non-local symbol (%lx)\n", 705 __FUNCTION__, reloc_name[r_type], val); 706 return -ENOEXEC; 707 } 708 format = RF_INSN21B; 709 } 710 val -= bundle(location); 711 break; 712 713 case RV_SPECIAL: 714 switch (r_type) { 715 case R_IA64_IPLTMSB: 716 case R_IA64_IPLTLSB: 717 val = get_fdesc(mod, get_plt(mod, location, val, &ok), &ok); 718 format = RF_64LSB; 719 if (r_type == R_IA64_IPLTMSB) 720 format = RF_64MSB; 721 break; 722 723 case R_IA64_SUB: 724 val = addend - sym->st_value; 725 format = RF_INSN64; 726 break; 727 728 case R_IA64_LTOFF22X: 729 if (gp_addressable(mod, val)) 730 val -= mod->arch.gp; 731 else 732 val = get_ltoff(mod, val, &ok); 733 format = RF_INSN22; 734 break; 735 736 case R_IA64_LDXMOV: 737 if (gp_addressable(mod, val)) { 738 /* turn "ld8" into "mov": */ 739 DEBUGP("%s: patching ld8 at %p to mov\n", __FUNCTION__, location); 740 ia64_patch((u64) location, 0x1fff80fe000UL, 0x10000000000UL); 741 } 742 return 0; 743 744 default: 745 if (reloc_name[r_type]) 746 printk(KERN_ERR "%s: special reloc %s not supported", 747 mod->name, reloc_name[r_type]); 748 else 749 printk(KERN_ERR "%s: unknown special reloc %x\n", 750 mod->name, r_type); 751 return -ENOEXEC; 752 } 753 break; 754 755 case RV_TPREL: 756 case RV_LTREL_TPREL: 757 case RV_DTPMOD: 758 case RV_LTREL_DTPMOD: 759 case RV_DTPREL: 760 case RV_LTREL_DTPREL: 761 printk(KERN_ERR "%s: %s reloc not supported\n", 762 mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?"); 763 return -ENOEXEC; 764 765 default: 766 printk(KERN_ERR "%s: unknown reloc %x\n", mod->name, r_type); 767 return -ENOEXEC; 768 } 769 770 if (!ok) 771 return -ENOEXEC; 772 773 DEBUGP("%s: [%p]<-%016lx = %s(%lx)\n", __FUNCTION__, location, val, 774 reloc_name[r_type] ? reloc_name[r_type] : "?", sym->st_value + addend); 775 776 switch (format) { 777 case RF_INSN21B: ok = apply_imm21b(mod, location, (int64_t) val / 16); break; 778 case RF_INSN22: ok = apply_imm22(mod, location, val); break; 779 case RF_INSN64: ok = apply_imm64(mod, location, val); break; 780 case RF_INSN60: ok = apply_imm60(mod, location, (int64_t) val / 16); break; 781 case RF_32LSB: put_unaligned(val, (uint32_t *) location); break; 782 case RF_64LSB: put_unaligned(val, (uint64_t *) location); break; 783 case RF_32MSB: /* ia64 Linux is little-endian... */ 784 case RF_64MSB: /* ia64 Linux is little-endian... */ 785 case RF_INSN14: /* must be within-module, i.e., resolved by "ld -r" */ 786 case RF_INSN21M: /* must be within-module, i.e., resolved by "ld -r" */ 787 case RF_INSN21F: /* must be within-module, i.e., resolved by "ld -r" */ 788 printk(KERN_ERR "%s: format %u needed by %s reloc is not supported\n", 789 mod->name, format, reloc_name[r_type] ? reloc_name[r_type] : "?"); 790 return -ENOEXEC; 791 792 default: 793 printk(KERN_ERR "%s: relocation %s resulted in unknown format %u\n", 794 mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?", format); 795 return -ENOEXEC; 796 } 797 return ok ? 0 : -ENOEXEC; 798} 799 800int 801apply_relocate_add (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, 802 unsigned int relsec, struct module *mod) 803{ 804 unsigned int i, n = sechdrs[relsec].sh_size / sizeof(Elf64_Rela); 805 Elf64_Rela *rela = (void *) sechdrs[relsec].sh_addr; 806 Elf64_Shdr *target_sec; 807 int ret; 808 809 DEBUGP("%s: applying section %u (%u relocs) to %u\n", __FUNCTION__, 810 relsec, n, sechdrs[relsec].sh_info); 811 812 target_sec = sechdrs + sechdrs[relsec].sh_info; 813 814 if (target_sec->sh_entsize == ~0UL) 815 /* 816 * If target section wasn't allocated, we don't need to relocate it. 817 * Happens, e.g., for debug sections. 818 */ 819 return 0; 820 821 if (!mod->arch.gp) { 822 uint64_t gp; 823 if (mod->core_size > MAX_LTOFF) 824 /* 825 * This takes advantage of fact that SHF_ARCH_SMALL gets allocated 826 * at the end of the module. 827 */ 828 gp = mod->core_size - MAX_LTOFF / 2; 829 else 830 gp = mod->core_size / 2; 831 gp = (uint64_t) mod->module_core + ((gp + 7) & -8); 832 mod->arch.gp = gp; 833 DEBUGP("%s: placing gp at 0x%lx\n", __FUNCTION__, gp); 834 } 835 836 for (i = 0; i < n; i++) { 837 ret = do_reloc(mod, ELF64_R_TYPE(rela[i].r_info), 838 ((Elf64_Sym *) sechdrs[symindex].sh_addr 839 + ELF64_R_SYM(rela[i].r_info)), 840 rela[i].r_addend, target_sec, 841 (void *) target_sec->sh_addr + rela[i].r_offset); 842 if (ret < 0) 843 return ret; 844 } 845 return 0; 846} 847 848int 849apply_relocate (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, 850 unsigned int relsec, struct module *mod) 851{ 852 printk(KERN_ERR "module %s: REL relocs in section %u unsupported\n", mod->name, relsec); 853 return -ENOEXEC; 854} 855 856/* 857 * Modules contain a single unwind table which covers both the core and the init text 858 * sections but since the two are not contiguous, we need to split this table up such that 859 * we can register (and unregister) each "segment" separately. Fortunately, this sounds 860 * more complicated than it really is. 861 */ 862static void 863register_unwind_table (struct module *mod) 864{ 865 struct unw_table_entry *start = (void *) mod->arch.unwind->sh_addr; 866 struct unw_table_entry *end = start + mod->arch.unwind->sh_size / sizeof (*start); 867 struct unw_table_entry tmp, *e1, *e2, *core, *init; 868 unsigned long num_init = 0, num_core = 0; 869 870 /* First, count how many init and core unwind-table entries there are. */ 871 for (e1 = start; e1 < end; ++e1) 872 if (in_init(mod, e1->start_offset)) 873 ++num_init; 874 else 875 ++num_core; 876 /* 877 * Second, sort the table such that all unwind-table entries for the init and core 878 * text sections are nicely separated. We do this with a stupid bubble sort 879 * (unwind tables don't get ridiculously huge). 880 */ 881 for (e1 = start; e1 < end; ++e1) { 882 for (e2 = e1 + 1; e2 < end; ++e2) { 883 if (e2->start_offset < e1->start_offset) { 884 tmp = *e1; 885 *e1 = *e2; 886 *e2 = tmp; 887 } 888 } 889 } 890 /* 891 * Third, locate the init and core segments in the unwind table: 892 */ 893 if (in_init(mod, start->start_offset)) { 894 init = start; 895 core = start + num_init; 896 } else { 897 core = start; 898 init = start + num_core; 899 } 900 901 DEBUGP("%s: name=%s, gp=%lx, num_init=%lu, num_core=%lu\n", __FUNCTION__, 902 mod->name, mod->arch.gp, num_init, num_core); 903 904 /* 905 * Fourth, register both tables (if not empty). 906 */ 907 if (num_core > 0) { 908 mod->arch.core_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp, 909 core, core + num_core); 910 DEBUGP("%s: core: handle=%p [%p-%p)\n", __FUNCTION__, 911 mod->arch.core_unw_table, core, core + num_core); 912 } 913 if (num_init > 0) { 914 mod->arch.init_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp, 915 init, init + num_init); 916 DEBUGP("%s: init: handle=%p [%p-%p)\n", __FUNCTION__, 917 mod->arch.init_unw_table, init, init + num_init); 918 } 919} 920 921int 922module_finalize (const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod) 923{ 924 DEBUGP("%s: init: entry=%p\n", __FUNCTION__, mod->init); 925 if (mod->arch.unwind) 926 register_unwind_table(mod); 927 return 0; 928} 929 930void 931module_arch_cleanup (struct module *mod) 932{ 933 if (mod->arch.init_unw_table) 934 unw_remove_unwind_table(mod->arch.init_unw_table); 935 if (mod->arch.core_unw_table) 936 unw_remove_unwind_table(mod->arch.core_unw_table); 937} 938 939#ifdef CONFIG_SMP 940void 941percpu_modcopy (void *pcpudst, const void *src, unsigned long size) 942{ 943 unsigned int i; 944 for_each_possible_cpu(i) { 945 memcpy(pcpudst + __per_cpu_offset[i], src, size); 946 } 947} 948#endif /* CONFIG_SMP */ 949