1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * crash.c - kernel crash support code. 4 * Copyright (C) 2002-2004 Eric Biederman <ebiederm@xmission.com> 5 */ 6 7#include <linux/buildid.h> 8#include <linux/init.h> 9#include <linux/utsname.h> 10#include <linux/vmalloc.h> 11#include <linux/sizes.h> 12#include <linux/kexec.h> 13#include <linux/memory.h> 14#include <linux/mm.h> 15#include <linux/cpuhotplug.h> 16#include <linux/memblock.h> 17#include <linux/kmemleak.h> 18#include <linux/crash_core.h> 19#include <linux/reboot.h> 20#include <linux/btf.h> 21#include <linux/objtool.h> 22 23#include <asm/page.h> 24#include <asm/sections.h> 25 26#include <crypto/sha1.h> 27 28#include "kallsyms_internal.h" 29#include "kexec_internal.h" 30 31/* Per cpu memory for storing cpu states in case of system crash. */ 32note_buf_t __percpu *crash_notes; 33 34#ifdef CONFIG_CRASH_DUMP 35 36int kimage_crash_copy_vmcoreinfo(struct kimage *image) 37{ 38 struct page *vmcoreinfo_page; 39 void *safecopy; 40 41 if (!IS_ENABLED(CONFIG_CRASH_DUMP)) 42 return 0; 43 if (image->type != KEXEC_TYPE_CRASH) 44 return 0; 45 46 /* 47 * For kdump, allocate one vmcoreinfo safe copy from the 48 * crash memory. as we have arch_kexec_protect_crashkres() 49 * after kexec syscall, we naturally protect it from write 50 * (even read) access under kernel direct mapping. But on 51 * the other hand, we still need to operate it when crash 52 * happens to generate vmcoreinfo note, hereby we rely on 53 * vmap for this purpose. 54 */ 55 vmcoreinfo_page = kimage_alloc_control_pages(image, 0); 56 if (!vmcoreinfo_page) { 57 pr_warn("Could not allocate vmcoreinfo buffer\n"); 58 return -ENOMEM; 59 } 60 safecopy = vmap(&vmcoreinfo_page, 1, VM_MAP, PAGE_KERNEL); 61 if (!safecopy) { 62 pr_warn("Could not vmap vmcoreinfo buffer\n"); 63 return -ENOMEM; 64 } 65 66 image->vmcoreinfo_data_copy = safecopy; 67 crash_update_vmcoreinfo_safecopy(safecopy); 68 69 return 0; 70} 71 72 73 74int kexec_should_crash(struct task_struct *p) 75{ 76 /* 77 * If crash_kexec_post_notifiers is enabled, don't run 78 * crash_kexec() here yet, which must be run after panic 79 * notifiers in panic(). 80 */ 81 if (crash_kexec_post_notifiers) 82 return 0; 83 /* 84 * There are 4 panic() calls in make_task_dead() path, each of which 85 * corresponds to each of these 4 conditions. 86 */ 87 if (in_interrupt() || !p->pid || is_global_init(p) || panic_on_oops) 88 return 1; 89 return 0; 90} 91 92int kexec_crash_loaded(void) 93{ 94 return !!kexec_crash_image; 95} 96EXPORT_SYMBOL_GPL(kexec_crash_loaded); 97 98/* 99 * No panic_cpu check version of crash_kexec(). This function is called 100 * only when panic_cpu holds the current CPU number; this is the only CPU 101 * which processes crash_kexec routines. 102 */ 103void __noclone __crash_kexec(struct pt_regs *regs) 104{ 105 /* Take the kexec_lock here to prevent sys_kexec_load 106 * running on one cpu from replacing the crash kernel 107 * we are using after a panic on a different cpu. 108 * 109 * If the crash kernel was not located in a fixed area 110 * of memory the xchg(&kexec_crash_image) would be 111 * sufficient. But since I reuse the memory... 112 */ 113 if (kexec_trylock()) { 114 if (kexec_crash_image) { 115 struct pt_regs fixed_regs; 116 117 crash_setup_regs(&fixed_regs, regs); 118 crash_save_vmcoreinfo(); 119 machine_crash_shutdown(&fixed_regs); 120 machine_kexec(kexec_crash_image); 121 } 122 kexec_unlock(); 123 } 124} 125STACK_FRAME_NON_STANDARD(__crash_kexec); 126 127__bpf_kfunc void crash_kexec(struct pt_regs *regs) 128{ 129 int old_cpu, this_cpu; 130 131 /* 132 * Only one CPU is allowed to execute the crash_kexec() code as with 133 * panic(). Otherwise parallel calls of panic() and crash_kexec() 134 * may stop each other. To exclude them, we use panic_cpu here too. 135 */ 136 old_cpu = PANIC_CPU_INVALID; 137 this_cpu = raw_smp_processor_id(); 138 139 if (atomic_try_cmpxchg(&panic_cpu, &old_cpu, this_cpu)) { 140 /* This is the 1st CPU which comes here, so go ahead. */ 141 __crash_kexec(regs); 142 143 /* 144 * Reset panic_cpu to allow another panic()/crash_kexec() 145 * call. 146 */ 147 atomic_set(&panic_cpu, PANIC_CPU_INVALID); 148 } 149} 150 151static inline resource_size_t crash_resource_size(const struct resource *res) 152{ 153 return !res->end ? 0 : resource_size(res); 154} 155 156 157 158 159int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, 160 void **addr, unsigned long *sz) 161{ 162 Elf64_Ehdr *ehdr; 163 Elf64_Phdr *phdr; 164 unsigned long nr_cpus = num_possible_cpus(), nr_phdr, elf_sz; 165 unsigned char *buf; 166 unsigned int cpu, i; 167 unsigned long long notes_addr; 168 unsigned long mstart, mend; 169 170 /* extra phdr for vmcoreinfo ELF note */ 171 nr_phdr = nr_cpus + 1; 172 nr_phdr += mem->nr_ranges; 173 174 /* 175 * kexec-tools creates an extra PT_LOAD phdr for kernel text mapping 176 * area (for example, ffffffff80000000 - ffffffffa0000000 on x86_64). 177 * I think this is required by tools like gdb. So same physical 178 * memory will be mapped in two ELF headers. One will contain kernel 179 * text virtual addresses and other will have __va(physical) addresses. 180 */ 181 182 nr_phdr++; 183 elf_sz = sizeof(Elf64_Ehdr) + nr_phdr * sizeof(Elf64_Phdr); 184 elf_sz = ALIGN(elf_sz, ELF_CORE_HEADER_ALIGN); 185 186 buf = vzalloc(elf_sz); 187 if (!buf) 188 return -ENOMEM; 189 190 ehdr = (Elf64_Ehdr *)buf; 191 phdr = (Elf64_Phdr *)(ehdr + 1); 192 memcpy(ehdr->e_ident, ELFMAG, SELFMAG); 193 ehdr->e_ident[EI_CLASS] = ELFCLASS64; 194 ehdr->e_ident[EI_DATA] = ELFDATA2LSB; 195 ehdr->e_ident[EI_VERSION] = EV_CURRENT; 196 ehdr->e_ident[EI_OSABI] = ELF_OSABI; 197 memset(ehdr->e_ident + EI_PAD, 0, EI_NIDENT - EI_PAD); 198 ehdr->e_type = ET_CORE; 199 ehdr->e_machine = ELF_ARCH; 200 ehdr->e_version = EV_CURRENT; 201 ehdr->e_phoff = sizeof(Elf64_Ehdr); 202 ehdr->e_ehsize = sizeof(Elf64_Ehdr); 203 ehdr->e_phentsize = sizeof(Elf64_Phdr); 204 205 /* Prepare one phdr of type PT_NOTE for each possible CPU */ 206 for_each_possible_cpu(cpu) { 207 phdr->p_type = PT_NOTE; 208 notes_addr = per_cpu_ptr_to_phys(per_cpu_ptr(crash_notes, cpu)); 209 phdr->p_offset = phdr->p_paddr = notes_addr; 210 phdr->p_filesz = phdr->p_memsz = sizeof(note_buf_t); 211 (ehdr->e_phnum)++; 212 phdr++; 213 } 214 215 /* Prepare one PT_NOTE header for vmcoreinfo */ 216 phdr->p_type = PT_NOTE; 217 phdr->p_offset = phdr->p_paddr = paddr_vmcoreinfo_note(); 218 phdr->p_filesz = phdr->p_memsz = VMCOREINFO_NOTE_SIZE; 219 (ehdr->e_phnum)++; 220 phdr++; 221 222 /* Prepare PT_LOAD type program header for kernel text region */ 223 if (need_kernel_map) { 224 phdr->p_type = PT_LOAD; 225 phdr->p_flags = PF_R|PF_W|PF_X; 226 phdr->p_vaddr = (unsigned long) _text; 227 phdr->p_filesz = phdr->p_memsz = _end - _text; 228 phdr->p_offset = phdr->p_paddr = __pa_symbol(_text); 229 ehdr->e_phnum++; 230 phdr++; 231 } 232 233 /* Go through all the ranges in mem->ranges[] and prepare phdr */ 234 for (i = 0; i < mem->nr_ranges; i++) { 235 mstart = mem->ranges[i].start; 236 mend = mem->ranges[i].end; 237 238 phdr->p_type = PT_LOAD; 239 phdr->p_flags = PF_R|PF_W|PF_X; 240 phdr->p_offset = mstart; 241 242 phdr->p_paddr = mstart; 243 phdr->p_vaddr = (unsigned long) __va(mstart); 244 phdr->p_filesz = phdr->p_memsz = mend - mstart + 1; 245 phdr->p_align = 0; 246 ehdr->e_phnum++; 247#ifdef CONFIG_KEXEC_FILE 248 kexec_dprintk("Crash PT_LOAD ELF header. phdr=%p vaddr=0x%llx, paddr=0x%llx, sz=0x%llx e_phnum=%d p_offset=0x%llx\n", 249 phdr, phdr->p_vaddr, phdr->p_paddr, phdr->p_filesz, 250 ehdr->e_phnum, phdr->p_offset); 251#endif 252 phdr++; 253 } 254 255 *addr = buf; 256 *sz = elf_sz; 257 return 0; 258} 259 260int crash_exclude_mem_range(struct crash_mem *mem, 261 unsigned long long mstart, unsigned long long mend) 262{ 263 int i; 264 unsigned long long start, end, p_start, p_end; 265 266 for (i = 0; i < mem->nr_ranges; i++) { 267 start = mem->ranges[i].start; 268 end = mem->ranges[i].end; 269 p_start = mstart; 270 p_end = mend; 271 272 if (p_start > end) 273 continue; 274 275 /* 276 * Because the memory ranges in mem->ranges are stored in 277 * ascending order, when we detect `p_end < start`, we can 278 * immediately exit the for loop, as the subsequent memory 279 * ranges will definitely be outside the range we are looking 280 * for. 281 */ 282 if (p_end < start) 283 break; 284 285 /* Truncate any area outside of range */ 286 if (p_start < start) 287 p_start = start; 288 if (p_end > end) 289 p_end = end; 290 291 /* Found completely overlapping range */ 292 if (p_start == start && p_end == end) { 293 memmove(&mem->ranges[i], &mem->ranges[i + 1], 294 (mem->nr_ranges - (i + 1)) * sizeof(mem->ranges[i])); 295 i--; 296 mem->nr_ranges--; 297 } else if (p_start > start && p_end < end) { 298 /* Split original range */ 299 if (mem->nr_ranges >= mem->max_nr_ranges) 300 return -ENOMEM; 301 302 memmove(&mem->ranges[i + 2], &mem->ranges[i + 1], 303 (mem->nr_ranges - (i + 1)) * sizeof(mem->ranges[i])); 304 305 mem->ranges[i].end = p_start - 1; 306 mem->ranges[i + 1].start = p_end + 1; 307 mem->ranges[i + 1].end = end; 308 309 i++; 310 mem->nr_ranges++; 311 } else if (p_start != start) 312 mem->ranges[i].end = p_start - 1; 313 else 314 mem->ranges[i].start = p_end + 1; 315 } 316 317 return 0; 318} 319 320ssize_t crash_get_memory_size(void) 321{ 322 ssize_t size = 0; 323 324 if (!kexec_trylock()) 325 return -EBUSY; 326 327 size += crash_resource_size(&crashk_res); 328 size += crash_resource_size(&crashk_low_res); 329 330 kexec_unlock(); 331 return size; 332} 333 334static int __crash_shrink_memory(struct resource *old_res, 335 unsigned long new_size) 336{ 337 struct resource *ram_res; 338 339 ram_res = kzalloc(sizeof(*ram_res), GFP_KERNEL); 340 if (!ram_res) 341 return -ENOMEM; 342 343 ram_res->start = old_res->start + new_size; 344 ram_res->end = old_res->end; 345 ram_res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM; 346 ram_res->name = "System RAM"; 347 348 if (!new_size) { 349 release_resource(old_res); 350 old_res->start = 0; 351 old_res->end = 0; 352 } else { 353 crashk_res.end = ram_res->start - 1; 354 } 355 356 crash_free_reserved_phys_range(ram_res->start, ram_res->end); 357 insert_resource(&iomem_resource, ram_res); 358 359 return 0; 360} 361 362int crash_shrink_memory(unsigned long new_size) 363{ 364 int ret = 0; 365 unsigned long old_size, low_size; 366 367 if (!kexec_trylock()) 368 return -EBUSY; 369 370 if (kexec_crash_image) { 371 ret = -ENOENT; 372 goto unlock; 373 } 374 375 low_size = crash_resource_size(&crashk_low_res); 376 old_size = crash_resource_size(&crashk_res) + low_size; 377 new_size = roundup(new_size, KEXEC_CRASH_MEM_ALIGN); 378 if (new_size >= old_size) { 379 ret = (new_size == old_size) ? 0 : -EINVAL; 380 goto unlock; 381 } 382 383 /* 384 * (low_size > new_size) implies that low_size is greater than zero. 385 * This also means that if low_size is zero, the else branch is taken. 386 * 387 * If low_size is greater than 0, (low_size > new_size) indicates that 388 * crashk_low_res also needs to be shrunken. Otherwise, only crashk_res 389 * needs to be shrunken. 390 */ 391 if (low_size > new_size) { 392 ret = __crash_shrink_memory(&crashk_res, 0); 393 if (ret) 394 goto unlock; 395 396 ret = __crash_shrink_memory(&crashk_low_res, new_size); 397 } else { 398 ret = __crash_shrink_memory(&crashk_res, new_size - low_size); 399 } 400 401 /* Swap crashk_res and crashk_low_res if needed */ 402 if (!crashk_res.end && crashk_low_res.end) { 403 crashk_res.start = crashk_low_res.start; 404 crashk_res.end = crashk_low_res.end; 405 release_resource(&crashk_low_res); 406 crashk_low_res.start = 0; 407 crashk_low_res.end = 0; 408 insert_resource(&iomem_resource, &crashk_res); 409 } 410 411unlock: 412 kexec_unlock(); 413 return ret; 414} 415 416void crash_save_cpu(struct pt_regs *regs, int cpu) 417{ 418 struct elf_prstatus prstatus; 419 u32 *buf; 420 421 if ((cpu < 0) || (cpu >= nr_cpu_ids)) 422 return; 423 424 /* Using ELF notes here is opportunistic. 425 * I need a well defined structure format 426 * for the data I pass, and I need tags 427 * on the data to indicate what information I have 428 * squirrelled away. ELF notes happen to provide 429 * all of that, so there is no need to invent something new. 430 */ 431 buf = (u32 *)per_cpu_ptr(crash_notes, cpu); 432 if (!buf) 433 return; 434 memset(&prstatus, 0, sizeof(prstatus)); 435 prstatus.common.pr_pid = current->pid; 436 elf_core_copy_regs(&prstatus.pr_reg, regs); 437 buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS, 438 &prstatus, sizeof(prstatus)); 439 final_note(buf); 440} 441 442 443 444static int __init crash_notes_memory_init(void) 445{ 446 /* Allocate memory for saving cpu registers. */ 447 size_t size, align; 448 449 /* 450 * crash_notes could be allocated across 2 vmalloc pages when percpu 451 * is vmalloc based . vmalloc doesn't guarantee 2 continuous vmalloc 452 * pages are also on 2 continuous physical pages. In this case the 453 * 2nd part of crash_notes in 2nd page could be lost since only the 454 * starting address and size of crash_notes are exported through sysfs. 455 * Here round up the size of crash_notes to the nearest power of two 456 * and pass it to __alloc_percpu as align value. This can make sure 457 * crash_notes is allocated inside one physical page. 458 */ 459 size = sizeof(note_buf_t); 460 align = min(roundup_pow_of_two(sizeof(note_buf_t)), PAGE_SIZE); 461 462 /* 463 * Break compile if size is bigger than PAGE_SIZE since crash_notes 464 * definitely will be in 2 pages with that. 465 */ 466 BUILD_BUG_ON(size > PAGE_SIZE); 467 468 crash_notes = __alloc_percpu(size, align); 469 if (!crash_notes) { 470 pr_warn("Memory allocation for saving cpu register states failed\n"); 471 return -ENOMEM; 472 } 473 return 0; 474} 475subsys_initcall(crash_notes_memory_init); 476 477#endif /*CONFIG_CRASH_DUMP*/ 478 479#ifdef CONFIG_CRASH_HOTPLUG 480#undef pr_fmt 481#define pr_fmt(fmt) "crash hp: " fmt 482 483/* 484 * Different than kexec/kdump loading/unloading/jumping/shrinking which 485 * usually rarely happen, there will be many crash hotplug events notified 486 * during one short period, e.g one memory board is hot added and memory 487 * regions are online. So mutex lock __crash_hotplug_lock is used to 488 * serialize the crash hotplug handling specifically. 489 */ 490static DEFINE_MUTEX(__crash_hotplug_lock); 491#define crash_hotplug_lock() mutex_lock(&__crash_hotplug_lock) 492#define crash_hotplug_unlock() mutex_unlock(&__crash_hotplug_lock) 493 494/* 495 * This routine utilized when the crash_hotplug sysfs node is read. 496 * It reflects the kernel's ability/permission to update the crash 497 * elfcorehdr directly. 498 */ 499int crash_check_update_elfcorehdr(void) 500{ 501 int rc = 0; 502 503 crash_hotplug_lock(); 504 /* Obtain lock while reading crash information */ 505 if (!kexec_trylock()) { 506 pr_info("kexec_trylock() failed, elfcorehdr may be inaccurate\n"); 507 crash_hotplug_unlock(); 508 return 0; 509 } 510 if (kexec_crash_image) { 511 if (kexec_crash_image->file_mode) 512 rc = 1; 513 else 514 rc = kexec_crash_image->update_elfcorehdr; 515 } 516 /* Release lock now that update complete */ 517 kexec_unlock(); 518 crash_hotplug_unlock(); 519 520 return rc; 521} 522 523/* 524 * To accurately reflect hot un/plug changes of cpu and memory resources 525 * (including onling and offlining of those resources), the elfcorehdr 526 * (which is passed to the crash kernel via the elfcorehdr= parameter) 527 * must be updated with the new list of CPUs and memories. 528 * 529 * In order to make changes to elfcorehdr, two conditions are needed: 530 * First, the segment containing the elfcorehdr must be large enough 531 * to permit a growing number of resources; the elfcorehdr memory size 532 * is based on NR_CPUS_DEFAULT and CRASH_MAX_MEMORY_RANGES. 533 * Second, purgatory must explicitly exclude the elfcorehdr from the 534 * list of segments it checks (since the elfcorehdr changes and thus 535 * would require an update to purgatory itself to update the digest). 536 */ 537static void crash_handle_hotplug_event(unsigned int hp_action, unsigned int cpu) 538{ 539 struct kimage *image; 540 541 crash_hotplug_lock(); 542 /* Obtain lock while changing crash information */ 543 if (!kexec_trylock()) { 544 pr_info("kexec_trylock() failed, elfcorehdr may be inaccurate\n"); 545 crash_hotplug_unlock(); 546 return; 547 } 548 549 /* Check kdump is not loaded */ 550 if (!kexec_crash_image) 551 goto out; 552 553 image = kexec_crash_image; 554 555 /* Check that updating elfcorehdr is permitted */ 556 if (!(image->file_mode || image->update_elfcorehdr)) 557 goto out; 558 559 if (hp_action == KEXEC_CRASH_HP_ADD_CPU || 560 hp_action == KEXEC_CRASH_HP_REMOVE_CPU) 561 pr_debug("hp_action %u, cpu %u\n", hp_action, cpu); 562 else 563 pr_debug("hp_action %u\n", hp_action); 564 565 /* 566 * The elfcorehdr_index is set to -1 when the struct kimage 567 * is allocated. Find the segment containing the elfcorehdr, 568 * if not already found. 569 */ 570 if (image->elfcorehdr_index < 0) { 571 unsigned long mem; 572 unsigned char *ptr; 573 unsigned int n; 574 575 for (n = 0; n < image->nr_segments; n++) { 576 mem = image->segment[n].mem; 577 ptr = kmap_local_page(pfn_to_page(mem >> PAGE_SHIFT)); 578 if (ptr) { 579 /* The segment containing elfcorehdr */ 580 if (memcmp(ptr, ELFMAG, SELFMAG) == 0) 581 image->elfcorehdr_index = (int)n; 582 kunmap_local(ptr); 583 } 584 } 585 } 586 587 if (image->elfcorehdr_index < 0) { 588 pr_err("unable to locate elfcorehdr segment"); 589 goto out; 590 } 591 592 /* Needed in order for the segments to be updated */ 593 arch_kexec_unprotect_crashkres(); 594 595 /* Differentiate between normal load and hotplug update */ 596 image->hp_action = hp_action; 597 598 /* Now invoke arch-specific update handler */ 599 arch_crash_handle_hotplug_event(image); 600 601 /* No longer handling a hotplug event */ 602 image->hp_action = KEXEC_CRASH_HP_NONE; 603 image->elfcorehdr_updated = true; 604 605 /* Change back to read-only */ 606 arch_kexec_protect_crashkres(); 607 608 /* Errors in the callback is not a reason to rollback state */ 609out: 610 /* Release lock now that update complete */ 611 kexec_unlock(); 612 crash_hotplug_unlock(); 613} 614 615static int crash_memhp_notifier(struct notifier_block *nb, unsigned long val, void *v) 616{ 617 switch (val) { 618 case MEM_ONLINE: 619 crash_handle_hotplug_event(KEXEC_CRASH_HP_ADD_MEMORY, 620 KEXEC_CRASH_HP_INVALID_CPU); 621 break; 622 623 case MEM_OFFLINE: 624 crash_handle_hotplug_event(KEXEC_CRASH_HP_REMOVE_MEMORY, 625 KEXEC_CRASH_HP_INVALID_CPU); 626 break; 627 } 628 return NOTIFY_OK; 629} 630 631static struct notifier_block crash_memhp_nb = { 632 .notifier_call = crash_memhp_notifier, 633 .priority = 0 634}; 635 636static int crash_cpuhp_online(unsigned int cpu) 637{ 638 crash_handle_hotplug_event(KEXEC_CRASH_HP_ADD_CPU, cpu); 639 return 0; 640} 641 642static int crash_cpuhp_offline(unsigned int cpu) 643{ 644 crash_handle_hotplug_event(KEXEC_CRASH_HP_REMOVE_CPU, cpu); 645 return 0; 646} 647 648static int __init crash_hotplug_init(void) 649{ 650 int result = 0; 651 652 if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) 653 register_memory_notifier(&crash_memhp_nb); 654 655 if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) { 656 result = cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN, 657 "crash/cpuhp", crash_cpuhp_online, crash_cpuhp_offline); 658 } 659 660 return result; 661} 662 663subsys_initcall(crash_hotplug_init); 664#endif 665