1281494Sandrew/*- 2286958Sandrew * Copyright (c) 2006 Peter Wemm 3281494Sandrew * Copyright (c) 2015 The FreeBSD Foundation 4281494Sandrew * All rights reserved. 5281494Sandrew * 6281494Sandrew * This software was developed by Andrew Turner under 7281494Sandrew * sponsorship from the FreeBSD Foundation. 8281494Sandrew * 9281494Sandrew * Redistribution and use in source and binary forms, with or without 10281494Sandrew * modification, are permitted provided that the following conditions 11281494Sandrew * are met: 12286958Sandrew * 13281494Sandrew * 1. Redistributions of source code must retain the above copyright 14281494Sandrew * notice, this list of conditions and the following disclaimer. 15281494Sandrew * 2. Redistributions in binary form must reproduce the above copyright 16281494Sandrew * notice, this list of conditions and the following disclaimer in the 17281494Sandrew * documentation and/or other materials provided with the distribution. 18281494Sandrew * 19281494Sandrew * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20281494Sandrew * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21281494Sandrew * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22281494Sandrew * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23281494Sandrew * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24281494Sandrew * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25281494Sandrew * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26281494Sandrew * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27281494Sandrew * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28281494Sandrew * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29281494Sandrew * SUCH DAMAGE. 30281494Sandrew */ 31281494Sandrew 32281494Sandrew#include <sys/cdefs.h> 33281494Sandrew__FBSDID("$FreeBSD: stable/11/sys/arm64/arm64/minidump_machdep.c 331017 2018-03-15 19:08:33Z kevans $"); 34281494Sandrew 35281494Sandrew#include "opt_watchdog.h" 36281494Sandrew 37286958Sandrew#include "opt_watchdog.h" 38286958Sandrew 39281494Sandrew#include <sys/param.h> 40281494Sandrew#include <sys/systm.h> 41281494Sandrew#include <sys/conf.h> 42286958Sandrew#include <sys/cons.h> 43281494Sandrew#include <sys/kernel.h> 44281494Sandrew#include <sys/kerneldump.h> 45286958Sandrew#include <sys/msgbuf.h> 46286958Sandrew#include <sys/watchdog.h> 47331017Skevans#include <sys/vmmeter.h> 48281494Sandrew 49286958Sandrew#include <vm/vm.h> 50286958Sandrew#include <vm/vm_param.h> 51286958Sandrew#include <vm/vm_page.h> 52286958Sandrew#include <vm/vm_phys.h> 53286958Sandrew#include <vm/pmap.h> 54286958Sandrew 55281494Sandrew#include <machine/md_var.h> 56286958Sandrew#include <machine/pte.h> 57286958Sandrew#include <machine/minidump.h> 58281494Sandrew 59286958SandrewCTASSERT(sizeof(struct kerneldumpheader) == 512); 60286958Sandrew 61286958Sandrew/* 62286958Sandrew * Don't touch the first SIZEOF_METADATA bytes on the dump device. This 63286958Sandrew * is to protect us from metadata and to protect metadata from us. 64286958Sandrew */ 65286958Sandrew#define SIZEOF_METADATA (64*1024) 66286958Sandrew 67286958Sandrewuint64_t *vm_page_dump; 68286958Sandrewint vm_page_dump_size; 69286958Sandrew 70286958Sandrewstatic struct kerneldumpheader kdh; 71286958Sandrewstatic off_t dumplo; 72286958Sandrew 73286958Sandrew/* Handle chunked writes. */ 74286958Sandrewstatic size_t fragsz; 75286958Sandrewstatic void *dump_va; 76286958Sandrewstatic size_t counter, progress, dumpsize; 77286958Sandrew 78286958Sandrewstatic uint64_t tmpbuffer[PAGE_SIZE / sizeof(uint64_t)]; 79286958Sandrew 80286958SandrewCTASSERT(sizeof(*vm_page_dump) == 8); 81286958Sandrew 82286958Sandrewstatic int 83286958Sandrewis_dumpable(vm_paddr_t pa) 84286958Sandrew{ 85286958Sandrew vm_page_t m; 86286958Sandrew int i; 87286958Sandrew 88286958Sandrew if ((m = vm_phys_paddr_to_vm_page(pa)) != NULL) 89286958Sandrew return ((m->flags & PG_NODUMP) == 0); 90286958Sandrew for (i = 0; dump_avail[i] != 0 || dump_avail[i + 1] != 0; i += 2) { 91286958Sandrew if (pa >= dump_avail[i] && pa < dump_avail[i + 1]) 92286958Sandrew return (1); 93286958Sandrew } 94286958Sandrew return (0); 95286958Sandrew} 96286958Sandrew 97286958Sandrewstatic int 98286958Sandrewblk_flush(struct dumperinfo *di) 99286958Sandrew{ 100286958Sandrew int error; 101286958Sandrew 102286958Sandrew if (fragsz == 0) 103286958Sandrew return (0); 104286958Sandrew 105286958Sandrew error = dump_write(di, dump_va, 0, dumplo, fragsz); 106286958Sandrew dumplo += fragsz; 107286958Sandrew fragsz = 0; 108286958Sandrew return (error); 109286958Sandrew} 110286958Sandrew 111286958Sandrewstatic struct { 112286958Sandrew int min_per; 113286958Sandrew int max_per; 114286958Sandrew int visited; 115286958Sandrew} progress_track[10] = { 116286958Sandrew { 0, 10, 0}, 117286958Sandrew { 10, 20, 0}, 118286958Sandrew { 20, 30, 0}, 119286958Sandrew { 30, 40, 0}, 120286958Sandrew { 40, 50, 0}, 121286958Sandrew { 50, 60, 0}, 122286958Sandrew { 60, 70, 0}, 123286958Sandrew { 70, 80, 0}, 124286958Sandrew { 80, 90, 0}, 125286958Sandrew { 90, 100, 0} 126286958Sandrew}; 127286958Sandrew 128286958Sandrewstatic void 129286958Sandrewreport_progress(size_t progress, size_t dumpsize) 130286958Sandrew{ 131286958Sandrew int sofar, i; 132286958Sandrew 133286958Sandrew sofar = 100 - ((progress * 100) / dumpsize); 134286958Sandrew for (i = 0; i < nitems(progress_track); i++) { 135286958Sandrew if (sofar < progress_track[i].min_per || 136286958Sandrew sofar > progress_track[i].max_per) 137286958Sandrew continue; 138286958Sandrew if (progress_track[i].visited) 139286958Sandrew return; 140286958Sandrew progress_track[i].visited = 1; 141286958Sandrew printf("..%d%%", sofar); 142286958Sandrew return; 143286958Sandrew } 144286958Sandrew} 145286958Sandrew 146286958Sandrewstatic int 147286958Sandrewblk_write(struct dumperinfo *di, char *ptr, vm_paddr_t pa, size_t sz) 148286958Sandrew{ 149286958Sandrew size_t len; 150286958Sandrew int error, c; 151286958Sandrew u_int maxdumpsz; 152286958Sandrew 153286958Sandrew maxdumpsz = min(di->maxiosize, MAXDUMPPGS * PAGE_SIZE); 154286958Sandrew if (maxdumpsz == 0) /* seatbelt */ 155286958Sandrew maxdumpsz = PAGE_SIZE; 156286958Sandrew error = 0; 157286958Sandrew if ((sz % PAGE_SIZE) != 0) { 158286958Sandrew printf("size not page aligned\n"); 159286958Sandrew return (EINVAL); 160286958Sandrew } 161286958Sandrew if (ptr != NULL && pa != 0) { 162286958Sandrew printf("cant have both va and pa!\n"); 163286958Sandrew return (EINVAL); 164286958Sandrew } 165286958Sandrew if ((((uintptr_t)pa) % PAGE_SIZE) != 0) { 166286958Sandrew printf("address not page aligned %p\n", ptr); 167286958Sandrew return (EINVAL); 168286958Sandrew } 169286958Sandrew if (ptr != NULL) { 170286958Sandrew /* 171286958Sandrew * If we're doing a virtual dump, flush any 172286958Sandrew * pre-existing pa pages. 173286958Sandrew */ 174286958Sandrew error = blk_flush(di); 175286958Sandrew if (error) 176286958Sandrew return (error); 177286958Sandrew } 178286958Sandrew while (sz) { 179286958Sandrew len = maxdumpsz - fragsz; 180286958Sandrew if (len > sz) 181286958Sandrew len = sz; 182286958Sandrew counter += len; 183286958Sandrew progress -= len; 184286958Sandrew if (counter >> 22) { 185286958Sandrew report_progress(progress, dumpsize); 186286958Sandrew counter &= (1 << 22) - 1; 187286958Sandrew } 188286958Sandrew 189286958Sandrew wdog_kern_pat(WD_LASTVAL); 190286958Sandrew 191286958Sandrew if (ptr) { 192286958Sandrew error = dump_write(di, ptr, 0, dumplo, len); 193286958Sandrew if (error) 194286958Sandrew return (error); 195286958Sandrew dumplo += len; 196286958Sandrew ptr += len; 197286958Sandrew sz -= len; 198286958Sandrew } else { 199286958Sandrew dump_va = (void *)PHYS_TO_DMAP(pa); 200286958Sandrew fragsz += len; 201286958Sandrew pa += len; 202286958Sandrew sz -= len; 203286958Sandrew error = blk_flush(di); 204286958Sandrew if (error) 205286958Sandrew return (error); 206286958Sandrew } 207286958Sandrew 208286958Sandrew /* Check for user abort. */ 209286958Sandrew c = cncheckc(); 210286958Sandrew if (c == 0x03) 211286958Sandrew return (ECANCELED); 212286958Sandrew if (c != -1) 213286958Sandrew printf(" (CTRL-C to abort) "); 214286958Sandrew } 215286958Sandrew 216286958Sandrew return (0); 217286958Sandrew} 218286958Sandrew 219281494Sandrewint 220281494Sandrewminidumpsys(struct dumperinfo *di) 221281494Sandrew{ 222297446Sandrew pd_entry_t *l0, *l1, *l2; 223286958Sandrew pt_entry_t *l3; 224286958Sandrew uint32_t pmapsize; 225286958Sandrew vm_offset_t va; 226286958Sandrew vm_paddr_t pa; 227286958Sandrew int error; 228286958Sandrew uint64_t bits; 229286958Sandrew int i, bit; 230286958Sandrew int retry_count; 231286958Sandrew struct minidumphdr mdhdr; 232281494Sandrew 233286958Sandrew retry_count = 0; 234286958Sandrew retry: 235286958Sandrew retry_count++; 236286958Sandrew error = 0; 237286958Sandrew pmapsize = 0; 238286958Sandrew for (va = VM_MIN_KERNEL_ADDRESS; va < kernel_vm_end; va += L2_SIZE) { 239286958Sandrew pmapsize += PAGE_SIZE; 240297446Sandrew if (!pmap_get_tables(pmap_kernel(), va, &l0, &l1, &l2, &l3)) 241286958Sandrew continue; 242286958Sandrew 243286958Sandrew /* We should always be using the l2 table for kvm */ 244286958Sandrew if (l2 == NULL) 245286958Sandrew continue; 246286958Sandrew 247286958Sandrew if ((*l2 & ATTR_DESCR_MASK) == L2_BLOCK) { 248286958Sandrew pa = *l2 & ~ATTR_MASK; 249286958Sandrew for (i = 0; i < Ln_ENTRIES; i++, pa += PAGE_SIZE) { 250286958Sandrew if (is_dumpable(pa)) 251286958Sandrew dump_add_page(pa); 252286958Sandrew } 253286958Sandrew } else if ((*l2 & ATTR_DESCR_MASK) == L2_TABLE) { 254286958Sandrew for (i = 0; i < Ln_ENTRIES; i++) { 255286958Sandrew if ((l3[i] & ATTR_DESCR_MASK) != L3_PAGE) 256286958Sandrew continue; 257286958Sandrew pa = l3[i] & ~ATTR_MASK; 258286958Sandrew if (is_dumpable(pa)) 259286958Sandrew dump_add_page(pa); 260286958Sandrew } 261286958Sandrew } 262286958Sandrew } 263286958Sandrew 264286958Sandrew /* Calculate dump size. */ 265286958Sandrew dumpsize = pmapsize; 266286958Sandrew dumpsize += round_page(msgbufp->msg_size); 267286958Sandrew dumpsize += round_page(vm_page_dump_size); 268286958Sandrew for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { 269286958Sandrew bits = vm_page_dump[i]; 270286958Sandrew while (bits) { 271286958Sandrew bit = ffsl(bits) - 1; 272286958Sandrew pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + 273286958Sandrew bit) * PAGE_SIZE; 274286958Sandrew /* Clear out undumpable pages now if needed */ 275286958Sandrew if (is_dumpable(pa)) 276286958Sandrew dumpsize += PAGE_SIZE; 277286958Sandrew else 278286958Sandrew dump_drop_page(pa); 279286958Sandrew bits &= ~(1ul << bit); 280286958Sandrew } 281286958Sandrew } 282286958Sandrew dumpsize += PAGE_SIZE; 283286958Sandrew 284286958Sandrew /* Determine dump offset on device. */ 285286958Sandrew if (di->mediasize < SIZEOF_METADATA + dumpsize + sizeof(kdh) * 2) { 286286958Sandrew error = E2BIG; 287286958Sandrew goto fail; 288286958Sandrew } 289286958Sandrew dumplo = di->mediaoffset + di->mediasize - dumpsize; 290286958Sandrew dumplo -= sizeof(kdh) * 2; 291286958Sandrew progress = dumpsize; 292286958Sandrew 293286958Sandrew /* Initialize mdhdr */ 294286958Sandrew bzero(&mdhdr, sizeof(mdhdr)); 295286958Sandrew strcpy(mdhdr.magic, MINIDUMP_MAGIC); 296286958Sandrew mdhdr.version = MINIDUMP_VERSION; 297286958Sandrew mdhdr.msgbufsize = msgbufp->msg_size; 298286958Sandrew mdhdr.bitmapsize = vm_page_dump_size; 299286958Sandrew mdhdr.pmapsize = pmapsize; 300286958Sandrew mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; 301286958Sandrew mdhdr.dmapphys = DMAP_MIN_PHYSADDR; 302286958Sandrew mdhdr.dmapbase = DMAP_MIN_ADDRESS; 303286958Sandrew mdhdr.dmapend = DMAP_MAX_ADDRESS; 304286958Sandrew 305286958Sandrew mkdumpheader(&kdh, KERNELDUMPMAGIC, KERNELDUMP_AARCH64_VERSION, 306286958Sandrew dumpsize, di->blocksize); 307286958Sandrew 308286958Sandrew printf("Dumping %llu out of %ju MB:", (long long)dumpsize >> 20, 309286958Sandrew ptoa((uintmax_t)physmem) / 1048576); 310286958Sandrew 311286958Sandrew /* Dump leader */ 312286958Sandrew error = dump_write(di, &kdh, 0, dumplo, sizeof(kdh)); 313286958Sandrew if (error) 314286958Sandrew goto fail; 315286958Sandrew dumplo += sizeof(kdh); 316286958Sandrew 317286958Sandrew /* Dump my header */ 318286958Sandrew bzero(&tmpbuffer, sizeof(tmpbuffer)); 319286958Sandrew bcopy(&mdhdr, &tmpbuffer, sizeof(mdhdr)); 320286958Sandrew error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 321286958Sandrew if (error) 322286958Sandrew goto fail; 323286958Sandrew 324286958Sandrew /* Dump msgbuf up front */ 325286958Sandrew error = blk_write(di, (char *)msgbufp->msg_ptr, 0, 326286958Sandrew round_page(msgbufp->msg_size)); 327286958Sandrew if (error) 328286958Sandrew goto fail; 329286958Sandrew 330286958Sandrew /* Dump bitmap */ 331286958Sandrew error = blk_write(di, (char *)vm_page_dump, 0, 332286958Sandrew round_page(vm_page_dump_size)); 333286958Sandrew if (error) 334286958Sandrew goto fail; 335286958Sandrew 336286958Sandrew /* Dump kernel page directory pages */ 337286958Sandrew bzero(&tmpbuffer, sizeof(tmpbuffer)); 338286958Sandrew for (va = VM_MIN_KERNEL_ADDRESS; va < kernel_vm_end; va += L2_SIZE) { 339297446Sandrew if (!pmap_get_tables(pmap_kernel(), va, &l0, &l1, &l2, &l3)) { 340286958Sandrew /* We always write a page, even if it is zero */ 341286958Sandrew error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 342286958Sandrew if (error) 343286958Sandrew goto fail; 344286958Sandrew /* flush, in case we reuse tmpbuffer in the same block*/ 345286958Sandrew error = blk_flush(di); 346286958Sandrew if (error) 347286958Sandrew goto fail; 348286958Sandrew } else if (l2 == NULL) { 349286958Sandrew pa = (*l1 & ~ATTR_MASK) | (va & L1_OFFSET); 350286958Sandrew 351286958Sandrew /* Generate fake l3 entries based upon the l1 entry */ 352286958Sandrew for (i = 0; i < Ln_ENTRIES; i++) { 353286958Sandrew tmpbuffer[i] = pa + (i * PAGE_SIZE) | 354286958Sandrew ATTR_DEFAULT | L3_PAGE; 355286958Sandrew } 356286958Sandrew /* We always write a page, even if it is zero */ 357286958Sandrew error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 358286958Sandrew if (error) 359286958Sandrew goto fail; 360286958Sandrew /* flush, in case we reuse tmpbuffer in the same block*/ 361286958Sandrew error = blk_flush(di); 362286958Sandrew if (error) 363286958Sandrew goto fail; 364286958Sandrew bzero(&tmpbuffer, sizeof(tmpbuffer)); 365286958Sandrew } else if ((*l2 & ATTR_DESCR_MASK) == L2_BLOCK) { 366286958Sandrew /* TODO: Handle an invalid L2 entry */ 367286958Sandrew pa = (*l2 & ~ATTR_MASK) | (va & L2_OFFSET); 368286958Sandrew 369286958Sandrew /* Generate fake l3 entries based upon the l1 entry */ 370286958Sandrew for (i = 0; i < Ln_ENTRIES; i++) { 371286958Sandrew tmpbuffer[i] = pa + (i * PAGE_SIZE) | 372286958Sandrew ATTR_DEFAULT | L3_PAGE; 373286958Sandrew } 374286958Sandrew /* We always write a page, even if it is zero */ 375286958Sandrew error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); 376286958Sandrew if (error) 377286958Sandrew goto fail; 378286958Sandrew /* flush, in case we reuse fakepd in the same block */ 379286958Sandrew error = blk_flush(di); 380286958Sandrew if (error) 381286958Sandrew goto fail; 382286958Sandrew bzero(&tmpbuffer, sizeof(tmpbuffer)); 383286958Sandrew continue; 384286958Sandrew } else { 385286958Sandrew pa = *l2 & ~ATTR_MASK; 386286958Sandrew 387286958Sandrew /* We always write a page, even if it is zero */ 388286958Sandrew error = blk_write(di, NULL, pa, PAGE_SIZE); 389286958Sandrew if (error) 390286958Sandrew goto fail; 391286958Sandrew } 392286958Sandrew } 393286958Sandrew 394286958Sandrew /* Dump memory chunks */ 395286958Sandrew /* XXX cluster it up and use blk_dump() */ 396286958Sandrew for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { 397286958Sandrew bits = vm_page_dump[i]; 398286958Sandrew while (bits) { 399286958Sandrew bit = ffsl(bits) - 1; 400286958Sandrew pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + 401286958Sandrew bit) * PAGE_SIZE; 402286958Sandrew error = blk_write(di, 0, pa, PAGE_SIZE); 403286958Sandrew if (error) 404286958Sandrew goto fail; 405286958Sandrew bits &= ~(1ul << bit); 406286958Sandrew } 407286958Sandrew } 408286958Sandrew 409286958Sandrew error = blk_flush(di); 410286958Sandrew if (error) 411286958Sandrew goto fail; 412286958Sandrew 413286958Sandrew /* Dump trailer */ 414286958Sandrew error = dump_write(di, &kdh, 0, dumplo, sizeof(kdh)); 415286958Sandrew if (error) 416286958Sandrew goto fail; 417286958Sandrew dumplo += sizeof(kdh); 418286958Sandrew 419286958Sandrew /* Signal completion, signoff and exit stage left. */ 420286958Sandrew dump_write(di, NULL, 0, 0, 0); 421286958Sandrew printf("\nDump complete\n"); 422286958Sandrew return (0); 423286958Sandrew 424286958Sandrew fail: 425286958Sandrew if (error < 0) 426286958Sandrew error = -error; 427286958Sandrew 428286958Sandrew printf("\n"); 429286958Sandrew if (error == ENOSPC) { 430286958Sandrew printf("Dump map grown while dumping. "); 431286958Sandrew if (retry_count < 5) { 432286958Sandrew printf("Retrying...\n"); 433286958Sandrew goto retry; 434286958Sandrew } 435286958Sandrew printf("Dump failed.\n"); 436286958Sandrew } 437286958Sandrew else if (error == ECANCELED) 438286958Sandrew printf("Dump aborted\n"); 439286958Sandrew else if (error == E2BIG) 440286958Sandrew printf("Dump failed. Partition too small.\n"); 441286958Sandrew else 442286958Sandrew printf("** DUMP FAILED (ERROR %d) **\n", error); 443286958Sandrew return (error); 444281494Sandrew} 445281494Sandrew 446286958Sandrewvoid 447286958Sandrewdump_add_page(vm_paddr_t pa) 448286958Sandrew{ 449286958Sandrew int idx, bit; 450286958Sandrew 451286958Sandrew pa >>= PAGE_SHIFT; 452286958Sandrew idx = pa >> 6; /* 2^6 = 64 */ 453286958Sandrew bit = pa & 63; 454286958Sandrew atomic_set_long(&vm_page_dump[idx], 1ul << bit); 455286958Sandrew} 456286958Sandrew 457286958Sandrewvoid 458286958Sandrewdump_drop_page(vm_paddr_t pa) 459286958Sandrew{ 460286958Sandrew int idx, bit; 461286958Sandrew 462286958Sandrew pa >>= PAGE_SHIFT; 463286958Sandrew idx = pa >> 6; /* 2^6 = 64 */ 464286958Sandrew bit = pa & 63; 465286958Sandrew atomic_clear_long(&vm_page_dump[idx], 1ul << bit); 466286958Sandrew} 467