1/*
2 * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <stdint.h>
30#include <mach/mach_types.h>
31#include <mach/vm_param.h>
32#include <IOKit/IOHibernatePrivate.h>
33#include <IOKit/IOLib.h>
34#include <pexpert/boot.h>
35#include <libkern/libkern.h>
36
37#include <vm/WKdm_new.h>
38#include "IOHibernateInternal.h"
39
40#include <machine/pal_hibernate.h>
41
42/*
43This code is linked into the kernel but part of the "__HIB" section, which means
44its used by code running in the special context of restoring the kernel text and data
45from the hibernation image read by the booter. hibernate_kernel_entrypoint() and everything
46it calls or references needs to be careful to only touch memory also in the "__HIB" section.
47*/
48
49uint32_t gIOHibernateState;
50
51uint32_t gIOHibernateDebugFlags;
52
53static IOHibernateImageHeader _hibernateHeader;
54IOHibernateImageHeader * gIOHibernateCurrentHeader = &_hibernateHeader;
55
56ppnum_t gIOHibernateHandoffPages[64];
57uint32_t gIOHibernateHandoffPageCount = sizeof(gIOHibernateHandoffPages)
58					/ sizeof(gIOHibernateHandoffPages[0]);
59
60#if CONFIG_DEBUG
61void hibprintf(const char *fmt, ...);
62#else
63#define hibprintf(x...)
64#endif
65
66
67#if CONFIG_SLEEP
68#if defined(__i386__) || defined(__x86_64__)
69extern void acpi_wake_prot_entry(void);
70#endif
71#endif
72
73/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
74
75#if defined(__i386__) || defined(__x86_64__)
76
77#define rdtsc(lo,hi) \
78    __asm__ volatile("lfence; rdtsc; lfence" : "=a" (lo), "=d" (hi))
79
80static inline uint64_t rdtsc64(void)
81{
82    uint64_t lo, hi;
83    rdtsc(lo, hi);
84    return ((hi) << 32) | (lo);
85}
86
87#else
88
89static inline uint64_t rdtsc64(void)
90{
91    return (0);
92}
93
94#endif /* defined(__i386__) || defined(__x86_64__) */
95
96#if defined(__i386__) || defined(__x86_64__)
97
98#define DBGLOG	1
99
100#include <architecture/i386/pio.h>
101
102/* standard port addresses */
103enum {
104    COM1_PORT_ADDR = 0x3f8,
105    COM2_PORT_ADDR = 0x2f8
106};
107
108/* UART register offsets */
109enum {
110    UART_RBR = 0,  /* receive buffer Register   (R) */
111    UART_THR = 0,  /* transmit holding register (W) */
112    UART_DLL = 0,  /* DLAB = 1, divisor latch (LSB) */
113    UART_IER = 1,  /* interrupt enable register     */
114    UART_DLM = 1,  /* DLAB = 1, divisor latch (MSB) */
115    UART_IIR = 2,  /* interrupt ident register (R)  */
116    UART_FCR = 2,  /* fifo control register (W)     */
117    UART_LCR = 3,  /* line control register         */
118    UART_MCR = 4,  /* modem control register        */
119    UART_LSR = 5,  /* line status register          */
120    UART_MSR = 6,  /* modem status register         */
121    UART_SCR = 7   /* scratch register              */
122};
123
124enum {
125    UART_LCR_8BITS = 0x03,
126    UART_LCR_DLAB  = 0x80
127};
128
129enum {
130    UART_MCR_DTR   = 0x01,
131    UART_MCR_RTS   = 0x02,
132    UART_MCR_OUT1  = 0x04,
133    UART_MCR_OUT2  = 0x08,
134    UART_MCR_LOOP  = 0x10
135};
136
137enum {
138    UART_LSR_DR    = 0x01,
139    UART_LSR_OE    = 0x02,
140    UART_LSR_PE    = 0x04,
141    UART_LSR_FE    = 0x08,
142    UART_LSR_THRE  = 0x20
143};
144
145static void uart_putc(char c)
146{
147    while (!(inb(COM1_PORT_ADDR + UART_LSR) & UART_LSR_THRE))
148	{}
149    outb(COM1_PORT_ADDR + UART_THR, c);
150}
151
152static int debug_probe( void )
153{
154    /* Verify that the Scratch Register is accessible */
155    outb(COM1_PORT_ADDR + UART_SCR, 0x5a);
156    if (inb(COM1_PORT_ADDR + UART_SCR) != 0x5a) return false;
157    outb(COM1_PORT_ADDR + UART_SCR, 0xa5);
158    if (inb(COM1_PORT_ADDR + UART_SCR) != 0xa5) return false;
159    uart_putc('\n');
160    return true;
161}
162
163static void uart_puthex(uint64_t num)
164{
165    int bit;
166    char c;
167    bool leading = true;
168
169    for (bit = 60; bit >= 0; bit -= 4)
170    {
171	c = 0xf & (num >> bit);
172	if (c)
173	    leading = false;
174	else if (leading && bit)
175	    continue;
176	if (c <= 9)
177	    c += '0';
178	else
179	    c+= 'a' - 10;
180	uart_putc(c);
181    }
182}
183
184static void debug_code(uint32_t code, uint64_t value)
185{
186    int bit;
187    char c;
188
189    if (!(kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags))
190	return;
191
192    for (bit = 24; bit >= 0; bit -= 8)
193    {
194	c = 0xFF & (code >> bit);
195	if (c)
196	    uart_putc(c);
197    }
198    uart_putc('=');
199    uart_puthex(value);
200    uart_putc('\n');
201    uart_putc('\r');
202}
203
204#endif /* defined(__i386__) || defined(__x86_64__) */
205
206#if !defined(DBGLOG)
207#define debug_probe()	    (false)
208#define debug_code(c, v)    {}
209#endif
210
211enum
212{
213    kIOHibernateRestoreCodeImageStart	    = 'imgS',
214    kIOHibernateRestoreCodeImageEnd	    = 'imgE',
215    kIOHibernateRestoreCodePageIndexStart   = 'pgiS',
216    kIOHibernateRestoreCodePageIndexEnd	    = 'pgiE',
217    kIOHibernateRestoreCodeMapStart	    = 'mapS',
218    kIOHibernateRestoreCodeMapEnd	    = 'mapE',
219    kIOHibernateRestoreCodeWakeMapSize	    = 'wkms',
220    kIOHibernateRestoreCodeConflictPage	    = 'cfpg',
221    kIOHibernateRestoreCodeConflictSource   = 'cfsr',
222    kIOHibernateRestoreCodeNoMemory         = 'nomm',
223    kIOHibernateRestoreCodeTag              = 'tag ',
224    kIOHibernateRestoreCodeSignature        = 'sign',
225    kIOHibernateRestoreCodeMapVirt          = 'mapV',
226    kIOHibernateRestoreCodeHandoffPages     = 'hand',
227    kIOHibernateRestoreCodeHandoffCount     = 'hndc',
228};
229
230/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
231
232
233static void fatal(void)
234{
235#if defined(__i386__) || defined(__x86_64__)
236    outb(0xcf9, 6);
237#else
238    while (true) {}
239#endif
240}
241
242/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
243
244uint32_t
245hibernate_sum_page(uint8_t *buf, uint32_t ppnum)
246{
247    return (((uint32_t *)buf)[((PAGE_SIZE >> 2) - 1) & ppnum]);
248}
249
250/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
251
252static hibernate_bitmap_t *
253hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
254{
255    uint32_t             bank;
256    hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
257
258    for (bank = 0; bank < list->bank_count; bank++)
259    {
260	if ((page >= bitmap->first_page) && (page <= bitmap->last_page))
261	    break;
262	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
263    }
264    if (bank == list->bank_count)
265	bitmap = NULL;
266
267    return (bitmap);
268}
269
270hibernate_bitmap_t *
271hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
272{
273    uint32_t             bank, page = *pPage;
274    hibernate_bitmap_t * bitmap = &list->bank_bitmap[0];
275
276    for (bank = 0; bank < list->bank_count; bank++)
277    {
278	if (page <= bitmap->first_page)
279	{
280	    *pPage = bitmap->first_page;
281	    break;
282	}
283	if (page <= bitmap->last_page)
284	    break;
285	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
286    }
287    if (bank == list->bank_count)
288	bitmap = NULL;
289
290    return (bitmap);
291}
292
293void
294hibernate_page_bitset(hibernate_page_list_t * list, boolean_t set, uint32_t page)
295{
296    hibernate_bitmap_t * bitmap;
297
298    bitmap = hibernate_page_bitmap(list, page);
299    if (bitmap)
300    {
301	page -= bitmap->first_page;
302	if (set)
303	    bitmap->bitmap[page >> 5] |= (0x80000000 >> (page & 31));
304	    //setbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
305	else
306	    bitmap->bitmap[page >> 5] &= ~(0x80000000 >> (page & 31));
307	    //clrbit(page - bitmap->first_page, (int *) &bitmap->bitmap[0]);
308    }
309}
310
311boolean_t
312hibernate_page_bittst(hibernate_page_list_t * list, uint32_t page)
313{
314    boolean_t		 result = TRUE;
315    hibernate_bitmap_t * bitmap;
316
317    bitmap = hibernate_page_bitmap(list, page);
318    if (bitmap)
319    {
320	page -= bitmap->first_page;
321	result = (0 != (bitmap->bitmap[page >> 5] & (0x80000000 >> (page & 31))));
322    }
323    return (result);
324}
325
326// count bits clear or set (set == TRUE) starting at page.
327uint32_t
328hibernate_page_bitmap_count(hibernate_bitmap_t * bitmap, uint32_t set, uint32_t page)
329{
330    uint32_t index, bit, bits;
331    uint32_t count;
332
333    count = 0;
334
335    index = (page - bitmap->first_page) >> 5;
336    bit = (page - bitmap->first_page) & 31;
337
338    bits = bitmap->bitmap[index];
339    if (set)
340	bits = ~bits;
341    bits = (bits << bit);
342    if (bits)
343	count += __builtin_clz(bits);
344    else
345    {
346	count += 32 - bit;
347	while (++index < bitmap->bitmapwords)
348	{
349	    bits = bitmap->bitmap[index];
350	    if (set)
351		bits = ~bits;
352	    if (bits)
353	    {
354		count += __builtin_clz(bits);
355		break;
356	    }
357	    count += 32;
358	}
359    }
360
361    if ((page + count) > (bitmap->last_page + 1)) count = (bitmap->last_page + 1) - page;
362
363    return (count);
364}
365
366static ppnum_t
367hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
368{
369    uint32_t		 nextFree = *pNextFree;
370    uint32_t		 nextFreeInBank;
371    hibernate_bitmap_t * bitmap;
372
373    nextFreeInBank = nextFree + 1;
374    while ((bitmap = hibernate_page_bitmap_pin(list, &nextFreeInBank)))
375    {
376	nextFreeInBank += hibernate_page_bitmap_count(bitmap, FALSE, nextFreeInBank);
377	if (nextFreeInBank <= bitmap->last_page)
378	{
379	    *pNextFree = nextFreeInBank;
380	    break;
381	}
382    }
383
384    if (!bitmap)
385    {
386	debug_code(kIOHibernateRestoreCodeNoMemory, nextFree);
387	fatal();
388	nextFree = 0;
389    }
390
391    return (nextFree);
392}
393
394static uint32_t
395store_one_page(uint32_t procFlags, uint32_t * src, uint32_t compressedSize,
396		uint32_t * buffer, uint32_t ppnum)
397{
398	uint64_t dst = ptoa_64(ppnum);
399	uint8_t scratch[WKdm_SCRATCH_BUF_SIZE] __attribute__ ((aligned (16)));
400
401	if (compressedSize != PAGE_SIZE)
402	{
403		dst = pal_hib_map(DEST_COPY_AREA, dst);
404		if (compressedSize) WKdm_decompress_new((WK_word*) src, (WK_word*)(uintptr_t)dst, (WK_word*) &scratch[0], PAGE_SIZE);
405		else bzero((void *) dst, PAGE_SIZE);
406	}
407	else
408	{
409		dst = hibernate_restore_phys_page((uint64_t) (uintptr_t) src, dst, PAGE_SIZE, procFlags);
410	}
411
412	return hibernate_sum_page((uint8_t *)(uintptr_t)dst, ppnum);
413}
414
415#define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
416
417long
418hibernate_kernel_entrypoint(uint32_t p1,
419                            uint32_t p2, uint32_t p3, uint32_t p4)
420{
421    uint64_t headerPhys;
422    uint64_t mapPhys;
423    uint64_t srcPhys;
424    uint64_t imageReadPhys;
425    uint64_t pageIndexPhys;
426    uint32_t * pageIndexSource;
427    hibernate_page_list_t * map;
428    uint32_t stage;
429    uint32_t count;
430    uint32_t ppnum;
431    uint32_t page;
432    uint32_t conflictCount;
433    uint32_t compressedSize;
434    uint32_t uncompressedPages;
435    uint32_t copyPageListHeadPage;
436    uint32_t pageListPage;
437    uint32_t * copyPageList;
438    uint32_t * src;
439    uint32_t copyPageIndex;
440    uint32_t sum;
441    uint32_t pageSum;
442    uint32_t nextFree;
443    uint32_t lastImagePage;
444    uint32_t lastMapPage;
445    uint32_t lastPageIndexPage;
446    uint32_t handoffPages;
447    uint32_t handoffPageCount;
448
449    uint64_t timeStart;
450    timeStart = rdtsc64();
451
452    C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
453
454    headerPhys = ptoa_64(p1);
455
456    if ((kIOHibernateDebugRestoreLogs & gIOHibernateDebugFlags) && !debug_probe())
457	gIOHibernateDebugFlags &= ~kIOHibernateDebugRestoreLogs;
458
459    debug_code(kIOHibernateRestoreCodeImageStart, headerPhys);
460
461    memcpy(gIOHibernateCurrentHeader,
462	   (void *) pal_hib_map(IMAGE_AREA, headerPhys),
463	   sizeof(IOHibernateImageHeader));
464
465    debug_code(kIOHibernateRestoreCodeSignature, gIOHibernateCurrentHeader->signature);
466
467    mapPhys = headerPhys
468             + (offsetof(IOHibernateImageHeader, fileExtentMap)
469	     + gIOHibernateCurrentHeader->fileExtentMapSize
470	     + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount)
471	     + gIOHibernateCurrentHeader->previewSize);
472
473    map = (hibernate_page_list_t *) pal_hib_map(BITMAP_AREA, mapPhys);
474
475    lastImagePage = atop_64(headerPhys + gIOHibernateCurrentHeader->image1Size);
476    lastMapPage = atop_64(mapPhys + gIOHibernateCurrentHeader->bitmapSize);
477
478    handoffPages     = gIOHibernateCurrentHeader->handoffPages;
479    handoffPageCount = gIOHibernateCurrentHeader->handoffPageCount;
480
481    debug_code(kIOHibernateRestoreCodeImageEnd,       ptoa_64(lastImagePage));
482    debug_code(kIOHibernateRestoreCodeMapStart,       mapPhys);
483    debug_code(kIOHibernateRestoreCodeMapEnd,         ptoa_64(lastMapPage));
484
485    debug_code(kIOHibernateRestoreCodeMapVirt, (uintptr_t) map);
486    debug_code(kIOHibernateRestoreCodeHandoffPages, ptoa_64(handoffPages));
487    debug_code(kIOHibernateRestoreCodeHandoffCount, handoffPageCount);
488
489    // knock all the image pages to be used out of free map
490    for (ppnum = atop_64(headerPhys); ppnum <= lastImagePage; ppnum++)
491    {
492	hibernate_page_bitset(map, FALSE, ppnum);
493    }
494    // knock all the handoff pages to be used out of free map
495    for (ppnum = handoffPages; ppnum < (handoffPages + handoffPageCount); ppnum++)
496    {
497	hibernate_page_bitset(map, FALSE, ppnum);
498    }
499
500    nextFree = 0;
501    hibernate_page_list_grab(map, &nextFree);
502
503    sum = gIOHibernateCurrentHeader->actualRestore1Sum;
504    gIOHibernateCurrentHeader->diag[0] = atop_64(headerPhys);
505    gIOHibernateCurrentHeader->diag[1] = sum;
506    gIOHibernateCurrentHeader->trampolineTime = 0;
507
508    uncompressedPages    = 0;
509    conflictCount        = 0;
510    copyPageListHeadPage = 0;
511    copyPageList         = 0;
512    copyPageIndex        = PAGE_SIZE >> 2;
513
514    compressedSize       = PAGE_SIZE;
515    stage                = 2;
516    count                = 0;
517    srcPhys              = 0;
518
519    if (gIOHibernateCurrentHeader->previewSize)
520    {
521	pageIndexPhys     = headerPhys
522	                   + (offsetof(IOHibernateImageHeader, fileExtentMap)
523			   + gIOHibernateCurrentHeader->fileExtentMapSize
524			   + ptoa_32(gIOHibernateCurrentHeader->restore1PageCount));
525	imageReadPhys     = (pageIndexPhys + gIOHibernateCurrentHeader->previewPageListSize);
526	lastPageIndexPage = atop_64(imageReadPhys);
527	pageIndexSource   = (uint32_t *) pal_hib_map(IMAGE2_AREA, pageIndexPhys);
528    }
529    else
530    {
531	pageIndexPhys     = 0;
532	lastPageIndexPage = 0;
533	imageReadPhys     = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
534    }
535
536    debug_code(kIOHibernateRestoreCodePageIndexStart, pageIndexPhys);
537    debug_code(kIOHibernateRestoreCodePageIndexEnd,   ptoa_64(lastPageIndexPage));
538
539    while (1)
540    {
541	switch (stage)
542	{
543	    case 2:
544		// copy handoff data
545		count = srcPhys ? 0 : handoffPageCount;
546		if (!count)
547		    break;
548		if (count > gIOHibernateHandoffPageCount) count = gIOHibernateHandoffPageCount;
549		srcPhys = ptoa_64(handoffPages);
550		break;
551
552	    case 1:
553		// copy pageIndexSource pages == preview image data
554		if (!srcPhys)
555		{
556		    if (!pageIndexPhys) break;
557		    srcPhys = imageReadPhys;
558		}
559		ppnum = pageIndexSource[0];
560		count = pageIndexSource[1];
561		pageIndexSource += 2;
562		pageIndexPhys   += 2 * sizeof(pageIndexSource[0]);
563		imageReadPhys = srcPhys;
564		break;
565
566	    case 0:
567		// copy pages
568		if (!srcPhys) srcPhys = (mapPhys + gIOHibernateCurrentHeader->bitmapSize);
569		src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
570		ppnum = src[0];
571		count = src[1];
572		srcPhys += 2 * sizeof(*src);
573		imageReadPhys = srcPhys;
574		break;
575	}
576
577
578	if (!count)
579	{
580	    if (!stage)
581	        break;
582	    stage--;
583	    srcPhys = 0;
584	    continue;
585	}
586
587	for (page = 0; page < count; page++, ppnum++)
588	{
589	    uint32_t tag;
590	    int conflicts;
591
592	    src = (uint32_t *) pal_hib_map(IMAGE_AREA, srcPhys);
593
594	    if (2 == stage) ppnum = gIOHibernateHandoffPages[page];
595	    else if (!stage)
596	    {
597		tag = *src++;
598//		debug_code(kIOHibernateRestoreCodeTag, (uintptr_t) tag);
599		srcPhys += sizeof(*src);
600		compressedSize = kIOHibernateTagLength & tag;
601	    }
602
603	    conflicts = (ppnum >= atop_64(mapPhys)) && (ppnum <= lastMapPage);
604
605	    conflicts |= ((ppnum >= atop_64(imageReadPhys)) && (ppnum <= lastImagePage));
606
607	    if (stage >= 2)
608 		conflicts |= ((ppnum >= atop_64(srcPhys)) && (ppnum <= (handoffPages + handoffPageCount - 1)));
609
610	    if (stage >= 1)
611 		conflicts |= ((ppnum >= atop_64(pageIndexPhys)) && (ppnum <= lastPageIndexPage));
612
613	    if (!conflicts)
614	    {
615		pageSum = store_one_page(gIOHibernateCurrentHeader->processorFlags,
616					 src, compressedSize, 0, ppnum);
617		if (stage != 2)
618		    sum += pageSum;
619		uncompressedPages++;
620	    }
621	    else
622	    {
623		uint32_t   bufferPage = 0;
624		uint32_t * dst;
625
626//		debug_code(kIOHibernateRestoreCodeConflictPage,   ppnum);
627//		debug_code(kIOHibernateRestoreCodeConflictSource, (uintptr_t) src);
628		conflictCount++;
629		if (compressedSize)
630		{
631		    // alloc new buffer page
632		    bufferPage = hibernate_page_list_grab(map, &nextFree);
633		    dst = (uint32_t *)pal_hib_map(DEST_COPY_AREA, ptoa_64(bufferPage));
634		    memcpy(dst, src, compressedSize);
635		}
636		if (copyPageIndex > ((PAGE_SIZE >> 2) - 3))
637		{
638		    // alloc new copy list page
639		    pageListPage = hibernate_page_list_grab(map, &nextFree);
640		    // link to current
641		    if (copyPageList) {
642			    copyPageList[1] = pageListPage;
643		    } else {
644			    copyPageListHeadPage = pageListPage;
645		    }
646		    copyPageList = (uint32_t *)pal_hib_map(SRC_COPY_AREA,
647				    ptoa_64(pageListPage));
648		    copyPageList[1] = 0;
649		    copyPageIndex = 2;
650		}
651		copyPageList[copyPageIndex++] = ppnum;
652		copyPageList[copyPageIndex++] = bufferPage;
653		copyPageList[copyPageIndex++] = (compressedSize | (stage << 24));
654		copyPageList[0] = copyPageIndex;
655	    }
656	    srcPhys += ((compressedSize + 3) & ~3);
657	    src     += ((compressedSize + 3) >> 2);
658	}
659    }
660
661    /* src points to the last page restored, so we need to skip over that */
662    hibernateRestorePALState(src);
663
664    // -- copy back conflicts
665
666    pageListPage = copyPageListHeadPage;
667    while (pageListPage)
668    {
669	copyPageList = (uint32_t *)pal_hib_map(COPY_PAGE_AREA, ptoa_64(pageListPage));
670	for (copyPageIndex = 2; copyPageIndex < copyPageList[0]; copyPageIndex += 3)
671	{
672	    ppnum          = copyPageList[copyPageIndex + 0];
673	    srcPhys        = ptoa_64(copyPageList[copyPageIndex + 1]);
674	    src            = (uint32_t *) pal_hib_map(SRC_COPY_AREA, srcPhys);
675	    compressedSize = copyPageList[copyPageIndex + 2];
676	    stage 	   = compressedSize >> 24;
677	    compressedSize &= 0x1FFF;
678	    pageSum        = store_one_page(gIOHibernateCurrentHeader->processorFlags,
679			    			src, compressedSize, 0, ppnum);
680	    if (stage != 2)
681	    	sum += pageSum;
682	    uncompressedPages++;
683	}
684	pageListPage = copyPageList[1];
685    }
686
687    pal_hib_patchup();
688
689    // -- image has been destroyed...
690
691    gIOHibernateCurrentHeader->actualImage1Sum         = sum;
692    gIOHibernateCurrentHeader->actualUncompressedPages = uncompressedPages;
693    gIOHibernateCurrentHeader->conflictCount           = conflictCount;
694    gIOHibernateCurrentHeader->nextFree                = nextFree;
695
696    gIOHibernateState = kIOHibernateStateWakingFromHibernate;
697
698    gIOHibernateCurrentHeader->trampolineTime = (((rdtsc64() - timeStart)) >> 8);
699
700#if CONFIG_SLEEP
701#if defined(__i386__) || defined(__x86_64__)
702    typedef void (*ResetProc)(void);
703    ResetProc proc;
704    proc = HIB_ENTRYPOINT;
705    // flush caches
706    __asm__("wbinvd");
707    proc();
708#else
709// implement me
710#endif
711#endif
712
713    return -1;
714}
715
716#if CONFIG_DEBUG
717/* standalone printf implementation */
718/*-
719 * Copyright (c) 1986, 1988, 1991, 1993
720 *	The Regents of the University of California.  All rights reserved.
721 * (c) UNIX System Laboratories, Inc.
722 * All or some portions of this file are derived from material licensed
723 * to the University of California by American Telephone and Telegraph
724 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
725 * the permission of UNIX System Laboratories, Inc.
726 *
727 * Redistribution and use in source and binary forms, with or without
728 * modification, are permitted provided that the following conditions
729 * are met:
730 * 1. Redistributions of source code must retain the above copyright
731 *    notice, this list of conditions and the following disclaimer.
732 * 2. Redistributions in binary form must reproduce the above copyright
733 *    notice, this list of conditions and the following disclaimer in the
734 *    documentation and/or other materials provided with the distribution.
735 * 4. Neither the name of the University nor the names of its contributors
736 *    may be used to endorse or promote products derived from this software
737 *    without specific prior written permission.
738 *
739 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
740 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
741 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
742 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
743 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
744 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
745 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
746 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
747 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
748 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
749 * SUCH DAMAGE.
750 *
751 *	@(#)subr_prf.c	8.3 (Berkeley) 1/21/94
752 */
753
754typedef long ptrdiff_t;
755char const hibhex2ascii_data[] = "0123456789abcdefghijklmnopqrstuvwxyz";
756#define hibhex2ascii(hex)  (hibhex2ascii_data[hex])
757#define toupper(c)      ((c) - 0x20 * (((c) >= 'a') && ((c) <= 'z')))
758static size_t
759hibstrlen(const char *s)
760{
761	size_t l = 0;
762	while (*s++)
763		l++;
764	return l;
765}
766
767/* Max number conversion buffer length: a u_quad_t in base 2, plus NUL byte. */
768#define MAXNBUF	(sizeof(intmax_t) * NBBY + 1)
769
770/*
771 * Put a NUL-terminated ASCII number (base <= 36) in a buffer in reverse
772 * order; return an optional length and a pointer to the last character
773 * written in the buffer (i.e., the first character of the string).
774 * The buffer pointed to by `nbuf' must have length >= MAXNBUF.
775 */
776static char *
777ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper)
778{
779	char *p, c;
780
781	/* Truncate so we don't call umoddi3, which isn't in __HIB */
782#if !defined(__LP64__)
783	uint32_t num2 = (uint32_t) num;
784#else
785	uintmax_t num2 = num;
786#endif
787
788	p = nbuf;
789	*p = '\0';
790	do {
791		c = hibhex2ascii(num2 % base);
792		*++p = upper ? toupper(c) : c;
793	} while (num2 /= base);
794	if (lenp)
795		*lenp = (int)(p - nbuf);
796	return (p);
797}
798
799/*
800 * Scaled down version of printf(3).
801 *
802 * Two additional formats:
803 *
804 * The format %b is supported to decode error registers.
805 * Its usage is:
806 *
807 *	printf("reg=%b\n", regval, "*");
808 *
809 * where  is the output base expressed as a control character, e.g.
810 * \10 gives octal; \20 gives hex.  Each arg is a sequence of characters,
811 * the first of which gives the bit number to be inspected (origin 1), and
812 * the next characters (up to a control character, i.e. a character <= 32),
813 * give the name of the register.  Thus:
814 *
815 *	kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE\n");
816 *
817 * would produce output:
818 *
819 *	reg=3
820 *
821 * XXX:  %D  -- Hexdump, takes pointer and separator string:
822 *		("%6D", ptr, ":")   -> XX:XX:XX:XX:XX:XX
823 *		("%*D", len, ptr, " " -> XX XX XX XX ...
824 */
825static int
826hibkvprintf(char const *fmt, void (*func)(int, void*), void *arg, int radix, va_list ap)
827{
828#define PCHAR(c) {int cc=(c); if (func) (*func)(cc,arg); else *d++ = cc; retval++; }
829	char nbuf[MAXNBUF];
830	char *d;
831	const char *p, *percent, *q;
832	u_char *up;
833	int ch, n;
834	uintmax_t num;
835	int base, lflag, qflag, tmp, width, ladjust, sharpflag, neg, sign, dot;
836	int cflag, hflag, jflag, tflag, zflag;
837	int dwidth, upper;
838	char padc;
839	int stop = 0, retval = 0;
840
841	num = 0;
842	if (!func)
843		d = (char *) arg;
844	else
845		d = NULL;
846
847	if (fmt == NULL)
848		fmt = "(fmt null)\n";
849
850	if (radix < 2 || radix > 36)
851		radix = 10;
852
853	for (;;) {
854		padc = ' ';
855		width = 0;
856		while ((ch = (u_char)*fmt++) != '%' || stop) {
857			if (ch == '\0')
858				return (retval);
859			PCHAR(ch);
860		}
861		percent = fmt - 1;
862		qflag = 0; lflag = 0; ladjust = 0; sharpflag = 0; neg = 0;
863		sign = 0; dot = 0; dwidth = 0; upper = 0;
864		cflag = 0; hflag = 0; jflag = 0; tflag = 0; zflag = 0;
865reswitch:	switch (ch = (u_char)*fmt++) {
866		case '.':
867			dot = 1;
868			goto reswitch;
869		case '#':
870			sharpflag = 1;
871			goto reswitch;
872		case '+':
873			sign = 1;
874			goto reswitch;
875		case '-':
876			ladjust = 1;
877			goto reswitch;
878		case '%':
879			PCHAR(ch);
880			break;
881		case '*':
882			if (!dot) {
883				width = va_arg(ap, int);
884				if (width < 0) {
885					ladjust = !ladjust;
886					width = -width;
887				}
888			} else {
889				dwidth = va_arg(ap, int);
890			}
891			goto reswitch;
892		case '0':
893			if (!dot) {
894				padc = '0';
895				goto reswitch;
896			}
897		case '1': case '2': case '3': case '4':
898		case '5': case '6': case '7': case '8': case '9':
899				for (n = 0;; ++fmt) {
900					n = n * 10 + ch - '0';
901					ch = *fmt;
902					if (ch < '0' || ch > '9')
903						break;
904				}
905			if (dot)
906				dwidth = n;
907			else
908				width = n;
909			goto reswitch;
910		case 'b':
911			num = (u_int)va_arg(ap, int);
912			p = va_arg(ap, char *);
913			for (q = ksprintn(nbuf, num, *p++, NULL, 0); *q;)
914				PCHAR(*q--);
915
916			if (num == 0)
917				break;
918
919			for (tmp = 0; *p;) {
920				n = *p++;
921				if (num & (1 << (n - 1))) {
922					PCHAR(tmp ? ',' : '<');
923					for (; (n = *p) > ' '; ++p)
924						PCHAR(n);
925					tmp = 1;
926				} else
927					for (; *p > ' '; ++p)
928						continue;
929			}
930			if (tmp)
931				PCHAR('>');
932			break;
933		case 'c':
934			PCHAR(va_arg(ap, int));
935			break;
936		case 'D':
937			up = va_arg(ap, u_char *);
938			p = va_arg(ap, char *);
939			if (!width)
940				width = 16;
941			while(width--) {
942				PCHAR(hibhex2ascii(*up >> 4));
943				PCHAR(hibhex2ascii(*up & 0x0f));
944				up++;
945				if (width)
946					for (q=p;*q;q++)
947						PCHAR(*q);
948			}
949			break;
950		case 'd':
951		case 'i':
952			base = 10;
953			sign = 1;
954			goto handle_sign;
955		case 'h':
956			if (hflag) {
957				hflag = 0;
958				cflag = 1;
959			} else
960				hflag = 1;
961			goto reswitch;
962		case 'j':
963			jflag = 1;
964			goto reswitch;
965		case 'l':
966			if (lflag) {
967				lflag = 0;
968				qflag = 1;
969			} else
970				lflag = 1;
971			goto reswitch;
972		case 'n':
973			if (jflag)
974				*(va_arg(ap, intmax_t *)) = retval;
975			else if (qflag)
976				*(va_arg(ap, quad_t *)) = retval;
977			else if (lflag)
978				*(va_arg(ap, long *)) = retval;
979			else if (zflag)
980				*(va_arg(ap, size_t *)) = retval;
981			else if (hflag)
982				*(va_arg(ap, short *)) = retval;
983			else if (cflag)
984				*(va_arg(ap, char *)) = retval;
985			else
986				*(va_arg(ap, int *)) = retval;
987			break;
988		case 'o':
989			base = 8;
990			goto handle_nosign;
991		case 'p':
992			base = 16;
993			sharpflag = (width == 0);
994			sign = 0;
995			num = (uintptr_t)va_arg(ap, void *);
996			goto number;
997		case 'q':
998			qflag = 1;
999			goto reswitch;
1000		case 'r':
1001			base = radix;
1002			if (sign)
1003				goto handle_sign;
1004			goto handle_nosign;
1005		case 's':
1006			p = va_arg(ap, char *);
1007			if (p == NULL)
1008				p = "(null)";
1009			if (!dot)
1010				n = (typeof(n))hibstrlen (p);
1011			else
1012				for (n = 0; n < dwidth && p[n]; n++)
1013					continue;
1014
1015			width -= n;
1016
1017			if (!ladjust && width > 0)
1018				while (width--)
1019					PCHAR(padc);
1020			while (n--)
1021				PCHAR(*p++);
1022			if (ladjust && width > 0)
1023				while (width--)
1024					PCHAR(padc);
1025			break;
1026		case 't':
1027			tflag = 1;
1028			goto reswitch;
1029		case 'u':
1030			base = 10;
1031			goto handle_nosign;
1032		case 'X':
1033			upper = 1;
1034		case 'x':
1035			base = 16;
1036			goto handle_nosign;
1037		case 'y':
1038			base = 16;
1039			sign = 1;
1040			goto handle_sign;
1041		case 'z':
1042			zflag = 1;
1043			goto reswitch;
1044handle_nosign:
1045			sign = 0;
1046			if (jflag)
1047				num = va_arg(ap, uintmax_t);
1048			else if (qflag)
1049				num = va_arg(ap, u_quad_t);
1050			else if (tflag)
1051				num = va_arg(ap, ptrdiff_t);
1052			else if (lflag)
1053				num = va_arg(ap, u_long);
1054			else if (zflag)
1055				num = va_arg(ap, size_t);
1056			else if (hflag)
1057				num = (u_short)va_arg(ap, int);
1058			else if (cflag)
1059				num = (u_char)va_arg(ap, int);
1060			else
1061				num = va_arg(ap, u_int);
1062			goto number;
1063handle_sign:
1064			if (jflag)
1065				num = va_arg(ap, intmax_t);
1066			else if (qflag)
1067				num = va_arg(ap, quad_t);
1068			else if (tflag)
1069				num = va_arg(ap, ptrdiff_t);
1070			else if (lflag)
1071				num = va_arg(ap, long);
1072			else if (zflag)
1073				num = va_arg(ap, ssize_t);
1074			else if (hflag)
1075				num = (short)va_arg(ap, int);
1076			else if (cflag)
1077				num = (char)va_arg(ap, int);
1078			else
1079				num = va_arg(ap, int);
1080number:
1081			if (sign && (intmax_t)num < 0) {
1082				neg = 1;
1083				num = -(intmax_t)num;
1084			}
1085			p = ksprintn(nbuf, num, base, &tmp, upper);
1086			if (sharpflag && num != 0) {
1087				if (base == 8)
1088					tmp++;
1089				else if (base == 16)
1090					tmp += 2;
1091			}
1092			if (neg)
1093				tmp++;
1094
1095			if (!ladjust && padc != '0' && width
1096			    && (width -= tmp) > 0)
1097				while (width--)
1098					PCHAR(padc);
1099			if (neg)
1100				PCHAR('-');
1101			if (sharpflag && num != 0) {
1102				if (base == 8) {
1103					PCHAR('0');
1104				} else if (base == 16) {
1105					PCHAR('0');
1106					PCHAR('x');
1107				}
1108			}
1109			if (!ladjust && width && (width -= tmp) > 0)
1110				while (width--)
1111					PCHAR(padc);
1112
1113			while (*p)
1114				PCHAR(*p--);
1115
1116			if (ladjust && width && (width -= tmp) > 0)
1117				while (width--)
1118					PCHAR(padc);
1119
1120			break;
1121		default:
1122			while (percent < fmt)
1123				PCHAR(*percent++);
1124			/*
1125			 * Since we ignore an formatting argument it is no
1126			 * longer safe to obey the remaining formatting
1127			 * arguments as the arguments will no longer match
1128			 * the format specs.
1129			 */
1130			stop = 1;
1131			break;
1132		}
1133	}
1134#undef PCHAR
1135}
1136
1137
1138static void
1139putchar(int c, void *arg)
1140{
1141	(void)arg;
1142	uart_putc(c);
1143}
1144
1145void
1146hibprintf(const char *fmt, ...)
1147{
1148	/* http://www.pagetable.com/?p=298 */
1149	va_list ap;
1150
1151	va_start(ap, fmt);
1152	hibkvprintf(fmt, putchar, NULL, 10, ap);
1153	va_end(ap);
1154}
1155#endif /* CONFIG_DEBUG */
1156
1157