#include <sys/systm.h>
#include <IOKit/IOWorkLoop.h>
#include <IOKit/IOCommandGate.h>
#include <IOKit/IOTimerEventSource.h>
#include <IOKit/IOPlatformExpert.h>
#include <IOKit/IOKitDebug.h>
#include <IOKit/IOTimeStamp.h>
#include <IOKit/pwr_mgt/RootDomain.h>
#include <IOKit/pwr_mgt/IOPMPrivate.h>
#include <IOKit/IOMessage.h>
#include <IOKit/IODeviceTreeSupport.h>
#include <IOKit/IOBSD.h>
#include <IOKit/IOKitKeysPrivate.h>
#include "RootDomainUserClient.h"
#include <IOKit/pwr_mgt/IOPowerConnection.h>
#include "IOPMPowerStateQueue.h"
#include <IOKit/IOBufferMemoryDescriptor.h>
#include <IOKit/AppleKeyStoreInterface.h>
#include <libkern/crypto/aes.h>
#include <sys/uio.h>
#include <sys/conf.h>
#include <sys/stat.h>
#include <sys/fcntl.h> // (FWRITE, ...)
#include <sys/sysctl.h>
#include <sys/kdebug.h>
#include <stdint.h>
#include <IOKit/IOHibernatePrivate.h>
#include <IOKit/IOPolledInterface.h>
#include <IOKit/IONVRAM.h>
#include "IOHibernateInternal.h"
#include <vm/vm_protos.h>
#include "IOKitKernelInternal.h"
#include <pexpert/device_tree.h>
#include <machine/pal_routines.h>
#include <machine/pal_hibernate.h>
#if defined(__i386__) || defined(__x86_64__)
#include <i386/tsc.h>
#include <i386/cpuid.h>
#include <vm/WKdm_new.h>
#elif defined(__arm64__)
#include <arm64/amcc_rorgn.h>
#endif
#include <san/kasan.h>
extern "C" addr64_t kvtophys(vm_offset_t va);
extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
#define DISABLE_TRIM 0
#define TRIM_DELAY 25000
extern unsigned int save_kdebug_enable;
extern uint32_t gIOHibernateState;
uint32_t gIOHibernateMode;
static char gIOHibernateBootSignature[256 + 1];
static char gIOHibernateFilename[MAXPATHLEN + 1];
uint32_t gIOHibernateCount;
static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
static uint32_t gIOHibernateFreeRatio = 0; uint32_t gIOHibernateFreeTime = 0 * 1000; static uint64_t gIOHibernateCompression = 0x80; boolean_t gIOHibernateStandbyDisabled;
static IODTNVRAM * gIOOptionsEntry;
static IORegistryEntry * gIOChosenEntry;
static const OSSymbol * gIOHibernateBootImageKey;
static const OSSymbol * gIOHibernateBootSignatureKey;
static const OSSymbol * gIOBridgeBootSessionUUIDKey;
#if defined(__i386__) || defined(__x86_64__)
static const OSSymbol * gIOHibernateRTCVariablesKey;
static const OSSymbol * gIOHibernateBoot0082Key;
static const OSSymbol * gIOHibernateBootNextKey;
static OSData * gIOHibernateBoot0082Data;
static OSData * gIOHibernateBootNextData;
static OSObject * gIOHibernateBootNextSave;
#endif
static IOLock * gFSLock;
uint32_t gFSState;
static thread_call_t gIOHibernateTrimCalloutEntry;
static IOPolledFileIOVars gFileVars;
static IOHibernateVars gIOHibernateVars;
static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
static hibernate_graphics_t _hibernateGraphics;
static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
static hibernate_statistics_t _hibernateStats;
static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
enum{
kFSIdle = 0,
kFSOpening = 2,
kFSOpened = 3,
kFSTimedOut = 4,
kFSTrimDelay = 5
};
static IOReturn IOHibernateDone(IOHibernateVars * vars);
static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
static void IOSetBootImageNVRAM(OSData * data);
static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
enum { kDefaultIOSize = 128 * 1024 };
enum { kVideoMapSize = 80 * 1024 * 1024 };
static IOReturn
IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
IOByteCount offset, addr64_t bytes, IOByteCount length)
{
addr64_t srcAddr = bytes;
IOByteCount remaining;
remaining = length = min(length, md->getLength() - offset);
while (remaining) { addr64_t dstAddr64;
IOByteCount dstLen;
dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
if (!dstAddr64) {
break;
}
if (dstLen > remaining) {
dstLen = remaining;
}
#if 1
bcopy_phys(srcAddr, dstAddr64, dstLen);
#else
copypv(srcAddr, dstAddr64, dstLen,
cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
#endif
srcAddr += dstLen;
offset += dstLen;
remaining -= dstLen;
}
assert(!remaining);
return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
}
static IOReturn
IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
IOByteCount offset, addr64_t bytes, IOByteCount length)
{
addr64_t dstAddr = bytes;
IOByteCount remaining;
remaining = length = min(length, md->getLength() - offset);
while (remaining) { addr64_t srcAddr64;
IOByteCount dstLen;
srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
if (!srcAddr64) {
break;
}
if (dstLen > remaining) {
dstLen = remaining;
}
#if 1
bcopy_phys(srcAddr64, dstAddr, dstLen);
#else
copypv(srcAddr, dstAddr64, dstLen,
cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
#endif
dstAddr += dstLen;
offset += dstLen;
remaining -= dstLen;
}
assert(!remaining);
return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
}
void
hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
{
count += ppnum;
if (count > UINT_MAX) {
panic("hibernate_set_page_state ppnum");
}
switch (kind) {
case kIOHibernatePageStateUnwiredSave:
for (; ppnum < count; ppnum++) {
hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
}
break;
case kIOHibernatePageStateWiredSave:
for (; ppnum < count; ppnum++) {
hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
hibernate_page_bitset(page_list_wired, FALSE, (uint32_t) ppnum);
}
break;
case kIOHibernatePageStateFree:
for (; ppnum < count; ppnum++) {
hibernate_page_bitset(page_list, TRUE, (uint32_t) ppnum);
hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
}
break;
default:
panic("hibernate_set_page_state");
}
}
static void
hibernate_set_descriptor_page_state(IOHibernateVars *vars,
IOMemoryDescriptor *descriptor,
uint32_t kind,
uint32_t *pageCount)
{
IOItemCount count;
addr64_t phys64;
IOByteCount segLen;
if (descriptor) {
for (count = 0;
(phys64 = descriptor->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
count += segLen) {
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
atop_64(phys64), atop_32(segLen),
kind);
*pageCount -= atop_32(segLen);
}
}
}
static vm_offset_t
hibernate_page_list_iterate(hibernate_page_list_t * list, ppnum_t * pPage)
{
uint32_t page = ((typeof(page)) * pPage);
uint32_t count;
hibernate_bitmap_t * bitmap;
while ((bitmap = hibernate_page_bitmap_pin(list, &page))) {
count = hibernate_page_bitmap_count(bitmap, TRUE, page);
if (!count) {
break;
}
page += count;
if (page <= bitmap->last_page) {
break;
}
}
*pPage = page;
if (bitmap) {
count = hibernate_page_bitmap_count(bitmap, FALSE, page);
} else {
count = 0;
}
return count;
}
IOReturn
IOHibernateSystemSleep(void)
{
IOReturn err;
OSData * nvramData;
OSObject * obj;
OSString * str;
OSNumber * num;
bool dsSSD, vmflush, swapPinned;
IOHibernateVars * vars;
uint64_t setFileSize = 0;
gIOHibernateState = kIOHibernateStateInactive;
gIOHibernateDebugFlags = 0;
if (kIOLogHibernate & gIOKitDebug) {
gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
}
if (IOService::getPMRootDomain()->getHibernateSettings(
&gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) {
if (kIOHibernateModeSleep & gIOHibernateMode) {
gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
| kIOHibernateModeDiscardCleanActive);
}
}
if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) {
if ((str = OSDynamicCast(OSString, obj))) {
strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
sizeof(gIOHibernateFilename));
}
obj->release();
}
if (!gIOHibernateMode || !gIOHibernateFilename[0]) {
return kIOReturnUnsupported;
}
HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
vars = IONew(IOHibernateVars, 1);
if (!vars) {
return kIOReturnNoMemory;
}
bzero(vars, sizeof(*vars));
IOLockLock(gFSLock);
if (!gIOHibernateTrimCalloutEntry) {
gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
}
IOHibernateSystemPostWakeTrim(NULL, NULL);
thread_call_cancel(gIOHibernateTrimCalloutEntry);
if (kFSIdle != gFSState) {
HIBLOG("hibernate file busy\n");
IOLockUnlock(gFSLock);
IODelete(vars, IOHibernateVars, 1);
return kIOReturnBusy;
}
gFSState = kFSOpening;
IOLockUnlock(gFSLock);
swapPinned = false;
do{
vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
HIBERNATION_SRC_BUFFER_SIZE, page_size);
vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
ptoa_64(gIOHibernateHandoffPageCount), page_size);
if (!vars->srcBuffer || !vars->handoffBuffer) {
err = kIOReturnNoMemory;
break;
}
if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) {
if ((num = OSDynamicCast(OSNumber, obj))) {
vars->fileMinSize = num->unsigned64BitValue();
}
obj->release();
}
if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) {
if ((num = OSDynamicCast(OSNumber, obj))) {
vars->fileMaxSize = num->unsigned64BitValue();
}
obj->release();
}
boolean_t encryptedswap = true;
uint32_t pageCount;
AbsoluteTime startTime, endTime;
uint64_t nsec;
bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
err = hibernate_alloc_page_lists(&vars->page_list,
&vars->page_list_wired,
&vars->page_list_pal);
if (KERN_SUCCESS != err) {
HIBLOG("%s err, hibernate_alloc_page_lists return 0x%x\n", __FUNCTION__, err);
break;
}
err = hibernate_pin_swap(TRUE);
if (KERN_SUCCESS != err) {
HIBLOG("%s error, hibernate_pin_swap return 0x%x\n", __FUNCTION__, err);
break;
}
swapPinned = true;
if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) {
hibernate_page_list_setall(vars->page_list,
vars->page_list_wired,
vars->page_list_pal,
true ,
vmflush ,
&pageCount);
PE_Video consoleInfo;
bzero(&consoleInfo, sizeof(consoleInfo));
IOService::getPlatform()->getConsoleInfo(&consoleInfo);
setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
+ vars->page_list->list_size
+ (consoleInfo.v_width * consoleInfo.v_height * 8);
enum { setFileRound = 1024 * 1024ULL };
setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
pageCount, (100ULL * gIOHibernateCompression) >> 8,
setFileSize, vars->fileMinSize);
if (!(kIOHibernateModeFileResize & gIOHibernateMode)
&& (setFileSize < vars->fileMinSize)) {
setFileSize = vars->fileMinSize;
}
}
vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
err = IOPolledFileOpen(gIOHibernateFilename,
(kIOPolledFileCreate | kIOPolledFileHibernate),
setFileSize, 0,
gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
&vars->fileVars, &nvramData,
&vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
if (KERN_SUCCESS != err) {
IOLockLock(gFSLock);
if (kFSOpening != gFSState) {
err = kIOReturnTimeout;
}
IOLockUnlock(gFSLock);
}
if (KERN_SUCCESS != err) {
HIBLOG("IOPolledFileOpen(%x)\n", err);
break;
}
IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
if (KERN_SUCCESS != err) {
break;
}
clock_get_uptime(&startTime);
err = hibernate_setup(gIOHibernateCurrentHeader,
vmflush,
vars->page_list, vars->page_list_wired, vars->page_list_pal);
clock_get_uptime(&endTime);
SUB_ABSOLUTETIME(&endTime, &startTime);
absolutetime_to_nanoseconds(endTime, &nsec);
boolean_t haveSwapPin, hibFileSSD;
haveSwapPin = vm_swap_files_pinned();
hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
err, nsec / 1000000ULL,
haveSwapPin, hibFileSSD);
if (KERN_SUCCESS != err) {
break;
}
gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
&& (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
if (dsSSD) {
gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
} else {
gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
}
#if defined(__i386__) || defined(__x86_64__)
if (vars->volumeCryptKeySize &&
(kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) {
uintptr_t smcVars[2];
smcVars[0] = vars->volumeCryptKeySize;
smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
bzero(smcVars, sizeof(smcVars));
}
#endif
if (encryptedswap || vars->volumeCryptKeySize) {
gIOHibernateMode ^= kIOHibernateModeEncrypt;
}
if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) {
vars->videoAllocSize = kVideoMapSize;
if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT)) {
vars->videoMapping = 0;
}
}
for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) {
vars->wiredCryptKey[i] = ((uint8_t) random());
}
for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++) {
vars->cryptKey[i] = ((uint8_t) random());
}
IOSetBootImageNVRAM(nvramData);
nvramData->release();
#if defined(__i386__) || defined(__x86_64__)
{
struct AppleRTCHibernateVars {
uint8_t signature[4];
uint32_t revision;
uint8_t booterSignature[20];
uint8_t wiredCryptKey[16];
};
AppleRTCHibernateVars rtcVars;
OSData * data;
rtcVars.signature[0] = 'A';
rtcVars.signature[1] = 'A';
rtcVars.signature[2] = 'P';
rtcVars.signature[3] = 'L';
rtcVars.revision = 1;
bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
if (gIOChosenEntry
&& (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
&& (sizeof(rtcVars.booterSignature) <= data->getLength())) {
bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
} else if (gIOHibernateBootSignature[0]) {
char c;
uint8_t value = 0;
uint32_t in, out, digits;
for (in = out = digits = 0;
(c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
in++) {
if ((c >= 'a') && (c <= 'f')) {
c -= 'a' - 10;
} else if ((c >= 'A') && (c <= 'F')) {
c -= 'A' - 10;
} else if ((c >= '0') && (c <= '9')) {
c -= '0';
} else {
if (c == '=') {
out = digits = value = 0;
}
continue;
}
value = ((uint8_t) ((value << 4) | c));
if (digits & 1) {
rtcVars.booterSignature[out++] = value;
if (out >= sizeof(rtcVars.booterSignature)) {
break;
}
}
digits++;
}
}
#if DEBUG || DEVELOPMENT
if (kIOLogHibernate & gIOKitDebug) {
IOKitKernelLogBuffer("H> rtc:",
&rtcVars, sizeof(rtcVars), &kprintf);
}
#endif
data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
if (data) {
if (gIOHibernateRTCVariablesKey) {
IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
}
data->release();
}
if (gIOChosenEntry && gIOOptionsEntry) {
data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
if (data) {
gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
}
if (!gIOHibernateBoot0082Data) {
OSData * fileData = NULL;
data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
if (data && data->getLength() >= 4) {
fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
}
if (data && (data->getLength() <= UINT16_MAX)) {
struct {
uint32_t Attributes;
uint16_t FilePathLength;
uint16_t Desc;
} loadOptionHeader;
loadOptionHeader.Attributes = 1;
loadOptionHeader.FilePathLength = ((uint16_t) data->getLength());
loadOptionHeader.Desc = 0;
if (fileData) {
loadOptionHeader.FilePathLength -= 4;
loadOptionHeader.FilePathLength += fileData->getLength();
}
gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
if (gIOHibernateBoot0082Data) {
gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
if (fileData) {
gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
gIOHibernateBoot0082Data->appendBytes(fileData);
} else {
gIOHibernateBoot0082Data->appendBytes(data);
}
}
}
}
if (!gIOHibernateBootNextData) {
uint16_t bits = 0x0082;
gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
}
#if DEBUG || DEVELOPMENT
if (kIOLogHibernate & gIOKitDebug) {
IOKitKernelLogBuffer("H> bootnext:",
gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
}
#endif
if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) {
gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
}
}
}
#endif
}while (false);
if (swapPinned) {
hibernate_pin_swap(FALSE);
}
IOLockLock(gFSLock);
if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) {
HIBLOG("hibernate file close due timeout\n");
err = kIOReturnTimeout;
}
if (kIOReturnSuccess == err) {
gFSState = kFSOpened;
gIOHibernateVars = *vars;
gFileVars = *vars->fileVars;
gFileVars.allocated = false;
gIOHibernateVars.fileVars = &gFileVars;
gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
gIOHibernateCurrentHeader->kernVirtSlide = vm_kernel_slide;
gIOHibernateState = kIOHibernateStateHibernating;
#if DEBUG || DEVELOPMENT
if (kIOLogHibernate & gIOKitDebug) {
OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
if (data) {
uintptr_t * smcVars = (typeof(smcVars))data->getBytesNoCopy();
IOKitKernelLogBuffer("H> smc:",
(const void *)smcVars[1], smcVars[0], &kprintf);
}
}
#endif
} else {
IOPolledFileIOVars * fileVars = vars->fileVars;
IOHibernateDone(vars);
IOPolledFileClose(&fileVars,
#if DISABLE_TRIM
0, NULL, 0, 0, 0);
#else
0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
#endif
gFSState = kFSIdle;
}
IOLockUnlock(gFSLock);
if (vars->fileVars) {
IODelete(vars->fileVars, IOPolledFileIOVars, 1);
}
IODelete(vars, IOHibernateVars, 1);
return err;
}
static void
IOSetBootImageNVRAM(OSData * data)
{
IORegistryEntry * regEntry;
if (!gIOOptionsEntry) {
regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
if (regEntry && !gIOOptionsEntry) {
regEntry->release();
}
}
if (gIOOptionsEntry && gIOHibernateBootImageKey) {
if (data) {
gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
#if DEBUG || DEVELOPMENT
if (kIOLogHibernate & gIOKitDebug) {
IOKitKernelLogBuffer("H> boot-image:",
data->getBytesNoCopy(), data->getLength(), &kprintf);
}
#endif
} else {
gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
#if __x86_64__
gIOOptionsEntry->sync();
#else
if (gIOHibernateState == kIOHibernateStateWakingFromHibernate) {
gIOOptionsEntry->sync();
}
#endif
}
}
}
static IOReturn
IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
{
IOHibernateImageHeader hdr;
IOItemCount count;
IOReturn err = kIOReturnSuccess;
int rc;
IOPolledFileExtent * fileExtents;
fileExtents = (typeof(fileExtents))vars->fileExtents->getBytesNoCopy();
memset(&hdr, 0, sizeof(IOHibernateImageHeader));
count = vars->fileExtents->getLength();
if (count > sizeof(hdr.fileExtentMap)) {
hdr.fileExtentMapSize = count;
count = sizeof(hdr.fileExtentMap);
} else {
hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
}
bcopy(fileExtents, &hdr.fileExtentMap[0], count);
if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) {
count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
rc = kern_write_file(vars->fileRef, vars->blockSize,
(caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
count, IO_SKIP_ENCRYPTION);
if (rc != 0) {
HIBLOG("kern_write_file returned %d\n", rc);
err = kIOReturnIOError;
goto exit;
}
}
hdr.signature = signature;
hdr.deviceBlockSize = vars->blockSize;
rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
if (rc != 0) {
HIBLOG("kern_write_file returned %d\n", rc);
err = kIOReturnIOError;
goto exit;
}
exit:
return err;
}
DECLARE_IOHIBERNATEPROGRESSALPHA
static void
ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
{
uint32_t rowBytes, pixelShift;
uint32_t x, y;
int32_t blob;
uint32_t alpha, color, result;
uint8_t * out, in;
uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
rowBytes = display->rowBytes;
pixelShift = display->depth >> 4;
if (pixelShift < 1) {
return;
}
screen += ((display->width
- kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+ (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
for (y = 0; y < kIOHibernateProgressHeight; y++) {
out = screen + y * rowBytes;
for (blob = 0; blob < kIOHibernateProgressCount; blob++) {
color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
for (x = 0; x < kIOHibernateProgressWidth; x++) {
alpha = gIOHibernateProgressAlpha[y][x];
result = color;
if (alpha) {
if (0xff != alpha) {
if (1 == pixelShift) {
in = *((uint16_t *)out) & 0x1f; in = ((uint8_t)(in << 3)) | ((uint8_t)(in >> 2));
} else {
in = *((uint32_t *)out) & 0xff; }
saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
}
if (1 == pixelShift) {
result >>= 3;
*((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); } else {
*((uint32_t *)out) = (result << 16) | (result << 8) | result; }
}
out += (1 << pixelShift);
}
out += (kIOHibernateProgressSpacing << pixelShift);
}
}
}
static void
ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
{
uint32_t rowBytes, pixelShift;
uint32_t x, y;
int32_t blob, lastBlob;
uint32_t alpha, in, color, result;
uint8_t * out;
uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
pixelShift = display->depth >> 4;
if (pixelShift < 1) {
return;
}
rowBytes = display->rowBytes;
screen += ((display->width
- kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+ (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
for (y = 0; y < kIOHibernateProgressHeight; y++) {
out = screen + y * rowBytes;
for (blob = firstBlob; blob <= lastBlob; blob++) {
color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
for (x = 0; x < kIOHibernateProgressWidth; x++) {
alpha = gIOHibernateProgressAlpha[y][x];
result = color;
if (alpha) {
if (0xff != alpha) {
in = display->progressSaveUnder[blob][saveindex[blob]++];
result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
}
if (1 == pixelShift) {
result >>= 3;
*((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); } else {
*((uint32_t *)out) = (result << 16) | (result << 8) | result; }
}
out += (1 << pixelShift);
}
out += (kIOHibernateProgressSpacing << pixelShift);
}
}
}
IOReturn
IOHibernateIOKitSleep(void)
{
IOReturn ret = kIOReturnSuccess;
IOLockLock(gFSLock);
if (kFSOpening == gFSState) {
gFSState = kFSTimedOut;
HIBLOG("hibernate file open timed out\n");
ret = kIOReturnTimeout;
}
IOLockUnlock(gFSLock);
return ret;
}
IOReturn
IOHibernateSystemHasSlept(void)
{
IOReturn ret = kIOReturnSuccess;
IOHibernateVars * vars = &gIOHibernateVars;
OSObject * obj = NULL;
OSData * data;
IOLockLock(gFSLock);
if ((kFSOpened != gFSState) && gIOHibernateMode) {
ret = kIOReturnTimeout;
}
IOLockUnlock(gFSLock);
if (kIOReturnSuccess != ret) {
return ret;
}
if (gIOHibernateMode) {
obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
}
vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
if (obj && !vars->previewBuffer) {
obj->release();
}
if (vars->previewBuffer && (vars->previewBuffer->getLength() > UINT_MAX)) {
OSSafeReleaseNULL(vars->previewBuffer);
}
vars->consoleMapping = NULL;
if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) {
vars->previewBuffer->release();
vars->previewBuffer = NULL;
}
if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
&& vars->previewBuffer
&& (data = OSDynamicCast(OSData,
IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) {
UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
if (kIOHibernatePreviewUpdates & flags) {
PE_Video consoleInfo;
hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
IOService::getPlatform()->getConsoleInfo(&consoleInfo);
graphicsInfo->width = (uint32_t) consoleInfo.v_width;
graphicsInfo->height = (uint32_t) consoleInfo.v_height;
graphicsInfo->rowBytes = (uint32_t) consoleInfo.v_rowBytes;
graphicsInfo->depth = (uint32_t) consoleInfo.v_depth;
vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
HIBPRINT("video %p %d %d %d\n",
vars->consoleMapping, graphicsInfo->depth,
graphicsInfo->width, graphicsInfo->height);
if (vars->consoleMapping) {
ProgressInit(graphicsInfo, vars->consoleMapping,
&graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
}
}
}
if (gIOOptionsEntry) {
#if __x86_64__
gIOOptionsEntry->sync();
#else
if (gIOHibernateMode) {
gIOOptionsEntry->sync();
}
#endif
}
return ret;
}
static const DeviceTreeNode *
MergeDeviceTree(const DeviceTreeNode * entry, IORegistryEntry * regEntry, OSSet * entriesToUpdate, vm_offset_t region_start, vm_size_t region_size)
{
DeviceTreeNodeProperty * prop;
const DeviceTreeNode * child;
IORegistryEntry * childRegEntry;
const char * nameProp;
unsigned int propLen, idx;
bool updateEntry = true;
if (!regEntry) {
updateEntry = false;
} else if (entriesToUpdate && !entriesToUpdate->containsObject(regEntry)) {
updateEntry = false;
}
prop = (DeviceTreeNodeProperty *) (entry + 1);
for (idx = 0; idx < entry->nProperties; idx++) {
if (updateEntry && (0 != strcmp("name", prop->name))) {
regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
}
prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
}
if (entriesToUpdate) {
entriesToUpdate->removeObject(regEntry);
if (entriesToUpdate->getCount() == 0) {
return NULL;
}
}
child = (const DeviceTreeNode *) prop;
for (idx = 0; idx < entry->nChildren; idx++) {
if (kSuccess != SecureDTGetPropertyRegion(child, "name", (void const **) &nameProp, &propLen,
region_start, region_size)) {
panic("no name");
}
childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
child = MergeDeviceTree(child, childRegEntry, entriesToUpdate, region_start, region_size);
if (!child) {
break;
}
}
return child;
}
IOReturn
IOHibernateSystemWake(void)
{
if (kFSOpened == gFSState) {
IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
IOHibernateDone(&gIOHibernateVars);
} else {
IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
}
if (gIOOptionsEntry && gIOHibernateBootImageKey) {
gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
}
return kIOReturnSuccess;
}
static IOReturn
IOHibernateDone(IOHibernateVars * vars)
{
IOReturn err;
OSData * data;
hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
if (vars->videoMapping) {
if (vars->videoMapSize) {
IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
}
if (vars->videoAllocSize) {
kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
}
}
if (vars->previewBuffer) {
vars->previewBuffer->release();
vars->previewBuffer = NULL;
}
if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
gIOHibernateCurrentHeader->options, 32);
} else {
IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
}
if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
&& (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) {
IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
&gIOHibernateGraphicsInfo->gfxStatus,
sizeof(gIOHibernateGraphicsInfo->gfxStatus));
} else {
IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
}
#if defined(__i386__) || defined(__x86_64__)
IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
if (gIOOptionsEntry) {
if (gIOHibernateRTCVariablesKey) {
if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
}
}
if (gIOHibernateBootNextKey) {
if (gIOHibernateBootNextSave) {
gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
gIOHibernateBootNextSave->release();
gIOHibernateBootNextSave = NULL;
} else {
gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
}
}
if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
gIOOptionsEntry->sync();
}
}
#endif
if (vars->srcBuffer) {
vars->srcBuffer->release();
}
bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
if (vars->handoffBuffer) {
if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
IOHibernateHandoff * handoff;
bool done = false;
for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
!done;
handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
uint8_t * __unused data = &handoff->data[0];
switch (handoff->type) {
case kIOHibernateHandoffTypeEnd:
done = true;
break;
case kIOHibernateHandoffTypeDeviceTree:
{
#if defined(__i386__) || defined(__x86_64__)
OSSet * entriesToUpdate = NULL;
#elif defined(__arm64__)
const char *mergePaths[] = {
kIODeviceTreePlane ":/chosen/boot-object-manifests",
kIODeviceTreePlane ":/chosen/secure-boot-hashes",
};
const size_t mergePathCount = sizeof(mergePaths) / sizeof(mergePaths[0]);
OSSet * entriesToUpdate = OSSet::withCapacity(mergePathCount);
for (size_t i = 0; i < mergePathCount; i++) {
IORegistryEntry *entry = IORegistryEntry::fromPath(mergePaths[i]);
if (!entry) {
panic("failed to find %s in IORegistry", mergePaths[i]);
}
entriesToUpdate->setObject(entry);
OSSafeReleaseNULL(entry);
}
#endif
MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot(), entriesToUpdate,
(vm_offset_t)data, (vm_size_t)handoff->bytecount);
OSSafeReleaseNULL(entriesToUpdate);
break;
}
case kIOHibernateHandoffTypeKeyStore:
#if defined(__i386__) || defined(__x86_64__)
{
IOBufferMemoryDescriptor *
md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
if (md) {
IOSetKeyStoreData(md);
}
}
#endif
break;
default:
done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
break;
}
}
#if defined(__i386__) || defined(__x86_64__)
if (vars->volumeCryptKeySize) {
IOBufferMemoryDescriptor *
bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
vars->volumeCryptKeySize, kIODirectionOutIn);
if (!bmd) {
panic("IOBufferMemoryDescriptor");
}
IOSetAPFSKeyStoreData(bmd);
bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
}
#endif
}
vars->handoffBuffer->release();
}
if (gIOChosenEntry
&& (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
&& (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
sizeof(gIOHibernateBridgeBootSessionUUIDString));
}
if (vars->hwEncrypt) {
err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
}
bzero(vars, sizeof(*vars));
gIOHibernateCount++;
return kIOReturnSuccess;
}
static void
IOHibernateSystemPostWakeTrim(void * p1, void * p2)
{
if (p1) {
IOLockLock(gFSLock);
}
if (kFSTrimDelay == gFSState) {
IOPolledFileIOVars * vars = &gFileVars;
IOPolledFileClose(&vars,
#if DISABLE_TRIM
0, NULL, 0, 0, 0);
#else
0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
#endif
gFSState = kFSIdle;
}
if (p1) {
IOLockUnlock(gFSLock);
}
}
IOReturn
IOHibernateSystemPostWake(bool now)
{
gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
IOSetBootImageNVRAM(NULL);
IOLockLock(gFSLock);
if (kFSTrimDelay == gFSState) {
thread_call_cancel(gIOHibernateTrimCalloutEntry);
IOHibernateSystemPostWakeTrim(NULL, NULL);
} else if (kFSOpened != gFSState) {
gFSState = kFSIdle;
} else {
gFSState = kFSTrimDelay;
if (now) {
thread_call_cancel(gIOHibernateTrimCalloutEntry);
IOHibernateSystemPostWakeTrim(NULL, NULL);
} else {
AbsoluteTime deadline;
clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
}
}
IOLockUnlock(gFSLock);
return kIOReturnSuccess;
}
uint32_t
IOHibernateWasScreenLocked(void)
{
uint32_t ret = 0;
if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) {
OSData *
data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
if (data) {
ret = ((uint32_t *)data->getBytesNoCopy())[0];
gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
}
} else {
gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
}
return ret;
}
SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
&gIOHibernateMode, 0, "");
SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
&_hibernateStats, hibernate_statistics_t, "");
SYSCTL_STRING(_kern_bridge, OID_AUTO, bootsessionuuid,
CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString), "");
SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
&_hibernateStats.graphicsReadyTime, 0, "");
SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
&_hibernateStats.wakeNotificationTime, 0, "");
SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
&_hibernateStats.lockScreenReadyTime, 0, "");
SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
&_hibernateStats.hidReadyTime, 0, "");
SYSCTL_UINT(_kern, OID_AUTO, hibernatecount,
CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
&gIOHibernateCount, 0, "");
static int
hibernate_set_preview SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
if (!IOTaskHasEntitlement(current_task(), kIOHibernateSetPreviewEntitlementKey)) {
return EPERM;
}
if ((req->newptr == USER_ADDR_NULL) || (!req->newlen)) {
IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewBufferKey);
return 0;
}
size_t rounded_size = round_page(req->newlen);
IOBufferMemoryDescriptor *md = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, rounded_size, page_size);
if (!md) {
return ENOMEM;
}
uint8_t *bytes = (uint8_t *)md->getBytesNoCopy();
int error = SYSCTL_IN(req, bytes, req->newlen);
if (error) {
md->release();
return error;
}
IOService::getPMRootDomain()->setProperty(kIOHibernatePreviewBufferKey, md);
md->release();
return 0;
}
SYSCTL_PROC(_kern, OID_AUTO, hibernatepreview,
CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, NULL, 0,
hibernate_set_preview, "S", "");
void
IOHibernateSystemInit(IOPMrootDomain * rootDomain)
{
gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
#if defined(__i386__) || defined(__x86_64__)
gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
#endif
OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
if (data) {
rootDomain->setProperty(kIOHibernateStateKey, data);
data->release();
}
if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
gIOHibernateMode = kIOHibernateModeOn;
} else {
gIOHibernateFilename[0] = 0;
}
sysctl_register_oid(&sysctl__kern_hibernatefile);
sysctl_register_oid(&sysctl__kern_bootsignature);
sysctl_register_oid(&sysctl__kern_hibernatemode);
sysctl_register_oid(&sysctl__kern_hibernatestatistics);
sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
sysctl_register_oid(&sysctl__kern_hibernatehidready);
sysctl_register_oid(&sysctl__kern_hibernatecount);
gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
if (gIOChosenEntry
&& (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
&& (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
}
gFSLock = IOLockAlloc();
gIOHibernateCount = 0;
}
static IOReturn
IOHibernatePolledFileWrite(IOHibernateVars * vars,
const uint8_t * bytes, IOByteCount size,
IOPolledFileCryptVars * cryptvars)
{
IOReturn err;
err = IOPolledFileWrite(vars->fileVars, bytes, size, cryptvars);
if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
err = kIOReturnAborted;
}
return err;
}
extern "C" uint32_t
hibernate_write_image(void)
{
IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
IOHibernateVars * vars = &gIOHibernateVars;
IOPolledFileExtent * fileExtents;
#if !defined(__arm64__)
_static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
#endif
uint32_t pageCount, pagesDone;
IOReturn err;
ppnum_t ppnum, page;
vm_offset_t count;
uint8_t * src;
uint8_t * data;
uint8_t * compressed;
uint8_t * scratch;
IOByteCount pageCompressedSize;
uint64_t compressedSize, uncompressedSize;
uint64_t image1Size = 0;
uint32_t bitmap_size;
bool iterDone, pollerOpen, needEncrypt;
int wkresult;
uint32_t tag;
uint32_t pageType;
uint32_t pageAndCount[2];
addr64_t phys64;
IOByteCount segLen;
uint32_t restore1Sum = 0, sum = 0, sum1 = 0, sum2 = 0;
uintptr_t hibernateBase;
uintptr_t hibernateEnd;
AbsoluteTime startTime, endTime;
AbsoluteTime allTime, compTime;
uint64_t compBytes;
uint64_t nsec;
uint64_t lastProgressStamp = 0;
uint64_t progressStamp;
uint32_t blob, lastBlob = (uint32_t) -1L;
uint32_t wiredPagesEncrypted;
uint32_t dirtyPagesEncrypted;
uint32_t wiredPagesClear;
uint32_t svPageCount;
uint32_t zvPageCount;
IOPolledFileCryptVars _cryptvars;
IOPolledFileCryptVars * cryptvars = NULL;
wiredPagesEncrypted = 0;
dirtyPagesEncrypted = 0;
wiredPagesClear = 0;
svPageCount = 0;
zvPageCount = 0;
if (!vars->fileVars
|| !vars->fileVars->pollers
|| !(kIOHibernateModeOn & gIOHibernateMode)) {
return kIOHibernatePostWriteSleep;
}
#if !defined(__arm64__)
if (kIOHibernateModeSleep & gIOHibernateMode) {
kdebug_enable = save_kdebug_enable;
}
#endif
pal_hib_write_hook();
KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
#if CRYPTO
if (kIOHibernateModeEncrypt & gIOHibernateMode) {
static const unsigned char first_iv[AES_BLOCK_SIZE]
= { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
cryptvars = &gIOHibernateCryptWakeContext;
bzero(cryptvars, sizeof(IOPolledFileCryptVars));
aes_encrypt_key(vars->cryptKey,
kIOHibernateAESKeySize,
&cryptvars->ctx.encrypt);
aes_decrypt_key(vars->cryptKey,
kIOHibernateAESKeySize,
&cryptvars->ctx.decrypt);
cryptvars = &_cryptvars;
bzero(cryptvars, sizeof(IOPolledFileCryptVars));
for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
}
aes_encrypt_key(vars->wiredCryptKey,
kIOHibernateAESKeySize,
&cryptvars->ctx.encrypt);
bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
}
#endif
hibernate_page_list_setall(vars->page_list,
vars->page_list_wired,
vars->page_list_pal,
false ,
((0 == (kIOHibernateModeSleep & gIOHibernateMode))
&& (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
&pageCount);
HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
#if 0
count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
for (page = 0; page < count; page++) {
HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
fileExtents[page].start, fileExtents[page].length,
fileExtents[page].start + fileExtents[page].length);
}
#endif
needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
AbsoluteTime_to_scalar(&compTime) = 0;
compBytes = 0;
clock_get_uptime(&allTime);
IOService::getPMRootDomain()->pmStatsRecordEvent(
kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
do{
compressedSize = 0;
uncompressedSize = 0;
svPageCount = 0;
zvPageCount = 0;
IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
ml_get_interrupts_enabled());
err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
!IOService::getPMRootDomain()->mustHibernate());
HIBLOG("IOHibernatePollerOpen(%x)\n", err);
pollerOpen = (kIOReturnSuccess == err);
if (!pollerOpen) {
break;
}
if (vars->volumeCryptKeySize) {
err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
vars->hwEncrypt = (kIOReturnSuccess == err);
bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
if (vars->hwEncrypt) {
header->options |= kIOHibernateOptionHWEncrypt;
}
}
count = vars->fileVars->fileExtents->getLength();
if (count > sizeof(header->fileExtentMap)) {
count -= sizeof(header->fileExtentMap);
err = IOHibernatePolledFileWrite(vars,
((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
if (kIOReturnSuccess != err) {
break;
}
}
for (count = 0;
(phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
count += segLen) {
for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64_ppnum(phys64) + pagesDone;
}
}
hibernateBase = HIB_BASE;
hibernateEnd = (segHIBB + segSizeHIB);
page = atop_32(kvtophys(hibernateBase));
count = atop_32(round_page(hibernateEnd) - hibernateBase);
uintptr_t entrypoint = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
uintptr_t stack = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
if ((count > UINT_MAX) || (entrypoint > UINT_MAX) || (stack > UINT_MAX)) {
panic("malformed kernel layout");
}
header->restore1CodePhysPage = (ppnum_t) page;
header->restore1CodeVirt = hibernateBase;
header->restore1PageCount = (uint32_t) count;
header->restore1CodeOffset = (uint32_t) entrypoint;
header->restore1StackOffset = (uint32_t) stack;
if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
}
src = (uint8_t *) trunc_page(hibernateBase);
for (page = 0; page < count; page++) {
if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
restore1Sum += hibernate_sum_page(src, (uint32_t) (header->restore1CodeVirt + page));
} else {
restore1Sum += 0x00000000;
}
src += page_size;
}
sum1 = restore1Sum;
src = (uint8_t *) trunc_page(hibernateBase);
count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
if (count) {
err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
if (kIOReturnSuccess != err) {
break;
}
}
err = IOHibernatePolledFileWrite(vars,
(uint8_t *) NULL,
&gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
cryptvars);
if (kIOReturnSuccess != err) {
break;
}
src = &gIOHibernateRestoreStackEnd[0];
count = round_page(hibernateEnd) - ((uintptr_t) src);
if (count) {
err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
if (kIOReturnSuccess != err) {
break;
}
}
if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
vars->fileVars->encryptEnd = UINT64_MAX;
HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
}
if (vars->previewBuffer) {
ppnum = 0;
count = 0;
do{
phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
pageAndCount[0] = atop_64_ppnum(phys64);
pageAndCount[1] = atop_64_ppnum(segLen);
err = IOHibernatePolledFileWrite(vars,
(const uint8_t *) &pageAndCount, sizeof(pageAndCount),
cryptvars);
if (kIOReturnSuccess != err) {
break;
}
count += segLen;
ppnum += sizeof(pageAndCount);
}while (phys64);
if (kIOReturnSuccess != err) {
break;
}
src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
count = (uint32_t) vars->previewBuffer->getLength();
header->previewPageListSize = ((uint32_t) ppnum);
header->previewSize = ((uint32_t) (count + ppnum));
for (page = 0; page < count; page += page_size) {
phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
sum1 += hibernate_sum_page(src + page, atop_64_ppnum(phys64));
}
if (kIOReturnSuccess != err) {
break;
}
err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
if (kIOReturnSuccess != err) {
break;
}
}
hibernate_set_descriptor_page_state(vars, IOPolledFileGetIOBuffer(vars->fileVars),
kIOHibernatePageStateFree, &pageCount);
hibernate_set_descriptor_page_state(vars, vars->srcBuffer,
kIOHibernatePageStateFree, &pageCount);
bitmap_size = vars->page_list_wired->list_size;
src = (uint8_t *) vars->page_list_wired;
err = IOHibernatePolledFileWrite(vars, src, bitmap_size, cryptvars);
if (kIOReturnSuccess != err) {
break;
}
hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
#if defined(__i386__) || defined(__x86_64__)
page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
page, count,
kIOHibernatePageStateFree);
pageCount -= count;
#elif defined(__arm64__)
extern unsigned long gPhysBase, gPhysSize;
for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
page = segInfo->segments[i].physPage;
count = segInfo->segments[i].pageCount;
uint64_t physAddr = ptoa_64(page);
uint64_t size = ptoa_64(count);
if (size &&
(physAddr >= gPhysBase) &&
(physAddr + size <= gPhysBase + gPhysSize)) {
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
page, count,
kIOHibernatePageStateFree);
pageCount -= count;
}
}
#else
#error unimplemented
#endif
hibernate_set_descriptor_page_state(vars, vars->previewBuffer,
kIOHibernatePageStateFree, &pageCount);
hibernate_set_descriptor_page_state(vars, vars->handoffBuffer,
kIOHibernatePageStateFree, &pageCount);
#if KASAN
vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
hibernate_set_page_state(vars->page_list, vars->page_list_wired,
atop_64(shadow_pnext),
shadow_pages_free,
kIOHibernatePageStateFree);
#endif
src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
compressed = src + page_size;
scratch = compressed + page_size;
pagesDone = 0;
lastBlob = 0;
HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
bitmap_size, header->previewSize,
pageCount, vars->fileVars->position);
enum
{
kWired = 0x02,
kEncrypt = 0x01,
kWiredEncrypt = kWired | kEncrypt,
kWiredClear = kWired,
kUnwiredEncrypt = kEncrypt
};
#if defined(__i386__) || defined(__x86_64__)
bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
#else
static const bool cpuAES = true;
#endif
for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
if (kUnwiredEncrypt == pageType) {
if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
vars->fileVars->encryptEnd = UINT64_MAX;
HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
}
bcopy(&cryptvars->aes_iv[0],
&gIOHibernateCryptWakeContext.aes_iv[0],
sizeof(cryptvars->aes_iv));
cryptvars = &gIOHibernateCryptWakeContext;
}
for (iterDone = false, ppnum = 0; !iterDone;) {
if (cpuAES && (pageType == kWiredClear)) {
count = 0;
} else {
count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
&ppnum);
if (count > UINT_MAX) {
count = UINT_MAX;
}
}
iterDone = !count;
if (!cpuAES) {
if (count && (kWired & pageType) && needEncrypt) {
uint32_t checkIndex;
for (checkIndex = 0;
(checkIndex < count)
&& (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(((ppnum_t)(ppnum + checkIndex))));
checkIndex++) {
}
if (!checkIndex) {
ppnum++;
continue;
}
count = checkIndex;
}
}
switch (pageType) {
case kWiredEncrypt: wiredPagesEncrypted += count; break;
case kWiredClear: wiredPagesClear += count; break;
case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
}
if (iterDone && (kWiredEncrypt == pageType)) {
} else {
pageAndCount[0] = (uint32_t) ppnum;
pageAndCount[1] = (uint32_t) count;
err = IOHibernatePolledFileWrite(vars,
(const uint8_t *) &pageAndCount, sizeof(pageAndCount),
cryptvars);
if (kIOReturnSuccess != err) {
break;
}
}
for (page = ppnum; page < (ppnum + count); page++) {
err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
if (err) {
HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
break;
}
sum = hibernate_sum_page(src, (uint32_t) page);
if (kWired & pageType) {
sum1 += sum;
} else {
sum2 += sum;
}
clock_get_uptime(&startTime);
wkresult = WKdm_compress_new((const WK_word*) src,
(WK_word*) compressed,
(WK_word*) scratch,
(uint32_t) (page_size - 4));
clock_get_uptime(&endTime);
ADD_ABSOLUTETIME(&compTime, &endTime);
SUB_ABSOLUTETIME(&compTime, &startTime);
compBytes += page_size;
pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
if (pageCompressedSize == 0) {
pageCompressedSize = 4;
data = src;
if (*(uint32_t *)src) {
svPageCount++;
} else {
zvPageCount++;
}
} else {
if (pageCompressedSize != page_size) {
data = compressed;
} else {
data = src;
}
}
assert(pageCompressedSize <= page_size);
tag = ((uint32_t) pageCompressedSize) | kIOHibernateTagSignature;
err = IOHibernatePolledFileWrite(vars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
if (kIOReturnSuccess != err) {
break;
}
err = IOHibernatePolledFileWrite(vars, data, (pageCompressedSize + 3) & ~3, cryptvars);
if (kIOReturnSuccess != err) {
break;
}
compressedSize += pageCompressedSize;
uncompressedSize += page_size;
pagesDone++;
if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
if (blob != lastBlob) {
ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
lastBlob = blob;
}
}
if (0 == (8191 & pagesDone)) {
clock_get_uptime(&endTime);
SUB_ABSOLUTETIME(&endTime, &allTime);
absolutetime_to_nanoseconds(endTime, &nsec);
progressStamp = nsec / 750000000ULL;
if (progressStamp != lastProgressStamp) {
lastProgressStamp = progressStamp;
HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
}
}
}
if (kIOReturnSuccess != err) {
break;
}
ppnum = page;
}
if (kIOReturnSuccess != err) {
break;
}
if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
}
if (kWiredEncrypt != pageType) {
err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
if (kIOReturnSuccess != err) {
break;
}
}
if (kWiredClear == pageType) {
header->encryptStart = vars->fileVars->encryptStart;
header->encryptEnd = vars->fileVars->encryptEnd;
image1Size = vars->fileVars->position;
HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
image1Size, header->encryptStart, header->encryptEnd);
}
}
if (kIOReturnSuccess != err) {
if (kIOReturnOverrun == err) {
gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
}
header->imageSize = vars->fileVars->position;
break;
}
header->imageSize = vars->fileVars->position;
header->image1Size = image1Size;
header->bitmapSize = bitmap_size;
header->pageCount = pageCount;
header->restore1Sum = restore1Sum;
header->image1Sum = sum1;
header->image2Sum = sum2;
header->sleepTime = gIOLastSleepTime.tv_sec;
header->compression = ((uint32_t)((compressedSize << 8) / uncompressedSize));
gIOHibernateCompression = header->compression;
count = vars->fileVars->fileExtents->getLength();
if (count > sizeof(header->fileExtentMap)) {
header->fileExtentMapSize = ((uint32_t) count);
count = sizeof(header->fileExtentMap);
} else {
header->fileExtentMapSize = sizeof(header->fileExtentMap);
}
bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
header->deviceBase = vars->fileVars->block0;
header->deviceBlockSize = vars->fileVars->blockSize;
header->lastHibAbsTime = mach_absolute_time();
header->lastHibContTime = mach_continuous_time();
IOPolledFileSeek(vars->fileVars, 0);
err = IOHibernatePolledFileWrite(vars,
(uint8_t *) header, sizeof(IOHibernateImageHeader),
cryptvars);
if (kIOReturnSuccess != err) {
break;
}
err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
}while (false);
clock_get_uptime(&endTime);
IOService::getPMRootDomain()->pmStatsRecordEvent(
kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
SUB_ABSOLUTETIME(&endTime, &allTime);
absolutetime_to_nanoseconds(endTime, &nsec);
HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
absolutetime_to_nanoseconds(compTime, &nsec);
HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
compBytes,
nsec / 1000000ULL,
nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
vars->fileVars->cryptBytes,
nsec / 1000000ULL,
nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%)\n",
header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
uncompressedSize, atop_32(uncompressedSize), compressedSize,
uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0);
HIBLOG("\nsum1 %x, sum2 %x\n", sum1, sum2);
HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
if (pollerOpen) {
IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
}
if (vars->consoleMapping) {
ProgressUpdate(gIOHibernateGraphicsInfo,
vars->consoleMapping, 0, kIOHibernateProgressCount);
}
HIBLOG("hibernate_write_image done(%x)\n", err);
gIOHibernateState = kIOHibernateStateInactive;
KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
wiredPagesClear, dirtyPagesEncrypted);
#if defined(__arm64__)
if (kIOReturnSuccess == err) {
return kIOHibernatePostWriteHalt;
} else {
panic("hibernate_write_image encountered error 0x%x", err);
}
#else
if (kIOReturnSuccess == err) {
if (kIOHibernateModeSleep & gIOHibernateMode) {
return kIOHibernatePostWriteSleep;
} else if (kIOHibernateModeRestart & gIOHibernateMode) {
return kIOHibernatePostWriteRestart;
} else {
return kIOHibernatePostWriteHalt;
}
} else if (kIOReturnAborted == err) {
return kIOHibernatePostWriteWake;
} else {
return kIOHibernatePostWriteSleep;
}
#endif
}
extern "C" void
hibernate_machine_init(void)
{
IOReturn err;
uint32_t sum;
uint32_t pagesDone;
uint32_t pagesRead = 0;
AbsoluteTime startTime, compTime;
AbsoluteTime allTime, endTime;
AbsoluteTime startIOTime, endIOTime;
uint64_t nsec, nsecIO;
uint64_t compBytes;
uint64_t lastProgressStamp = 0;
uint64_t progressStamp;
IOPolledFileCryptVars * cryptvars = NULL;
IOHibernateVars * vars = &gIOHibernateVars;
bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
if (!vars->fileVars || !vars->fileVars->pollers) {
return;
}
sum = gIOHibernateCurrentHeader->actualImage1Sum;
pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
HIBLOG("regular wake\n");
return;
}
HIBPRINT("diag %x %x %x %x\n",
gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
#if defined(__i386__) || defined(__x86_64__)
#define t40ms(x) ((uint32_t)((tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)))
#else
#define t40ms(x) x
#endif
#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
tStat(booterStart, booterStart);
gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
tStat(booterDuration0, booterTime0);
tStat(booterDuration1, booterTime1);
tStat(booterDuration2, booterTime2);
tStat(booterDuration, booterTime);
tStat(booterConnectDisplayDuration, connectDisplayTime);
tStat(booterSplashDuration, splashTime);
tStat(trampolineDuration, trampolineTime);
gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
gIOHibernateStats->image1Pages = pagesDone;
KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
gIOHibernateStats->trampolineDuration);
HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
gIOHibernateStats->booterStart,
gIOHibernateStats->smcStart,
gIOHibernateStats->booterDuration0,
gIOHibernateStats->booterDuration1,
gIOHibernateStats->booterDuration2,
gIOHibernateStats->booterDuration,
gIOHibernateStats->booterConnectDisplayDuration,
gIOHibernateStats->booterSplashDuration,
gIOHibernateStats->trampolineDuration);
HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
&& (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) {
hibernate_page_list_discard(vars->page_list);
}
if (vars->hwEncrypt) {
cryptvars = NULL;
} else {
cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : NULL;
}
if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) {
panic("handoff overflow");
}
IOHibernateHandoff * handoff;
bool done = false;
bool foundCryptData = false;
bool foundVolumeEncryptData = false;
const uint8_t * handoffStart = (const uint8_t*)vars->handoffBuffer->getBytesNoCopy();
const uint8_t * handoffEnd = handoffStart + vars->handoffBuffer->getLength();
for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
!done;
handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
if (((uint8_t*)handoff < handoffStart) ||
(&handoff->data[handoff->bytecount] > handoffEnd)) {
panic("handoff out of range");
}
uint8_t * data = &handoff->data[0];
switch (handoff->type) {
case kIOHibernateHandoffTypeEnd:
done = true;
break;
case kIOHibernateHandoffTypeGraphicsInfo:
if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) {
bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
}
break;
case kIOHibernateHandoffTypeCryptVars:
if (cryptvars) {
hibernate_cryptwakevars_t *
wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
if (handoff->bytecount == sizeof(*wakevars)) {
bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
} else {
panic("kIOHibernateHandoffTypeCryptVars(%d)", handoff->bytecount);
}
}
foundCryptData = true;
bzero(data, handoff->bytecount);
break;
case kIOHibernateHandoffTypeVolumeCryptKey:
if (handoff->bytecount == vars->volumeCryptKeySize) {
bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
foundVolumeEncryptData = true;
} else {
panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
}
break;
#if defined(__i386__) || defined(__x86_64__)
case kIOHibernateHandoffTypeMemoryMap:
clock_get_uptime(&allTime);
hibernate_newruntime_map(data, handoff->bytecount,
gIOHibernateCurrentHeader->systemTableOffset);
clock_get_uptime(&endTime);
SUB_ABSOLUTETIME(&endTime, &allTime);
absolutetime_to_nanoseconds(endTime, &nsec);
HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
break;
case kIOHibernateHandoffTypeDeviceTree:
{
}
break;
#endif
default:
done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
break;
}
}
if (vars->hwEncrypt && !foundVolumeEncryptData) {
panic("no volumeCryptKey");
} else if (cryptvars && !foundCryptData) {
panic("hibernate handoff");
}
HIBPRINT("video 0x%llx %d %d %d status %x\n",
gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) {
vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
* gIOHibernateGraphicsInfo->rowBytes);
if (vars->videoMapSize > vars->videoAllocSize) {
vars->videoMapSize = 0;
} else {
IOMapPages(kernel_map,
vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
vars->videoMapSize, kIOMapInhibitCache );
}
}
if (vars->videoMapSize) {
ProgressUpdate(gIOHibernateGraphicsInfo,
(uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
}
uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
uint8_t * compressed = src + page_size;
uint8_t * scratch = compressed + page_size;
uint32_t decoOffset;
clock_get_uptime(&allTime);
AbsoluteTime_to_scalar(&compTime) = 0;
compBytes = 0;
HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
clock_get_uptime(&startIOTime);
endTime = startIOTime;
SUB_ABSOLUTETIME(&endTime, &allTime);
absolutetime_to_nanoseconds(endTime, &nsec);
HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
if (vars->hwEncrypt) {
err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
&vars->volumeCryptKey[0], vars->volumeCryptKeySize);
HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
if (kIOReturnSuccess != err) {
panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
}
cryptvars = NULL;
}
IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
vars->fileVars->bufferHalf = 0;
vars->fileVars->bufferLimit = 0;
vars->fileVars->lastRead = 0;
vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
vars->fileVars->cryptBytes = 0;
AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
err = IOPolledFileRead(vars->fileVars, NULL, 0, cryptvars);
if (kIOReturnSuccess != err) {
panic("Hibernate restore error %x", err);
}
vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
HIBLOG("hibernate_machine_init reading\n");
uint32_t * header = (uint32_t *) src;
sum = 0;
while (kIOReturnSuccess == err) {
unsigned int count;
unsigned int page;
uint32_t tag;
vm_offset_t compressedSize;
ppnum_t ppnum;
err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
if (kIOReturnSuccess != err) {
panic("Hibernate restore error %x", err);
}
ppnum = header[0];
count = header[1];
if (!count) {
break;
}
for (page = 0; page < count; page++) {
err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
if (kIOReturnSuccess != err) {
panic("Hibernate restore error %x", err);
}
compressedSize = kIOHibernateTagLength & tag;
if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) {
err = kIOReturnIPCError;
panic("Hibernate restore error %x", err);
}
err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
if (kIOReturnSuccess != err) {
panic("Hibernate restore error %x", err);
}
if (compressedSize < page_size) {
decoOffset = ((uint32_t) page_size);
clock_get_uptime(&startTime);
if (compressedSize == 4) {
int i;
uint32_t *s, *d;
s = (uint32_t *)src;
d = (uint32_t *)(uintptr_t)compressed;
for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) {
*d++ = *s;
}
} else {
pal_hib_decompress_page(src, compressed, scratch, ((unsigned int) compressedSize));
}
clock_get_uptime(&endTime);
ADD_ABSOLUTETIME(&compTime, &endTime);
SUB_ABSOLUTETIME(&compTime, &startTime);
compBytes += page_size;
} else {
decoOffset = 0;
}
sum += hibernate_sum_page((src + decoOffset), ((uint32_t) ppnum));
err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
if (err) {
HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
panic("Hibernate restore error %x", err);
}
ppnum++;
pagesDone++;
pagesRead++;
if (0 == (8191 & pagesDone)) {
clock_get_uptime(&endTime);
SUB_ABSOLUTETIME(&endTime, &allTime);
absolutetime_to_nanoseconds(endTime, &nsec);
progressStamp = nsec / 750000000ULL;
if (progressStamp != lastProgressStamp) {
lastProgressStamp = progressStamp;
HIBPRINT("pages %d (%d%%)\n", pagesDone,
(100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
}
}
}
}
if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) {
err = kIOReturnLockedRead;
}
if (kIOReturnSuccess != err) {
panic("Hibernate restore error %x", err);
}
gIOHibernateCurrentHeader->actualImage2Sum = sum;
gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
clock_get_uptime(&endIOTime);
err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
clock_get_uptime(&endTime);
IOService::getPMRootDomain()->pmStatsRecordEvent(
kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
IOService::getPMRootDomain()->pmStatsRecordEvent(
kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
SUB_ABSOLUTETIME(&endTime, &allTime);
absolutetime_to_nanoseconds(endTime, &nsec);
SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
absolutetime_to_nanoseconds(endIOTime, &nsecIO);
gIOHibernateStats->kernelImageReadDuration = ((uint32_t) (nsec / 1000000ULL));
gIOHibernateStats->imagePages = pagesDone;
HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
absolutetime_to_nanoseconds(compTime, &nsec);
HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
compBytes,
nsec / 1000000ULL,
nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
vars->fileVars->cryptBytes,
nsec / 1000000ULL,
nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
}
void
IOHibernateSetWakeCapabilities(uint32_t capability)
{
if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
gIOHibernateStats->wakeCapability = capability;
if (kIOPMSystemCapabilityGraphics & capability) {
vm_compressor_do_warmup();
}
}
}
void
IOHibernateSystemRestart(void)
{
static uint8_t noteStore[32] __attribute__((aligned(32)));
IORegistryEntry * regEntry;
const OSSymbol * sym;
OSData * noteProp;
OSData * data;
uintptr_t * smcVars;
uint8_t * smcBytes;
size_t len;
addr64_t element;
data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
if (!data) {
return;
}
smcVars = (typeof(smcVars))data->getBytesNoCopy();
smcBytes = (typeof(smcBytes))smcVars[1];
len = smcVars[0];
if (len > sizeof(noteStore)) {
len = sizeof(noteStore);
}
noteProp = OSData::withCapacity(3 * sizeof(element));
if (!noteProp) {
return;
}
element = len;
noteProp->appendBytes(&element, sizeof(element));
element = crc32(0, smcBytes, len);
noteProp->appendBytes(&element, sizeof(element));
bcopy(smcBytes, noteStore, len);
element = (addr64_t) ¬eStore[0];
element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
noteProp->appendBytes(&element, sizeof(element));
if (!gIOOptionsEntry) {
regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
if (regEntry && !gIOOptionsEntry) {
regEntry->release();
}
}
sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
if (gIOOptionsEntry && sym) {
gIOOptionsEntry->setProperty(sym, noteProp);
}
if (noteProp) {
noteProp->release();
}
if (sym) {
sym->release();
}
}