/*
* Copyright (c) 2017 Apple Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this
* file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_LICENSE_HEADER_END@
*/
#include <set>
#include <string>
#include <sstream>
#include <iomanip> // std::setfill, std::setw
#include <pthread.h>
#include <mach/mach.h>
#include <dispatch/dispatch.h>
#include <Bom/Bom.h>
#include <Security/Security.h>
#include <Security/SecCodeSigner.h>
#include <CommonCrypto/CommonCrypto.h>
#include "Manifest.h"
#include "Diagnostics.h"
#include "FileUtils.h"
#include "BuilderUtils.h"
static dispatch_queue_t write_queue = dispatch_queue_create("com.apple.dyld.cache-builder.write", DISPATCH_QUEUE_CONCURRENT);
static dispatch_group_t build_group = dispatch_group_create();
dispatch_group_t buildGroup() {
return build_group;
}
void insertFileInBom(const std::string& path, BOMBom bom)
{
std::vector<std::string> components;
std::vector<std::string> processed_components;
std::stringstream ss(path);
std::string item;
while (std::getline(ss, item, '/')) {
if (!item.empty()) {
components.push_back(item);
}
}
std::string partialPath = ".";
std::string lastComponent = components.back();
components.pop_back();
BOMFSObject fso = BOMFSObjectNew(BOMDirectoryType);
BOMFSObjectSetFlags(fso, B_PATHONLY);
BOMFSObjectSetPathName(fso, ".", true);
BOMFSObjectSetShortName(fso, ".", true);
(void)BOMBomInsertFSObject(bom, fso, false);
BOMFSObjectFree(fso);
for (const auto& component : components) {
partialPath = partialPath + "/" + component;
fso = BOMFSObjectNew(BOMDirectoryType);
BOMFSObjectSetFlags(fso, B_PATHONLY);
BOMFSObjectSetPathName(fso, partialPath.c_str(), true);
BOMFSObjectSetShortName(fso, component.c_str(), true);
(void)BOMBomInsertFSObject(bom, fso, false);
BOMFSObjectFree(fso);
}
partialPath = partialPath + "/" + lastComponent;
fso = BOMFSObjectNew(BOMFileType);
BOMFSObjectSetFlags(fso, B_PATHONLY);
BOMFSObjectSetPathName(fso, partialPath.c_str(), true);
BOMFSObjectSetShortName(fso, lastComponent.c_str(), true);
(void)BOMBomInsertFSObject(bom, fso, false);
BOMFSObjectFree(fso);
}
void makeBoms(dyld3::Manifest& manifest, const std::string& masterDstRoot)
{
mkpath_np((masterDstRoot + "/Boms/").c_str(), 0755);
manifest.forEachConfiguration([&manifest, &masterDstRoot](const std::string& configName) {
auto config = manifest.configuration(configName);
std::vector<std::string> prodBomPaths;
std::vector<std::string> devBomPaths;
std::string runtimePath = "/System/Library/Caches/com.apple.dyld/";
if (manifest.platform() == dyld3::Platform::macOS) {
runtimePath = "/private/var/db/dyld/";
}
for (auto& arch : config.architectures) {
std::string cachePath = "dyld_shared_cache_" + arch.first;
prodBomPaths.push_back(cachePath);
if (manifest.platform() != dyld3::Platform::macOS) {
cachePath += ".development";
}
devBomPaths.push_back(cachePath);
char buffer[MAXPATHLEN];
sprintf(buffer, "%s/Boms/%s.prod.bom", masterDstRoot.c_str(), configName.c_str());
BOMBom bom = BOMBomNew(buffer);
for (auto& path : prodBomPaths) {
insertFileInBom(runtimePath + path, bom);
}
BOMBomFree(bom);
sprintf(buffer, "%s/Boms/%s.dev.bom", masterDstRoot.c_str(), configName.c_str());
bom = BOMBomNew(buffer);
for (auto& path : devBomPaths) {
insertFileInBom(runtimePath + path, bom);
}
BOMBomFree(bom);
sprintf(buffer, "%s/Boms/%s.full.bom", masterDstRoot.c_str(), configName.c_str());
bom = BOMBomNew(buffer);
for (auto& path : prodBomPaths) {
insertFileInBom(runtimePath + path, bom);
}
for (auto& path : devBomPaths) {
insertFileInBom(runtimePath + path, bom);
}
BOMBomFree(bom);
}
});
}
bool build(Diagnostics& diags, dyld3::Manifest& manifest, const std::string& masterDstRoot, bool dedupe, bool verbose,
bool skipWrites, bool agileChooseSHA256CdHash)
{
dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
dispatch_queue_t warningQueue = dispatch_queue_create("com.apple.dyld.cache-builder.warnings", DISPATCH_QUEUE_SERIAL);
std::vector<std::set<std::string>> dedupedCacheSets;
if (dedupe) {
manifest.forEachConfiguration([&manifest, &dedupedCacheSets](const std::string& configName) {
auto config = manifest.configuration(configName);
bool dupeFound = false;
for (auto& cacheSet : dedupedCacheSets) {
if (config == manifest.configuration(*cacheSet.begin())) {
cacheSet.insert(configName);
dupeFound = true;
break;
}
}
if (!dupeFound) {
std::set<std::string> temp;
temp.insert(configName);
dedupedCacheSets.push_back(temp);
}
});
} else {
manifest.forEachConfiguration([&manifest, &dedupedCacheSets](const std::string& configName) {
std::set<std::string> temp;
temp.insert(configName);
dedupedCacheSets.push_back(temp);
});
}
std::vector<dyld3::BuildQueueEntry> buildQueue;
for (auto& cacheSet : dedupedCacheSets) {
//FIXME we may want to consider moving to hashes of UUID sets
std::string setName;
for (auto& archName : cacheSet) {
if (!setName.empty()) {
setName += "|";
}
setName += archName;
}
std::stringstream fileNameStream;
std::array<uint8_t, CC_SHA1_DIGEST_LENGTH> digest = { 0 };
CC_SHA1(setName.c_str(), (unsigned int)setName.length(), &digest[0]);
fileNameStream << std::hex << std::uppercase << std::setfill('0');
for (int c : digest) {
fileNameStream << std::setw(2) << c;
}
std::string fileName(fileNameStream.str());
if (dedupe) {
for (auto& config : cacheSet) {
if (!skipWrites) {
int err = symlink(("DedupedConfigs/" + fileName).c_str(), (masterDstRoot + "/" + config).c_str());
if (err) {
diags.warning("Could not create symlink '%s' -> 'DedupedConfigs/%s' (%d)", config.c_str(), fileName.c_str(), err);
}
}
}
}
manifest.configuration(*cacheSet.begin()).forEachArchitecture([&masterDstRoot, &dedupe, &fileName, &setName, &manifest, &buildQueue, &cacheSet, verbose](const std::string& arch) {
std::string configPath;
std::string runtimePath = "/System/Library/Caches/com.apple.dyld/";
if (manifest.platform() == dyld3::Platform::macOS) {
runtimePath = "/private/var/db/dyld/";
}
if (dedupe) {
configPath = masterDstRoot + "/DedupedConfigs/" + fileName + runtimePath;
} else {
configPath = masterDstRoot + runtimePath;
}
if (manifest.platform() == dyld3::Platform::macOS) {
buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch, cacheSet, arch, false, setName + "/" + arch, verbose));
} else {
buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch + ".development", cacheSet, arch, false, setName + "/" + arch, verbose));
buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch, cacheSet, arch, true, setName + "/" + arch, verbose));
}
});
}
__block bool cacheBuildFailure = false;
__block std::set<std::string> warnings;
__block std::set<std::string> errors;
dispatch_sync(warningQueue, ^{
auto manifestWarnings = diags.warnings();
warnings.insert(manifestWarnings.begin(), manifestWarnings.end());
});
dispatch_apply(buildQueue.size(), queue, ^(size_t index) {
auto queueEntry = buildQueue[index];
pthread_setname_np(queueEntry.options.loggingPrefix.substr(0, MAXTHREADNAMESIZE - 1).c_str());
DyldSharedCache::CreateResults results = DyldSharedCache::create(queueEntry.options, queueEntry.dylibsForCache, queueEntry.otherDylibsAndBundles, queueEntry.mainExecutables);
dispatch_sync(warningQueue, ^{
warnings.insert(results.warnings.begin(), results.warnings.end());
bool chooseSecondCdHash = agileChooseSHA256CdHash;
if (agileChooseSHA256CdHash && !results.agileSignature) {
// Ignore this option for caches that are not signed agile (which is the majority).
chooseSecondCdHash = false;
}
for (const auto& configName : queueEntry.configNames) {
auto& configResults = manifest.configuration(configName).architecture(queueEntry.options.archName).results;
for (const auto& mh : results.evictions) {
auto parser = dyld3::MachOParser(mh);
configResults.exclude(&parser, "VM overflow, evicting");
}
configResults.warnings = results.warnings;
if (queueEntry.options.optimizeStubs) {
configResults.developmentCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst;
} else {
configResults.productionCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst;
}
}
});
if (!results.errorMessage.empty()) {
fprintf(stderr, "[%s] ERROR: %s\n", queueEntry.options.loggingPrefix.c_str(), results.errorMessage.c_str());
} else if (!skipWrites) {
dispatch_sync(write_queue, ^{
// save new cache file to disk and write new .map file
assert(results.cacheContent != nullptr);
mkpath_np(dirPath(queueEntry.outputPath).c_str(), 0755);
if (!safeSave(results.cacheContent, results.cacheLength, queueEntry.outputPath)) {
cacheBuildFailure = true;
fprintf(stderr, "[%s] ERROR: Could not write cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
} else {
fprintf(stderr, "[%s] Wrote cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
std::string mapStr = results.cacheContent->mapFile();
std::string outFileMap = queueEntry.outputPath + ".map";
safeSave(mapStr.c_str(), mapStr.size(), outFileMap);
}
// free created cache buffer
vm_deallocate(mach_task_self(), (vm_address_t)results.cacheContent, results.cacheLength);
});
} else {
fprintf(stderr, "[%s] Skipped writing cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
vm_deallocate(mach_task_self(), (vm_address_t)results.cacheContent, results.cacheLength);
}
});
// print any warnings
for (const std::string& warn : warnings) {
fprintf(stderr, "[WARNING] %s\n", warn.c_str());
}
int err = sync_volume_np(masterDstRoot.c_str(), SYNC_VOLUME_FULLSYNC | SYNC_VOLUME_WAIT);
if (err) {
fprintf(stderr, "Volume sync failed errnor=%d (%s)\n", err, strerror(err));
}
return !cacheBuildFailure;
}