/*
* Copyright (c) 2017 Apple Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this
* file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_LICENSE_HEADER_END@
*/
extern "C" {
#include <Bom/Bom.h>
#include <Metabom/MBTypes.h>
#include <Metabom/MBEntry.h>
#include <Metabom/MBMetabom.h>
#include <Metabom/MBIterator.h>
};
#include <algorithm>
#include <CommonCrypto/CommonDigest.h>
#include <CommonCrypto/CommonDigestSPI.h>
#include <Foundation/Foundation.h>
#include "MachOFileAbstraction.hpp"
#include "FileAbstraction.hpp"
#include "Trie.hpp"
#include "FileUtils.h"
#include "StringUtils.h"
#include <mach-o/loader.h>
#include <mach-o/fat.h>
#include <array>
#include <vector>
#include "Manifest.h"
namespace {
//FIXME this should be in a class
static inline NSString* cppToObjStr(const std::string& str) { return [NSString stringWithUTF8String:str.c_str()]; }
template <class Set1, class Set2>
inline bool is_disjoint(const Set1& set1, const Set2& set2)
{
if (set1.empty() || set2.empty())
return true;
typename Set1::const_iterator it1 = set1.begin(), it1End = set1.end();
typename Set2::const_iterator it2 = set2.begin(), it2End = set2.end();
if (*it1 > *set2.rbegin() || *it2 > *set1.rbegin())
return true;
while (it1 != it1End && it2 != it2End) {
if (*it1 == *it2)
return false;
if (*it1 < *it2) {
it1++;
} else {
it2++;
}
}
return true;
}
//hACK: If we declare this in manifest
static NSDictionary* gManifestDict;
} /* Anonymous namespace */
namespace dyld3 {
void Manifest::Results::exclude(MachOParser* parser, const std::string& reason)
{
auto dylibUUID = parser->uuid();
dylibs[dylibUUID].uuid = dylibUUID;
dylibs[dylibUUID].installname = parser->installName();
dylibs[dylibUUID].included = false;
dylibs[dylibUUID].exclusionInfo = reason;
}
void Manifest::Results::exclude(Manifest& manifest, const UUID& uuid, const std::string& reason)
{
auto parser = manifest.parserForUUID(uuid);
dylibs[uuid].uuid = uuid;
dylibs[uuid].installname = parser.installName();
dylibs[uuid].included = false;
dylibs[uuid].exclusionInfo = reason;
}
Manifest::CacheImageInfo& Manifest::Results::dylibForInstallname(const std::string& installname)
{
auto i = find_if(dylibs.begin(), dylibs.end(), [&installname](std::pair<UUID, CacheImageInfo> d) { return d.second.installname == installname; });
assert(i != dylibs.end());
return i->second;
}
bool Manifest::Architecture::operator==(const Architecture& O) const
{
for (auto& dylib : results.dylibs) {
if (dylib.second.included) {
auto Odylib = O.results.dylibs.find(dylib.first);
if (Odylib == O.results.dylibs.end()
|| Odylib->second.included == false
|| Odylib->second.uuid != dylib.second.uuid)
return false;
}
}
for (const auto& Odylib : O.results.dylibs) {
if (Odylib.second.included) {
auto dylib = results.dylibs.find(Odylib.first);
if (dylib == results.dylibs.end()
|| dylib->second.included == false
|| dylib->second.uuid != Odylib.second.uuid)
return false;
}
}
for (auto& bundle : results.bundles) {
if (bundle.second.included) {
auto Obundle = O.results.bundles.find(bundle.first);
if (Obundle == O.results.bundles.end()
|| Obundle->second.included == false
|| Obundle->second.uuid != bundle.second.uuid)
return false;
}
}
for (const auto& Obundle : O.results.bundles) {
if (Obundle.second.included) {
auto bundle = results.bundles.find(Obundle.first);
if (bundle == results.bundles.end()
|| bundle->second.included == false
|| bundle->second.uuid != Obundle.second.uuid)
return false;
}
}
for (auto& executable : results.executables) {
if (executable.second.included) {
auto Oexecutable = O.results.executables.find(executable.first);
if (Oexecutable == O.results.executables.end()
|| Oexecutable->second.included == false
|| Oexecutable->second.uuid != executable.second.uuid)
return false;
}
}
for (const auto& Oexecutable : O.results.executables) {
if (Oexecutable.second.included) {
auto executable = results.executables.find(Oexecutable.first);
if (executable == results.executables.end()
|| executable->second.included == false
|| executable->second.uuid != Oexecutable.second.uuid)
return false;
}
}
return true;
}
bool Manifest::Configuration::operator==(const Configuration& O) const
{
return architectures == O.architectures;
}
bool Manifest::Configuration::operator!=(const Configuration& other) const { return !(*this == other); }
const Manifest::Architecture& Manifest::Configuration::architecture(const std::string& architecture) const
{
assert(architectures.find(architecture) != architectures.end());
return architectures.find(architecture)->second;
}
void Manifest::Configuration::forEachArchitecture(std::function<void(const std::string& archName)> lambda) const
{
for (const auto& architecutre : architectures) {
lambda(architecutre.first);
}
}
bool Manifest::Architecture::operator!=(const Architecture& other) const { return !(*this == other); }
const std::map<std::string, Manifest::Project>& Manifest::projects()
{
return _projects;
}
const Manifest::Configuration& Manifest::configuration(const std::string& configuration) const
{
assert(_configurations.find(configuration) != _configurations.end());
return _configurations.find(configuration)->second;
}
void Manifest::forEachConfiguration(std::function<void(const std::string& configName)> lambda) const
{
for (const auto& configuration : _configurations) {
lambda(configuration.first);
}
}
void Manifest::addProjectSource(const std::string& project, const std::string& source, bool first)
{
auto& sources = _projects[project].sources;
if (std::find(sources.begin(), sources.end(), source) == sources.end()) {
if (first) {
sources.insert(sources.begin(), source);
} else {
sources.push_back(source);
}
}
}
const std::string Manifest::projectPath(const std::string& projectName)
{
auto project = _projects.find(projectName);
if (project == _projects.end())
return "";
if (project->second.sources.size() == 0)
return "";
return project->second.sources[0];
}
const bool Manifest::empty(void)
{
for (const auto& configuration : _configurations) {
if (configuration.second.architectures.size() != 0)
return false;
}
return true;
}
const std::string Manifest::dylibOrderFile() const { return _dylibOrderFile; };
void Manifest::setDylibOrderFile(const std::string& dylibOrderFile) { _dylibOrderFile = dylibOrderFile; };
const std::string Manifest::dirtyDataOrderFile() const { return _dirtyDataOrderFile; };
void Manifest::setDirtyDataOrderFile(const std::string& dirtyDataOrderFile) { _dirtyDataOrderFile = dirtyDataOrderFile; };
const std::string Manifest::metabomFile() const { return _metabomFile; };
void Manifest::setMetabomFile(const std::string& metabomFile) { _metabomFile = metabomFile; };
const Platform Manifest::platform() const { return _platform; };
void Manifest::setPlatform(const Platform platform) { _platform = platform; };
const std::string& Manifest::build() const { return _build; };
void Manifest::setBuild(const std::string& build) { _build = build; };
const uint32_t Manifest::version() const { return _manifestVersion; };
void Manifest::setVersion(const uint32_t manifestVersion) { _manifestVersion = manifestVersion; };
BuildQueueEntry Manifest::makeQueueEntry(const std::string& outputPath, const std::set<std::string>& configs, const std::string& arch, bool optimizeStubs, const std::string& prefix, bool verbose)
{
dyld3::BuildQueueEntry retval;
DyldSharedCache::CreateOptions options;
options.archName = arch;
options.platform = platform();
options.excludeLocalSymbols = true;
options.optimizeStubs = optimizeStubs;
options.optimizeObjC = true;
options.codeSigningDigestMode = (platform() == dyld3::Platform::watchOS) ?
DyldSharedCache::Agile : DyldSharedCache::SHA256only;
options.dylibsRemovedDuringMastering = true;
options.inodesAreSameAsRuntime = false;
options.cacheSupportsASLR = true;
options.forSimulator = false;
options.verbose = verbose;
options.evictLeafDylibsOnOverflow = true;
options.loggingPrefix = prefix;
options.pathPrefixes = { "" };
options.dylibOrdering = loadOrderFile(_dylibOrderFile);
options.dirtyDataSegmentOrdering = loadOrderFile(_dirtyDataOrderFile);
dyld3::BuildQueueEntry queueEntry;
retval.configNames = configs;
retval.options = options;
retval.outputPath = outputPath;
retval.dylibsForCache = dylibsForCache(*configs.begin(), arch);
retval.otherDylibsAndBundles = otherDylibsAndBundles(*configs.begin(), arch);
retval.mainExecutables = mainExecutables(*configs.begin(), arch);
return retval;
}
bool Manifest::loadParser(const void* p, size_t size, uint64_t sliceOffset, const std::string& runtimePath, const std::string& buildPath, const std::set<std::string>& architectures)
{
const mach_header* mh = reinterpret_cast<const mach_header*>(p);
if (!MachOParser::isValidMachO(_diags, "", _platform, p, size, runtimePath.c_str(), false)) {
return false;
}
auto parser = MachOParser(mh);
if (_diags.hasError()) {
// Clear the error and punt
_diags.verbose("MachoParser error: %s\n", _diags.errorMessage().c_str());
_diags.clearError();
return false;
}
auto uuid = parser.uuid();
auto archName = parser.archName();
if (parser.fileType() == MH_DYLIB && architectures.count(parser.archName()) != 0) {
std::string installName = parser.installName();
auto index = std::make_pair(installName, parser.archName());
auto i = _installNameMap.find(index);
if ( installName == "/System/Library/Caches/com.apple.xpc/sdk.dylib"
|| installName == "/System/Library/Caches/com.apple.xpcd/xpcd_cache.dylib" ) {
// HACK to deal with device specific dylibs. These must not be inseted into the installNameMap
_uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, installName)));
} else if (i == _installNameMap.end()) {
_installNameMap.insert(std::make_pair(index, uuid));
_uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, installName)));
if (installName[0] != '@' && installName != runtimePath) {
_diags.warning("Dylib located at '%s' has installname '%s'", runtimePath.c_str(), installName.c_str());
}
} else {
auto info = infoForUUID(i->second);
_diags.warning("Multiple dylibs claim installname '%s' ('%s' and '%s')", installName.c_str(), runtimePath.c_str(), info.runtimePath.c_str());
// This is the "Good" one, overwrite
if (runtimePath == installName) {
_uuidMap.erase(uuid);
_uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, installName)));
}
}
} else {
_uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, "")));
}
return true;
}
//FIXME: assert we have not errored first
bool Manifest::loadParsers(const std::string& buildPath, const std::string& runtimePath, const std::set<std::string>& architectures)
{
__block bool retval = false;
const void* p = (uint8_t*)(-1);
struct stat stat_buf;
std::tie(p, stat_buf) = fileCache.cacheLoad(_diags, buildPath);
if (p == (uint8_t*)(-1)) {
return false;
}
if (FatUtil::isFatFile(p)) {
FatUtil::forEachSlice(_diags, p, stat_buf.st_size, ^(uint32_t sliceCpuType, uint32_t sliceCpuSubType, const void* sliceStart, size_t sliceSize, bool& stop) {
if (loadParser(sliceStart, sliceSize, (uintptr_t)sliceStart-(uintptr_t)p, runtimePath, buildPath, architectures))
retval = true;
});
} else {
return loadParser(p, stat_buf.st_size, 0, runtimePath, buildPath, architectures);
}
return retval;
}
const Manifest::UUIDInfo& Manifest::infoForUUID(const UUID& uuid) const {
auto i = _uuidMap.find(uuid);
assert(i != _uuidMap.end());
return i->second;
}
const Manifest::UUIDInfo Manifest::infoForInstallNameAndarch(const std::string& installName, const std::string arch) const {
UUIDInfo retval;
auto uuidI = _installNameMap.find(std::make_pair(installName, arch));
if (uuidI == _installNameMap.end())
return UUIDInfo();
auto i = _uuidMap.find(uuidI->second);
if (i == _uuidMap.end())
return UUIDInfo();
return i->second;
}
MachOParser Manifest::parserForUUID(const UUID& uuid) const {
return MachOParser(infoForUUID(uuid).mh);
}
const std::string Manifest::buildPathForUUID(const UUID& uuid) {
return infoForUUID(uuid).buildPath;
}
const std::string Manifest::runtimePathForUUID(const UUID& uuid) {
return infoForUUID(uuid).runtimePath;
}
Manifest::Manifest(Diagnostics& D, const std::string& path) : Manifest(D, path, std::set<std::string>())
{
}
Manifest::Manifest(Diagnostics& D, const std::string& path, const std::set<std::string>& overlays) :
_diags(D)
{
NSMutableDictionary* manifestDict = [NSMutableDictionary dictionaryWithContentsOfFile:cppToObjStr(path)];
NSString* platStr = manifestDict[@"platform"];
std::set<std::string> architectures;
if (platStr == nullptr)
platStr = @"ios";
std::string platformString = [platStr UTF8String];
setMetabomFile([manifestDict[@"metabomFile"] UTF8String]);
if (platformString == "ios") {
setPlatform(dyld3::Platform::iOS);
} else if ( (platformString == "tvos") || (platformString == "atv") ) {
setPlatform(dyld3::Platform::tvOS);
} else if ( (platformString == "watchos") || (platformString == "watch") ) {
setPlatform(dyld3::Platform::watchOS);
} else if ( (platformString == "bridgeos") || (platformString == "bridge") ) {
setPlatform(dyld3::Platform::bridgeOS);
} else if ( (platformString == "macos") || (platformString == "osx") ) {
setPlatform(dyld3::Platform::macOS);
} else {
//Fixme should we error?
setPlatform(dyld3::Platform::iOS);
}
for (NSString* project in manifestDict[@"projects"]) {
for (NSString* source in manifestDict[@"projects"][project]) {
addProjectSource([project UTF8String], [source UTF8String]);
}
}
for (NSString* configuration in manifestDict[@"configurations"]) {
std::string configStr = [configuration UTF8String];
std::string configTag = [manifestDict[@"configurations"][configuration][@"metabomTag"] UTF8String];
if (manifestDict[@"configurations"][configuration][@"metabomExcludeTags"]) {
for (NSString* excludeTag in manifestDict[@"configurations"][configuration][@"metabomExcludeTags"]) {
_metabomExcludeTagMap[configStr].insert([excludeTag UTF8String]);
_configurations[configStr].metabomExcludeTags.insert([excludeTag UTF8String]);
}
}
if (manifestDict[@"configurations"][configuration][@"metabomRestrictTags"]) {
for (NSString* restrictTag in manifestDict[@"configurations"][configuration][@"metabomRestrictTags"]) {
_metabomRestrictedTagMap[configStr].insert([restrictTag UTF8String]);
_configurations[configStr].metabomRestrictTags.insert([restrictTag UTF8String]);
}
}
_configurations[configStr].metabomTag = configTag;
_configurations[configStr].metabomTags.insert(configTag);
_configurations[configStr].platformName =
[manifestDict[@"configurations"][configuration][@"platformName"] UTF8String];
if (endsWith(configStr, "InternalOS")) {
_configurations[configStr].disposition = "internal";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("InternalOS"));
} else if (endsWith(configStr, "VendorOS")) {
_configurations[configStr].disposition = "internal";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("VendorOS"));
} else if (endsWith(configStr, "VendorUIOS")) {
_configurations[configStr].disposition = "internal";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("VendorUIOS"));
} else if (endsWith(configStr, "CarrierOS")) {
_configurations[configStr].disposition = "internal";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("CarrierOS"));
} else if (endsWith(configStr, "FactoryOS")) {
_configurations[configStr].disposition = "internal";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("FactoryOS"));
} else if (endsWith(configStr, "DesenseOS")) {
_configurations[configStr].disposition = "internal";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("DesenseOS"));
} else if (endsWith(configStr, "MinosOS")) {
_configurations[configStr].disposition = "minos";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("MinosOS"));
} else if (endsWith(configStr, "DemoOS")) {
_configurations[configStr].disposition = "demo";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("DemoOS"));
} else if (endsWith(configStr, "MinosOS")) {
_configurations[configStr].disposition = "minos";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("MinosOS"));
} else if (endsWith(configStr, "DeveloperOS")) {
_configurations[configStr].disposition = "user";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("DeveloperOS"));
} else if (endsWith(configStr, "OS")) {
_configurations[configStr].disposition = "user";
_configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("OS"));
}
for (NSString* architecutre in manifestDict[@"configurations"][configuration][@"architectures"]) {
//HACK until B&I stops mastering armv7s
if ([architecutre isEqual:@"armv7s"]) break;
_configurations[configStr].architectures[[architecutre UTF8String]] = Architecture();
architectures.insert([architecutre UTF8String]);
}
}
setVersion([manifestDict[@"manifest-version"] unsignedIntValue]);
setBuild([manifestDict[@"build"] UTF8String]);
if (manifestDict[@"dylibOrderFile"]) {
setDylibOrderFile([manifestDict[@"dylibOrderFile"] UTF8String]);
}
if (manifestDict[@"dirtyDataOrderFile"]) {
setDirtyDataOrderFile([manifestDict[@"dirtyDataOrderFile"] UTF8String]);
}
auto metabom = MBMetabomOpen(metabomFile().c_str(), false);
auto metabomEnumerator = MBIteratorNewWithPath(metabom, ".", "");
MBEntry entry;
// FIXME error handling (NULL metabom)
//First we iterate through the bom and build our objects
while ((entry = MBIteratorNext(metabomEnumerator))) {
BOMFSObject fsObject = MBEntryGetFSObject(entry);
BOMFSObjType entryType = BOMFSObjectType(fsObject);
std::string entryPath = BOMFSObjectPathName(fsObject);
if (entryPath[0] == '.') {
entryPath.erase(0, 1);
}
// Skip artifacts that happen to be in the build chain
if ( startsWith(entryPath, "/Applications/Xcode.app") ) {
continue;
}
// Skip variants we can't deal with
if ( endsWith(entryPath, "_profile.dylib") || endsWith(entryPath, "_debug.dylib") || endsWith(entryPath, "_profile") || endsWith(entryPath, "_debug") || endsWith(entryPath, "/CoreADI") ) {
continue;
}
// Skip images that are only used in InternalOS variants
if ( startsWith(entryPath, "/AppleInternal/") || startsWith(entryPath, "/usr/local/") || startsWith(entryPath, "/Developer/")) {
continue;
}
// Skip genCache generated dylibs
if ( endsWith(entryPath, "/System/Library/Caches/com.apple.xpc/sdk.dylib") || endsWith(entryPath, "/System/Library/Caches/com.apple.xpcd/xpcd_cache.dylib")) {
continue;
}
MBTag tag;
auto tagCount = MBEntryGetNumberOfProjectTags(entry);
if (entryType == BOMFileType && BOMFSObjectIsBinaryObject(fsObject) && MBEntryGetNumberOfProjectTags(entry) != 0 && tagCount != 0) {
if (tagCount == 1) {
MBEntryGetProjectTags(entry, &tag);
} else {
MBTag* tags = (MBTag*)malloc(sizeof(MBTag) * tagCount);
MBEntryGetProjectTags(entry, tags);
//Sigh, we can have duplicate entries for the same tag, so build a set to work with
std::set<std::string> tagStrs;
std::map<std::string, MBTag> tagStrMap;
for (auto i = 0; i < tagCount; ++i) {
tagStrs.insert(MBMetabomGetProjectForTag(metabom, tags[i]));
tagStrMap.insert(std::make_pair(MBMetabomGetProjectForTag(metabom, tags[i]), tags[i]));
}
if (tagStrs.size() > 1) {
std::string projects;
for (const auto& tagStr : tagStrs) {
if (!projects.empty())
projects += ", ";
projects += "'" + tagStr + "'";
}
_diags.warning("Bom entry '%s' is claimed by multiple projects: %s, taking first entry", entryPath.c_str(), projects.c_str());
}
tag = tagStrMap[*tagStrs.begin()];
free(tags);
}
std::string projectName = MBMetabomGetProjectForTag(metabom, tag);
tagCount = MBEntryGetNumberOfPackageTags(entry);
MBTag* tags = (MBTag*)malloc(sizeof(MBTag) * tagCount);
MBEntryGetPackageTags(entry, tags);
std::set<std::string> tagStrs;
for (auto i = 0; i < tagCount; ++i) {
tagStrs.insert(MBMetabomGetPackageForTag(metabom, tags[i]));
}
_metabomTagMap.insert(std::make_pair(entryPath, tagStrs));
bool foundParser = false;
for (const auto& overlay : overlays) {
if (loadParsers(overlay + "/" + entryPath, entryPath, architectures)) {
foundParser = true;
break;
}
}
if (!foundParser) {
(void)loadParsers(projectPath(projectName) + "/" + entryPath, entryPath, architectures);
}
}
}
MBIteratorFree(metabomEnumerator);
MBMetabomFree(metabom);
}
void Manifest::insert(std::vector<DyldSharedCache::MappedMachO>& mappedMachOs, const CacheImageInfo& imageInfo) {
auto info = infoForUUID(imageInfo.uuid);
auto runtimePath = info.runtimePath;
mappedMachOs.emplace_back(runtimePath, info.mh, info.size, false, false, info.sliceFileOffset, 0, 0);
}
std::vector<DyldSharedCache::MappedMachO> Manifest::dylibsForCache(const std::string& configuration, const std::string& architecture)
{
std::vector<DyldSharedCache::MappedMachO> retval;
const auto& dylibs = _configurations[configuration].architectures[architecture].results.dylibs;
for (const auto& dylib : dylibs) {
if (dylib.second.included) {
insert(retval, dylib.second);
}
}
return retval;
}
std::vector<DyldSharedCache::MappedMachO> Manifest::otherDylibsAndBundles(const std::string& configuration, const std::string& architecture)
{
std::vector<DyldSharedCache::MappedMachO> retval;
const auto& dylibs = _configurations[configuration].architectures[architecture].results.dylibs;
for (const auto& dylib : dylibs) {
if (!dylib.second.included) {
insert(retval, dylib.second);
}
}
const auto& bundles = _configurations[configuration].architectures[architecture].results.bundles;
for (const auto& bundle : bundles) {
insert(retval, bundle.second);
}
return retval;
}
std::vector<DyldSharedCache::MappedMachO> Manifest::mainExecutables(const std::string& configuration, const std::string& architecture)
{
std::vector<DyldSharedCache::MappedMachO> retval;
const auto& executables = _configurations[configuration].architectures[architecture].results.executables;
for (const auto& executable : executables) {
insert(retval, executable.second);
}
return retval;
}
bool Manifest::filterForConfig(const std::string& configName)
{
for (const auto configuration : _configurations) {
if (configName == configuration.first) {
std::map<std::string, Configuration> filteredConfigs;
filteredConfigs[configName] = configuration.second;
_configurations = filteredConfigs;
for (auto& arch : configuration.second.architectures) {
arch.second.results = Manifest::Results();
}
return true;
}
}
return false;
}
void Manifest::dedupeDispositions(void) {
// Since this is all hacky and inference based for now only do it for iOS until XBS
// is reved to give us real info. All the other platforms are way smaller anyway.
if (_platform != Platform::iOS)
return;
std::map<std::pair<std::string, std::string>, std::set<std::string>> dispositionSets;
for (const auto& configuration : _configurations) {
dispositionSets[std::make_pair(configuration.second.device, configuration.second.disposition)].insert(configuration.first);
}
for (const auto& dSet : dispositionSets) {
for (const auto &c1 : dSet.second) {
for (const auto &c2 : dSet.second) {
_configurations[c1].metabomTags.insert(_configurations[c2].metabomTag);
}
}
}
}
void Manifest::calculateClosure()
{
auto closureSemaphore = dispatch_semaphore_create(32);
auto closureGroup = dispatch_group_create();
auto closureQueue = dispatch_queue_create("com.apple.dyld.cache.closure", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_CONCURRENT, QOS_CLASS_USER_INITIATED, 0));
dedupeDispositions();
for (auto& config : _configurations) {
for (auto& arch : config.second.architectures) {
dispatch_semaphore_wait(closureSemaphore, DISPATCH_TIME_FOREVER);
dispatch_group_async(closureGroup, closureQueue, [&] {
calculateClosure(config.first, arch.first);
dispatch_semaphore_signal(closureSemaphore);
});
}
}
dispatch_group_wait(closureGroup, DISPATCH_TIME_FOREVER);
}
void Manifest::remove(const std::string& config, const std::string& arch)
{
if (_configurations.count(config))
_configurations[config].architectures.erase(arch);
}
void Manifest::removeDylib(MachOParser parser, const std::string& reason, const std::string& configuration,
const std::string& architecture, std::unordered_set<UUID>& processedIdentifiers)
{
#if 0
auto configIter = _configurations.find(configuration);
if (configIter == _configurations.end())
return;
auto archIter = configIter->second.architectures.find( architecture );
if ( archIter == configIter->second.architectures.end() ) return;
auto& archManifest = archIter->second;
if (archManifest.results.dylibs.count(parser->uuid()) == 0) {
archManifest.results.dylibs[parser->uuid()].uuid = parser->uuid();
archManifest.results.dylibs[parser->uuid()].installname = parser->installName();
processedIdentifiers.insert(parser->uuid());
}
archManifest.results.exclude(MachOProxy::forIdentifier(parser->uuid(), architecture), reason);
processedIdentifiers.insert(parser->uuid());
for (const auto& dependent : proxy->dependentIdentifiers) {
auto dependentProxy = MachOProxy::forIdentifier(dependent, architecture);
auto dependentResultIter = archManifest.results.dylibs.find(dependentProxy->identifier);
if ( dependentProxy &&
( dependentResultIter == archManifest.results.dylibs.end() || dependentResultIter->second.included == true ) ) {
removeDylib(dependentProxy, "Missing dependency: " + proxy->installName, configuration, architecture,
processedIdentifiers);
}
}
#endif
}
const std::string Manifest::removeLargestLeafDylib(const std::set<std::string>& configurations, const std::string& architecture)
{
// Find the leaf nodes
__block std::map<std::string, uint64_t> dependentCounts;
for (const auto& dylib : _configurations[*configurations.begin()].architectures[architecture].results.dylibs) {
if (!dylib.second.included)
continue;
std::string installName;
auto info = infoForUUID(dylib.first);
auto parser = MachOParser(info.mh);
dependentCounts[parser.installName()] = 0;
}
for (const auto& dylib : _configurations[*configurations.begin()].architectures[architecture].results.dylibs) {
if (!dylib.second.included)
continue;
auto info = infoForUUID(dylib.first);
auto parser = MachOParser(info.mh);
parser.forEachDependentDylib(^(const char *loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool &stop) {
if (!isWeak) {
dependentCounts[loadPath]++;
}
});
}
// Figure out which leaf is largest
UUIDInfo largestLeaf;
for (const auto& dependentCount : dependentCounts) {
if (dependentCount.second == 0) {
auto info = infoForInstallNameAndarch(dependentCount.first, architecture);
assert(info.mh != nullptr);
if (info.size > largestLeaf.size) {
largestLeaf = info;
}
}
}
if (largestLeaf.mh == nullptr) {
_diags.error("Fatal overflow, could not evict more dylibs");
return "";
}
// Remove it ferom all configs
for (const auto& config : configurations) {
configuration(config).architecture(architecture).results.exclude(*this, largestLeaf.uuid, "Cache Overflow");
}
return largestLeaf.installName;
}
void Manifest::calculateClosure(const std::string& configuration, const std::string& architecture)
{
__block auto& configManifest = _configurations[configuration];
__block auto& archManifest = _configurations[configuration].architectures[architecture];
__block std::set<UUID> newUuids;
std::set<UUID> processedUuids;
std::set<UUID> cachedUUIDs;
// Seed anchors
for (auto& uuidInfo : _uuidMap) {
auto info = uuidInfo.second;
if (info.arch != architecture) {
continue;
}
auto i = _metabomTagMap.find(info.runtimePath);
assert(i != _metabomTagMap.end());
auto tags = i->second;
if (!is_disjoint(tags, configManifest.metabomTags)) {
newUuids.insert(info.uuid);
}
}
// Pull in all dependencies
while (!newUuids.empty()) {
std::set<UUID> uuidsToProcess = newUuids;
newUuids.clear();
for (const auto& uuid : uuidsToProcess) {
if (processedUuids.count(uuid) > 0) {
continue;
}
processedUuids.insert(uuid);
auto parser = parserForUUID(uuid);
auto runtimePath = runtimePathForUUID(uuid);
assert(parser.header() != 0);
parser.forEachDependentDylib(^(const char* loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool& stop) {
auto i = _installNameMap.find(std::make_pair(loadPath, architecture));
if (i != _installNameMap.end())
newUuids.insert(i->second);
});
if (parser.fileType() == MH_DYLIB) {
// Add the dylib to the results
if (archManifest.results.dylibs.count(uuid) == 0 ) {
archManifest.results.dylibs[uuid].uuid = uuid;
archManifest.results.dylibs[uuid].installname = parser.installName();
}
// HACK to insert device specific dylib closures into all caches
if ( parser.installName() == std::string("/System/Library/Caches/com.apple.xpc/sdk.dylib")
|| parser.installName() == std::string("/System/Library/Caches/com.apple.xpcd/xpcd_cache.dylib") ) {
archManifest.results.exclude(&parser, "Device specific dylib");
continue;
}
std::set<std::string> reasons;
if (parser.canBePlacedInDyldCache(runtimePath, reasons)) {
auto i = _metabomTagMap.find(runtimePath);
assert(i != _metabomTagMap.end());
auto restrictions = _metabomRestrictedTagMap.find(configuration);
if (restrictions != _metabomRestrictedTagMap.end() && !is_disjoint(restrictions->second, i->second)) {
archManifest.results.exclude(&parser, "Dylib '" + runtimePath + "' removed due to explict restriction");
}
// It can be placed in the cache, grab its dependents and queue them for inclusion
cachedUUIDs.insert(parser.uuid());
} else {
// It can't be placed in the cache, print out the reasons why
std::string reasonString = "Rejected from cached dylibs: " + runtimePath + " " + architecture + " (\"";
for (auto i = reasons.begin(); i != reasons.end(); ++i) {
reasonString += *i;
if (i != --reasons.end()) {
reasonString += "\", \"";
}
}
reasonString += "\")";
archManifest.results.exclude(&parser, reasonString);
}
} else if (parser.fileType() == MH_BUNDLE) {
if (archManifest.results.bundles.count(uuid) == 0) {
archManifest.results.bundles[uuid].uuid = uuid;
}
} else if (parser.fileType() == MH_EXECUTE) {
//HACK exclude all launchd and installd variants until we can do something about xpcd_cache.dylib and friends
if (runtimePath == "/sbin/launchd"
|| runtimePath == "/usr/local/sbin/launchd.debug"
|| runtimePath == "/usr/local/sbin/launchd.development"
|| runtimePath == "/usr/libexec/installd") {
continue;
}
if (archManifest.results.executables.count(uuid) == 0) {
archManifest.results.executables[uuid].uuid = uuid;
}
}
}
}
__block std::set<UUID> removedUUIDs;
__block bool doAgain = true;
//Trim out dylibs that are missing dependencies
while ( doAgain ) {
doAgain = false;
for (const auto& uuid : cachedUUIDs) {
__block std::set<std::string> badDependencies;
__block auto parser = parserForUUID(uuid);
parser.forEachDependentDylib(^(const char* loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool& stop) {
if (isWeak)
return;
auto i = _installNameMap.find(std::make_pair(loadPath, architecture));
if (i == _installNameMap.end() || removedUUIDs.count(i->second)) {
removedUUIDs.insert(uuid);
badDependencies.insert(loadPath);
doAgain = true;
}
if (badDependencies.size()) {
std::string reasonString = "Rejected from cached dylibs: " + std::string(parser.installName()) + " " + architecture + " (\"";
for (auto i = badDependencies.begin(); i != badDependencies.end(); ++i) {
reasonString += *i;
if (i != --badDependencies.end()) {
reasonString += "\", \"";
}
}
reasonString += "\")";
archManifest.results.exclude(&parser, reasonString);
}
});
}
for (const auto& removedUUID : removedUUIDs) {
cachedUUIDs.erase(removedUUID);
}
}
//Trim out excluded leaf dylibs
__block std::set<std::string> linkedDylibs;
for(const auto& uuid : cachedUUIDs) {
auto parser = parserForUUID(uuid);
parser.forEachDependentDylib(^(const char* loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool& stop) {
linkedDylibs.insert(loadPath);
});
}
for(const auto& uuid : cachedUUIDs) {
auto info = infoForUUID(uuid);
auto i = _metabomTagMap.find(info.runtimePath);
assert(i != _metabomTagMap.end());
auto exclusions = _metabomExcludeTagMap.find(configuration);
if (exclusions == _metabomExcludeTagMap.end() || is_disjoint(exclusions->second, i->second))
continue;
if (linkedDylibs.count(info.installName) != 0)
continue;
archManifest.results.exclude(*this, info.uuid, "Dylib '" + info.runtimePath + "' excluded leaf node");
}
}
void Manifest::writeJSON(const std::string& path) {
NSMutableDictionary* jsonDict = [[NSMutableDictionary alloc] init];
for (auto& configuration : _configurations) {
jsonDict[cppToObjStr(configuration.first)] = [[NSMutableDictionary alloc] init];
for (auto& arch : configuration.second.architectures) {
NSMutableOrderedSet* includedDylibsSet = [[NSMutableOrderedSet alloc] init];
NSMutableOrderedSet* executablesSet = [[NSMutableOrderedSet alloc] init];
NSMutableOrderedSet* otherSet = [[NSMutableOrderedSet alloc] init];
for (auto& dylib : arch.second.results.dylibs) {
NSString *runtimePath = cppToObjStr(runtimePathForUUID(dylib.second.uuid));
if (dylib.second.included) {
[includedDylibsSet addObject:runtimePath];
} else {
[otherSet addObject:runtimePath];
}
}
for (auto& executable : arch.second.results.executables) {
NSString *runtimePath = cppToObjStr(runtimePathForUUID(executable.second.uuid));
[executablesSet addObject:runtimePath];
}
for (auto& bundle : arch.second.results.bundles) {
NSString *runtimePath = cppToObjStr(runtimePathForUUID(bundle.second.uuid));
[otherSet addObject:runtimePath];
}
[includedDylibsSet sortUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
return [obj1 compare:obj2];
}];
[executablesSet sortUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
return [obj1 compare:obj2];
}];
[otherSet sortUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
return [obj1 compare:obj2];
}];
jsonDict[cppToObjStr(configuration.first)][cppToObjStr(arch.first)] = @{ @"cachedDylibs" : [includedDylibsSet array], @"mainExecutables" : [executablesSet array], @"other" : [otherSet array]};;
}
}
NSError* error = nil;
NSData *jsonData = [NSJSONSerialization dataWithJSONObject:jsonDict options:0x0 error:&error];
(void)[jsonData writeToFile:cppToObjStr(path) atomically:YES];
}
void Manifest::write(const std::string& path)
{
if (path.empty())
return;
NSMutableDictionary* cacheDict = [[NSMutableDictionary alloc] init];
NSMutableDictionary* projectDict = [[NSMutableDictionary alloc] init];
NSMutableDictionary* configurationsDict = [[NSMutableDictionary alloc] init];
NSMutableDictionary* resultsDict = [[NSMutableDictionary alloc] init];
cacheDict[@"manifest-version"] = @(version());
cacheDict[@"build"] = cppToObjStr(build());
cacheDict[@"dylibOrderFile"] = cppToObjStr(dylibOrderFile());
cacheDict[@"dirtyDataOrderFile"] = cppToObjStr(dirtyDataOrderFile());
cacheDict[@"metabomFile"] = cppToObjStr(metabomFile());
cacheDict[@"projects"] = projectDict;
cacheDict[@"results"] = resultsDict;
cacheDict[@"configurations"] = configurationsDict;
for (const auto& project : projects()) {
NSMutableArray* sources = [[NSMutableArray alloc] init];
for (const auto& source : project.second.sources) {
[sources addObject:cppToObjStr(source)];
}
projectDict[cppToObjStr(project.first)] = sources;
}
for (auto& configuration : _configurations) {
NSMutableArray* archArray = [[NSMutableArray alloc] init];
for (auto& arch : configuration.second.architectures) {
[archArray addObject:cppToObjStr(arch.first)];
}
NSMutableArray* excludeTags = [[NSMutableArray alloc] init];
for (const auto& excludeTag : configuration.second.metabomExcludeTags) {
[excludeTags addObject:cppToObjStr(excludeTag)];
}
configurationsDict[cppToObjStr(configuration.first)] = @{
@"platformName" : cppToObjStr(configuration.second.platformName),
@"metabomTag" : cppToObjStr(configuration.second.metabomTag),
@"metabomExcludeTags" : excludeTags,
@"architectures" : archArray
};
}
for (auto& configuration : _configurations) {
NSMutableDictionary* archResultsDict = [[NSMutableDictionary alloc] init];
for (auto& arch : configuration.second.architectures) {
NSMutableDictionary* dylibsDict = [[NSMutableDictionary alloc] init];
NSMutableArray* warningsArray = [[NSMutableArray alloc] init];
NSMutableDictionary* devRegionsDict = [[NSMutableDictionary alloc] init];
NSMutableDictionary* prodRegionsDict = [[NSMutableDictionary alloc] init];
NSString* prodCDHash = cppToObjStr(arch.second.results.productionCache.cdHash);
NSString* devCDHash = cppToObjStr(arch.second.results.developmentCache.cdHash);
for (auto& dylib : arch.second.results.dylibs) {
NSMutableDictionary* dylibDict = [[NSMutableDictionary alloc] init];
if (dylib.second.included) {
dylibDict[@"included"] = @YES;
} else {
dylibDict[@"included"] = @NO;
dylibDict[@"exclusionInfo"] = cppToObjStr(dylib.second.exclusionInfo);
}
dylibsDict[cppToObjStr(dylib.second.installname)] = dylibDict;
}
for (auto& warning : arch.second.results.warnings) {
[warningsArray addObject:cppToObjStr(warning)];
}
BOOL built = arch.second.results.failure.empty();
archResultsDict[cppToObjStr(arch.first)] = @{
@"dylibs" : dylibsDict,
@"built" : @(built),
@"failure" : cppToObjStr(arch.second.results.failure),
@"productionCache" : @{ @"cdhash" : prodCDHash, @"regions" : prodRegionsDict },
@"developmentCache" : @{ @"cdhash" : devCDHash, @"regions" : devRegionsDict },
@"warnings" : warningsArray
};
}
resultsDict[cppToObjStr(configuration.first)] = archResultsDict;
}
switch (platform()) {
case Platform::iOS:
cacheDict[@"platform"] = @"ios";
break;
case Platform::tvOS:
cacheDict[@"platform"] = @"tvos";
break;
case Platform::watchOS:
cacheDict[@"platform"] = @"watchos";
break;
case Platform::bridgeOS:
cacheDict[@"platform"] = @"bridgeos";
break;
case Platform::macOS:
cacheDict[@"platform"] = @"macos";
break;
case Platform::unknown:
cacheDict[@"platform"] = @"unknown";
break;
}
NSError* error = nil;
NSData* outData = [NSPropertyListSerialization dataWithPropertyList:cacheDict
format:NSPropertyListBinaryFormat_v1_0
options:0
error:&error];
(void)[outData writeToFile:cppToObjStr(path) atomically:YES];
}
}