OctagonTests+ErrorHandling.swift [plain text]
#if OCTAGON
class OctagonErrorHandlingTests: OctagonTestsBase {
func testRecoverFromImmediateTimeoutDuringEstablish() throws {
self.startCKAccountStatusMock()
let establishExpectation = self.expectation(description: "establishExpectation")
self.fakeCuttlefishServer.establishListener = { [unowned self] request in
self.fakeCuttlefishServer.establishListener = nil
establishExpectation.fulfill()
return CKPrettyError(domain: CKErrorDomain,
code: CKError.networkFailure.rawValue,
userInfo: [:])
}
_ = self.assertResetAndBecomeTrustedInDefaultContext()
self.wait(for: [establishExpectation], timeout: 10)
}
func testRecoverFromRetryableErrorDuringEstablish() throws {
self.startCKAccountStatusMock()
let establishExpectation = self.expectation(description: "establishExpectation")
var t0 = Date.distantPast
self.fakeCuttlefishServer.establishListener = { [unowned self] request in
self.fakeCuttlefishServer.establishListener = nil
establishExpectation.fulfill()
t0 = Date()
return FakeCuttlefishServer.makeCloudKitCuttlefishError(code: .retryableServerFailure)
}
_ = self.assertResetAndBecomeTrustedInDefaultContext()
self.wait(for: [establishExpectation], timeout: 10)
let t1 = Date()
let d = t0.distance(to: t1)
XCTAssertGreaterThanOrEqual(d, 4)
// Let slower devices have a few extra seconds: we expect this after 5s, but sometimes they need a bit.
XCTAssertLessThanOrEqual(d, 8)
}
func testRecoverFromTransactionalErrorDuringJoinWithVoucher() throws {
self.startCKAccountStatusMock()
self.assertResetAndBecomeTrustedInDefaultContext()
var t0 = Date.distantPast
let joinExpectation = self.expectation(description: "joinExpectation")
self.fakeCuttlefishServer.joinListener = { [unowned self] _ in
self.fakeCuttlefishServer.joinListener = nil
joinExpectation.fulfill()
t0 = Date()
return FakeCuttlefishServer.makeCloudKitCuttlefishError(code: .transactionalFailure)
}
let joiningContext = self.makeInitiatorContext(contextID: "joiner", authKitAdapter: self.mockAuthKit2)
_ = self.assertJoinViaEscrowRecovery(joiningContext: joiningContext, sponsor: self.cuttlefishContext)
self.wait(for: [joinExpectation], timeout: 10)
let t1 = Date()
let d = t0.distance(to: t1)
XCTAssertGreaterThanOrEqual(d, 4)
// Let slower devices have a few extra seconds: we expect this after 5s, but sometimes they need a bit.
XCTAssertLessThanOrEqual(d, 8)
}
func testReceiveUpdateWhileUntrustedAndLocked() {
self.startCKAccountStatusMock()
self.cuttlefishContext.startOctagonStateMachine()
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateUntrusted, within: 10 * NSEC_PER_SEC)
self.aksLockState = true
self.lockStateTracker.recheck()
self.sendContainerChange(context: self.cuttlefishContext)
XCTAssertEqual(0, self.cuttlefishContext.stateMachine.paused.wait(10 * NSEC_PER_SEC), "state machine should pause")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateUntrusted, within: 10 * NSEC_PER_SEC)
}
func testReceiveUpdateWhileReadyAndLocked() {
self.startCKAccountStatusMock()
self.cuttlefishContext.startOctagonStateMachine()
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateUntrusted, within: 10 * NSEC_PER_SEC)
do {
let clique = try OTClique.newFriends(withContextData: self.otcliqueContext, resetReason: .testGenerated)
XCTAssertNotNil(clique, "Clique should not be nil")
} catch {
XCTFail("Shouldn't have errored making new friends: \(error)")
}
// Now, we should be in 'ready'
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.assertConsidersSelfTrusted(context: self.cuttlefishContext)
self.assertConsidersSelfTrustedCachedAccountStatus(context: self.cuttlefishContext)
self.aksLockState = true
self.lockStateTracker.recheck()
self.sendContainerChange(context: self.cuttlefishContext)
XCTAssertEqual(0, self.cuttlefishContext.stateMachine.paused.wait(10 * NSEC_PER_SEC), "state machine should pause")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.aksLockState = false
self.lockStateTracker.recheck()
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
// and again!
self.aksLockState = true
self.lockStateTracker.recheck()
self.sendContainerChange(context: self.cuttlefishContext)
sleep(1)
XCTAssertTrue(self.cuttlefishContext.stateMachine.possiblePendingFlags().contains("recd_push"), "Should have recd_push pending flag")
let waitForUnlockStateCondition = self.cuttlefishContext.stateMachine.stateConditions[OctagonStateWaitForUnlock] as! CKKSCondition
XCTAssertEqual(0, self.cuttlefishContext.stateMachine.paused.wait(10 * NSEC_PER_SEC), "state machine should pause")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
sleep(1)
// Check that we haven't been spinning
let sameWaitForUnlockStateCondition = self.cuttlefishContext.stateMachine.stateConditions[OctagonStateWaitForUnlock] as! CKKSCondition
XCTAssert(waitForUnlockStateCondition == sameWaitForUnlockStateCondition, "Conditions should be the same (as the state machine should be halted)")
self.aksLockState = false
self.lockStateTracker.recheck()
sleep(1)
XCTAssertEqual(self.cuttlefishContext.stateMachine.possiblePendingFlags(), [], "Should have 0 pending flags")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
}
func testReceiveUpdateWhileReadyAndAuthkitRetry() {
self.startCKAccountStatusMock()
self.cuttlefishContext.startOctagonStateMachine()
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateUntrusted, within: 10 * NSEC_PER_SEC)
do {
let clique = try OTClique.newFriends(withContextData: self.otcliqueContext, resetReason: .testGenerated)
XCTAssertNotNil(clique, "Clique should not be nil")
} catch {
XCTFail("Shouldn't have errored making new friends: \(error)")
}
// Now, we should be in 'ready'
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.assertConsidersSelfTrusted(context: self.cuttlefishContext)
self.assertConsidersSelfTrustedCachedAccountStatus(context: self.cuttlefishContext)
self.mockAuthKit.machineIDFetchErrors.append(CKPrettyError(domain: CKErrorDomain,
code: CKError.networkUnavailable.rawValue,
userInfo: [CKErrorRetryAfterKey: 2]))
self.sendContainerChange(context: self.cuttlefishContext)
XCTAssertEqual(0, self.cuttlefishContext.stateMachine.paused.wait(10 * NSEC_PER_SEC), "state machine should pause")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.sendContainerChange(context: self.cuttlefishContext)
XCTAssertEqual(0, self.cuttlefishContext.stateMachine.paused.wait(10 * NSEC_PER_SEC), "state machine should pause")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
}
func testReceiveUpdateWhileReadyAndLockedAndAuthkitRetry() {
self.startCKAccountStatusMock()
self.cuttlefishContext.startOctagonStateMachine()
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateUntrusted, within: 10 * NSEC_PER_SEC)
do {
let clique = try OTClique.newFriends(withContextData: self.otcliqueContext, resetReason: .testGenerated)
XCTAssertNotNil(clique, "Clique should not be nil")
} catch {
XCTFail("Shouldn't have errored making new friends: \(error)")
}
// Now, we should be in 'ready'
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.assertConsidersSelfTrusted(context: self.cuttlefishContext)
self.assertConsidersSelfTrustedCachedAccountStatus(context: self.cuttlefishContext)
self.aksLockState = true
self.lockStateTracker.recheck()
self.mockAuthKit.machineIDFetchErrors.append(CKPrettyError(domain: CKErrorDomain,
code: CKError.networkUnavailable.rawValue,
userInfo: [CKErrorRetryAfterKey: 2]))
self.sendContainerChange(context: self.cuttlefishContext)
XCTAssertEqual(0, self.cuttlefishContext.stateMachine.paused.wait(10 * NSEC_PER_SEC), "state machine should pause")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.sendContainerChange(context: self.cuttlefishContext)
XCTAssertEqual(0, self.cuttlefishContext.stateMachine.paused.wait(10 * NSEC_PER_SEC), "state machine should pause")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
sleep(1)
XCTAssertTrue(self.cuttlefishContext.stateMachine.possiblePendingFlags().contains("recd_push"), "Should have recd_push pending flag")
self.aksLockState = false
self.lockStateTracker.recheck()
sleep(1)
XCTAssertEqual(self.cuttlefishContext.stateMachine.possiblePendingFlags(), [], "Should have 0 pending flags")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
}
func testReceiveTransactionErrorDuringUpdate() {
self.startCKAccountStatusMock()
self.cuttlefishContext.startOctagonStateMachine()
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateUntrusted, within: 10 * NSEC_PER_SEC)
do {
let clique = try OTClique.newFriends(withContextData: self.otcliqueContext, resetReason: .testGenerated)
XCTAssertNotNil(clique, "Clique should not be nil")
} catch {
XCTFail("Shouldn't have errored making new friends: \(error)")
}
// Now, we should be in 'ready'
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.assertConsidersSelfTrusted(context: self.cuttlefishContext)
self.assertConsidersSelfTrustedCachedAccountStatus(context: self.cuttlefishContext)
let pre = self.fakeCuttlefishServer.fetchChangesCalledCount
let ckError = FakeCuttlefishServer.makeCloudKitCuttlefishError(code: .transactionalFailure)
self.fakeCuttlefishServer.nextFetchErrors.append(ckError)
self.sendContainerChangeWaitForFetch(context: self.cuttlefishContext)
sleep(5)
XCTAssertEqual(0, self.cuttlefishContext.stateMachine.paused.wait(10 * NSEC_PER_SEC), "state machine should pause")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
XCTAssertEqual(self.cuttlefishContext.stateMachine.possiblePendingFlags(), [], "Should have zero pending flags after retry")
let post = self.fakeCuttlefishServer.fetchChangesCalledCount
XCTAssertEqual(post, pre + 2, "should have fetched two times, the first response would have been a transaction error")
}
func testPreapprovedPushWhileLocked() throws {
// Peer 1 becomes SOS+Octagon
self.putFakeKeyHierarchy(inCloudKit: self.manateeZoneID)
self.putSelfTLKShares(inCloudKit: self.manateeZoneID)
self.saveTLKMaterial(toKeychain: self.manateeZoneID)
XCTAssertTrue(OctagonPerformSOSUpgrade(), "SOS upgrade should be on")
self.mockSOSAdapter.circleStatus = SOSCCStatus(kSOSCCInCircle)
self.startCKAccountStatusMock()
self.cuttlefishContext.startOctagonStateMachine()
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.assertConsidersSelfTrusted(context: self.cuttlefishContext)
let peerID = try self.cuttlefishContext.accountMetadataStore.getEgoPeerID()
XCTAssertNotNil(peerID, "Should have a peer ID after making new friends")
assertAllCKKSViews(enter: SecCKKSZoneKeyStateReady, within: 10 * NSEC_PER_SEC)
// Peer 2 attempts to join via preapprovalh
let peer2SOSMockPeer = self.createSOSPeer(peerID: "peer2ID")
let peer2contextID = "peer2"
let peer2mockSOS = CKKSMockSOSPresentAdapter(selfPeer: peer2SOSMockPeer, trustedPeers: self.mockSOSAdapter.allPeers(), essential: false)
let peer2 = self.manager.context(forContainerName: OTCKContainerName,
contextID: peer2contextID,
sosAdapter: peer2mockSOS,
authKitAdapter: self.mockAuthKit2,
lockStateTracker: self.lockStateTracker,
accountStateTracker: self.accountStateTracker,
deviceInformationAdapter: self.makeInitiatorDeviceInfoAdapter())
peer2.startOctagonStateMachine()
self.assertEnters(context: peer2, state: OctagonStateUntrusted, within: 10 * NSEC_PER_SEC)
// Now, Peer1 should preapprove Peer2
let peer2Preapproval = TPHashBuilder.hash(with: .SHA256, of: peer2SOSMockPeer.publicSigningKey.encodeSubjectPublicKeyInfo())
self.mockSOSAdapter.trustedPeers.add(peer2SOSMockPeer)
self.mockSOSAdapter.sendTrustedPeerSetChangedUpdate()
// Peer1 should upload TLKs for Peer2
self.assertAllCKKSViewsUpload(tlkShares: 1)
let updateTrustExpectation = self.expectation(description: "updateTrust")
self.fakeCuttlefishServer.updateListener = { request in
XCTAssertEqual(peerID, request.peerID, "updateTrust request should be for ego peer ID")
XCTAssertTrue(request.hasDynamicInfoAndSig, "updateTrust request should have a dynamic info")
let newDynamicInfo = TPPeerDynamicInfo(data: request.dynamicInfoAndSig.peerDynamicInfo, sig: request.dynamicInfoAndSig.sig)
XCTAssertNotNil(newDynamicInfo, "should be able to make a dynamic info from protobuf")
XCTAssertEqual(newDynamicInfo!.preapprovals.count, 1, "Should have a single preapproval")
XCTAssertTrue(newDynamicInfo!.preapprovals.contains(peer2Preapproval), "Octagon peer should preapprove new SOS peer")
// But, since this is an SOS peer and TLK uploads for those peers are currently handled through CK CRUD operations, this update
// shouldn't have any TLKShares
XCTAssertEqual(0, request.tlkShares.count, "Trust update should not have any new TLKShares")
updateTrustExpectation.fulfill()
return nil
}
self.verifyDatabaseMocks()
self.wait(for: [updateTrustExpectation], timeout: 100)
self.fakeCuttlefishServer.updateListener = nil
// Now, peer 2 should lock and receive an Octagon push
self.aksLockState = true
self.lockStateTracker.recheck()
// Now, peer2 should receive an Octagon push, try to realize it is preapproved, and get stuck
self.sendContainerChange(context: peer2)
self.assertEnters(context: peer2, state: OctagonStateUntrusted, within: 10 * NSEC_PER_SEC)
sleep(1)
XCTAssertTrue(peer2.stateMachine.possiblePendingFlags().contains("recd_push"), "Should have recd_push pending flag")
self.aksLockState = false
self.lockStateTracker.recheck()
sleep(1)
XCTAssertEqual(self.cuttlefishContext.stateMachine.possiblePendingFlags(), [], "Should have 0 pending flags")
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.assertEnters(context: peer2, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
}
func testReceiveMachineListUpdateWhileReadyAndLocked() throws {
// Peer 1 becomes SOS+Octagon
self.putFakeKeyHierarchy(inCloudKit: self.manateeZoneID)
self.putSelfTLKShares(inCloudKit: self.manateeZoneID)
self.saveTLKMaterial(toKeychain: self.manateeZoneID)
XCTAssertTrue(OctagonPerformSOSUpgrade(), "SOS upgrade should be on")
self.startCKAccountStatusMock()
self.cuttlefishContext.startOctagonStateMachine()
let clique: OTClique
do {
clique = try OTClique.newFriends(withContextData: self.otcliqueContext, resetReason: .testGenerated)
XCTAssertNotNil(clique, "Clique should not be nil")
} catch {
XCTFail("Shouldn't have errored making new friends: \(error)")
throw error
}
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.assertConsidersSelfTrusted(context: self.cuttlefishContext)
assertAllCKKSViews(enter: SecCKKSZoneKeyStateReady, within: 10 * NSEC_PER_SEC)
// Peer 2 arrives (with a voucher), but is not on the trusted device list
let firstPeerID = clique.cliqueMemberIdentifier
XCTAssertNotNil(firstPeerID, "Clique should have a member identifier")
let bottle = self.fakeCuttlefishServer.state.bottles[0]
let entropy = try self.loadSecret(label: firstPeerID!)
XCTAssertNotNil(entropy, "entropy should not be nil")
let bNewOTCliqueContext = OTConfigurationContext()
bNewOTCliqueContext.context = "restoreB"
bNewOTCliqueContext.dsid = self.otcliqueContext.dsid
bNewOTCliqueContext.altDSID = self.otcliqueContext.altDSID
bNewOTCliqueContext.otControl = self.otcliqueContext.otControl
bNewOTCliqueContext.sbd = OTMockSecureBackup(bottleID: bottle.bottleID, entropy: entropy!)
let deviceBmockAuthKit = OTMockAuthKitAdapter(altDSID: self.otcliqueContext.altDSID,
machineID: "b-machine-id",
otherDevices: [self.mockAuthKit.currentMachineID])
let bRestoreContext = self.manager.context(forContainerName: OTCKContainerName,
contextID: bNewOTCliqueContext.context!,
sosAdapter: OTSOSMissingAdapter(),
authKitAdapter: deviceBmockAuthKit,
lockStateTracker: self.lockStateTracker,
accountStateTracker: self.accountStateTracker,
deviceInformationAdapter: self.makeInitiatorDeviceInfoAdapter())
bRestoreContext.startOctagonStateMachine()
self.sendContainerChange(context: bRestoreContext)
let bNewClique: OTClique
do {
bNewClique = try OTClique.performEscrowRecovery(withContextData: bNewOTCliqueContext, escrowArguments: [:])
XCTAssertNotNil(bNewClique, "bNewClique should not be nil")
} catch {
XCTFail("Shouldn't have errored recovering: \(error)")
throw error
}
self.assertEnters(context: bRestoreContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
// Device A notices, but doesn't update (because B isn't on the device list)
self.fakeCuttlefishServer.updateListener = { _ in
XCTFail("Should not have updated trust")
return nil
}
// Device A locks and gets the device list notification
self.aksLockState = true
self.lockStateTracker.recheck()
self.mockAuthKit.otherDevices.insert(deviceBmockAuthKit.currentMachineID)
self.cuttlefishContext.incompleteNotificationOfMachineIDListChange()
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateWaitForUnlock, within: 10 * NSEC_PER_SEC)
let updateTrustExpectation = self.expectation(description: "updateTrust")
self.fakeCuttlefishServer.updateListener = { request in
XCTAssertEqual(firstPeerID, request.peerID, "updateTrust request should be for ego peer ID")
XCTAssertTrue(request.hasDynamicInfoAndSig, "updateTrust request should have a dynamic info")
let newDynamicInfo = TPPeerDynamicInfo(data: request.dynamicInfoAndSig.peerDynamicInfo, sig: request.dynamicInfoAndSig.sig)
XCTAssertNotNil(newDynamicInfo, "should be able to make a dynamic info from protobuf")
XCTAssertEqual(newDynamicInfo?.includedPeerIDs.count, 2, "Should trust both peers")
updateTrustExpectation.fulfill()
return nil
}
self.sendContainerChange(context: self.cuttlefishContext)
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateWaitForUnlock, within: 10 * NSEC_PER_SEC)
// And on unlock, it should handle the update
self.aksLockState = false
self.lockStateTracker.recheck()
self.wait(for: [updateTrustExpectation], timeout: 30)
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
}
func testCKKSResetRecoverFromCKKSConflict() throws {
self.putFakeKeyHierarchy(inCloudKit: self.manateeZoneID)
self.putFakeDeviceStatus(inCloudKit: self.manateeZoneID)
// But do NOT add them to the keychain
// CKKS should get stuck in waitfortlk
self.startCKAccountStatusMock()
self.cuttlefishContext.startOctagonStateMachine()
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateUntrusted, within: 10 * NSEC_PER_SEC)
do {
let clique = try OTClique.newFriends(withContextData: self.otcliqueContext, resetReason: .testGenerated)
XCTAssertNotNil(clique, "Clique should not be nil")
} catch {
XCTFail("Shouldn't have errored making new friends: \(error)")
}
// Now, we should be in 'ready', and CKKS should be stuck
self.assertEnters(context: self.cuttlefishContext, state: OctagonStateReady, within: 10 * NSEC_PER_SEC)
self.assertConsidersSelfTrusted(context: self.cuttlefishContext)
self.assertAllCKKSViews(enter: SecCKKSZoneKeyStateWaitForTLK, within: 10 * NSEC_PER_SEC)
// Now, CKKS decides to reset the world, but a conflict occurs on hierarchy upload
self.silentZoneDeletesAllowed = true
var tlkUUIDs : [CKRecordZone.ID:String] = [:]
self.silentFetchesAllowed = false
self.expectCKFetchAndRun(beforeFinished: {
self.putFakeKeyHierarchy(inCloudKit: self.manateeZoneID)
self.putFakeDeviceStatus(inCloudKit: self.manateeZoneID)
self.silentFetchesAllowed = true
// Use the commented version below when multi-zone support is readded to the tets
tlkUUIDs[self.manateeZoneID!] = (self.keys![self.manateeZoneID!] as? ZoneKeys)?.tlk?.uuid
/*
for zoneID in self.ckksZones {
tlkUUIDs[zoneID as! CKRecordZone.ID] = (self.keys![zoneID] as? ZoneKeys)?.tlk?.uuid
}
*/
})
let resetExepctation = self.expectation(description: "reset callback is called")
self.cuttlefishContext.viewManager!.rpcResetCloudKit(nil, reason: "unit-test") {
error in
XCTAssertNil(error, "should be no error resetting cloudkit")
resetExepctation.fulfill()
}
// Deletions should occur, then the fetches, then get stuck (as we don't have the TLK)
self.wait(for: [resetExepctation], timeout: 10)
self.verifyDatabaseMocks()
// all subCKKSes should get stuck in waitfortlk
self.assertAllCKKSViews(enter: SecCKKSZoneKeyStateWaitForTLK, within: 10 * NSEC_PER_SEC)
self.verifyDatabaseMocks()
XCTAssertEqual(tlkUUIDs.count, self.ckksZones.count, "Should have the right number of conflicted TLKs")
for (zoneID,tlkUUID) in tlkUUIDs {
XCTAssertEqual(tlkUUID, (self.keys![zoneID] as? ZoneKeys)?.tlk?.uuid, "TLK should match conflicted version")
}
}
}
#endif