From: AsamK Date: Mon, 24 May 2021 14:51:36 +0000 (+0200) Subject: Implement retrieving data from remote storage X-Git-Tag: v0.9.0~18 X-Git-Url: https://git.nmode.ca/signal-cli/commitdiff_plain/2e01a05e7110b4f94abb10489a28b73d9f4be9c0 Implement retrieving data from remote storage Related #604 --- diff --git a/lib/src/main/java/org/asamk/signal/manager/Manager.java b/lib/src/main/java/org/asamk/signal/manager/Manager.java index 366fb371..a7a691cc 100644 --- a/lib/src/main/java/org/asamk/signal/manager/Manager.java +++ b/lib/src/main/java/org/asamk/signal/manager/Manager.java @@ -42,6 +42,7 @@ import org.asamk.signal.manager.helper.IncomingMessageHandler; import org.asamk.signal.manager.helper.PinHelper; import org.asamk.signal.manager.helper.ProfileHelper; import org.asamk.signal.manager.helper.SendHelper; +import org.asamk.signal.manager.helper.StorageHelper; import org.asamk.signal.manager.helper.SyncHelper; import org.asamk.signal.manager.helper.UnidentifiedAccessHelper; import org.asamk.signal.manager.jobs.Context; @@ -133,6 +134,7 @@ public class Manager implements Closeable { private final ProfileHelper profileHelper; private final PinHelper pinHelper; + private final StorageHelper storageHelper; private final SendHelper sendHelper; private final SyncHelper syncHelper; private final AttachmentHelper attachmentHelper; @@ -209,6 +211,7 @@ public class Manager implements Closeable { avatarStore, this::resolveSignalServiceAddress, account.getRecipientStore()); + this.storageHelper = new StorageHelper(account, dependencies, groupHelper); this.contactHelper = new ContactHelper(account); this.syncHelper = new SyncHelper(account, attachmentHelper, @@ -223,7 +226,8 @@ public class Manager implements Closeable { sendHelper, groupHelper, syncHelper, - profileHelper); + profileHelper, + storageHelper); var jobExecutor = new JobExecutor(context); this.incomingMessageHandler = new IncomingMessageHandler(account, @@ -747,6 +751,13 @@ public class Manager implements Closeable { public void requestAllSyncData() throws IOException { syncHelper.requestAllSyncData(); + retrieveRemoteStorage(); + } + + void retrieveRemoteStorage() throws IOException { + if (account.getStorageKey() != null) { + storageHelper.readDataFromStorage(); + } } private byte[] getSenderCertificate() { diff --git a/lib/src/main/java/org/asamk/signal/manager/ProvisioningManager.java b/lib/src/main/java/org/asamk/signal/manager/ProvisioningManager.java index 80c214f7..90dc6c66 100644 --- a/lib/src/main/java/org/asamk/signal/manager/ProvisioningManager.java +++ b/lib/src/main/java/org/asamk/signal/manager/ProvisioningManager.java @@ -103,6 +103,7 @@ public class ProvisioningManager { ? null : DeviceNameUtil.encryptDeviceName(deviceName, ret.getIdentity().getPrivateKey()); + logger.debug("Finishing new device registration"); var deviceId = accountManager.finishNewDeviceRegistration(ret.getProvisioningCode(), false, true, @@ -129,6 +130,7 @@ public class ProvisioningManager { try { m = new Manager(account, pathConfig, serviceEnvironmentConfig, userAgent); + logger.debug("Refreshing pre keys"); try { m.refreshPreKeys(); } catch (Exception e) { @@ -136,6 +138,7 @@ public class ProvisioningManager { throw e; } + logger.debug("Requesting sync data"); try { m.requestAllSyncData(); } catch (Exception e) { diff --git a/lib/src/main/java/org/asamk/signal/manager/RegistrationManager.java b/lib/src/main/java/org/asamk/signal/manager/RegistrationManager.java index 2be3f719..7cc0a7bc 100644 --- a/lib/src/main/java/org/asamk/signal/manager/RegistrationManager.java +++ b/lib/src/main/java/org/asamk/signal/manager/RegistrationManager.java @@ -174,7 +174,6 @@ public class RegistrationManager implements Closeable { masterKey = registrationLockData.getMasterKey(); } - // TODO response.isStorageCapable() //accountManager.setGcmId(Optional.of(GoogleCloudMessaging.getInstance(this).register(REGISTRATION_ID))); account.finishRegistration(UuidUtil.parseOrNull(response.getUuid()), masterKey, pin); @@ -186,6 +185,9 @@ public class RegistrationManager implements Closeable { m.refreshPreKeys(); // Set an initial empty profile so user can be added to groups m.setProfile(null, null, null, null, null); + if (response.isStorageCapable()) { + m.retrieveRemoteStorage(); + } final var result = m; m = null; diff --git a/lib/src/main/java/org/asamk/signal/manager/TrustLevel.java b/lib/src/main/java/org/asamk/signal/manager/TrustLevel.java index c9fa7a5e..5c712866 100644 --- a/lib/src/main/java/org/asamk/signal/manager/TrustLevel.java +++ b/lib/src/main/java/org/asamk/signal/manager/TrustLevel.java @@ -1,6 +1,7 @@ package org.asamk.signal.manager; import org.whispersystems.signalservice.api.messages.multidevice.VerifiedMessage; +import org.whispersystems.signalservice.internal.storage.protos.ContactRecord; public enum TrustLevel { UNTRUSTED, @@ -16,6 +17,20 @@ public enum TrustLevel { return TrustLevel.cachedValues[i]; } + public static TrustLevel fromIdentityState(ContactRecord.IdentityState identityState) { + switch (identityState) { + case DEFAULT: + return TRUSTED_UNVERIFIED; + case UNVERIFIED: + return UNTRUSTED; + case VERIFIED: + return TRUSTED_VERIFIED; + case UNRECOGNIZED: + return null; + } + throw new RuntimeException("Unknown identity state: " + identityState); + } + public static TrustLevel fromVerifiedState(VerifiedMessage.VerifiedState verifiedState) { switch (verifiedState) { case DEFAULT: diff --git a/lib/src/main/java/org/asamk/signal/manager/actions/RetrieveStorageDataAction.java b/lib/src/main/java/org/asamk/signal/manager/actions/RetrieveStorageDataAction.java new file mode 100644 index 00000000..6585a99a --- /dev/null +++ b/lib/src/main/java/org/asamk/signal/manager/actions/RetrieveStorageDataAction.java @@ -0,0 +1,26 @@ +package org.asamk.signal.manager.actions; + +import org.asamk.signal.manager.jobs.Context; + +public class RetrieveStorageDataAction implements HandleAction { + + private static final RetrieveStorageDataAction INSTANCE = new RetrieveStorageDataAction(); + + private RetrieveStorageDataAction() { + } + + public static RetrieveStorageDataAction create() { + return INSTANCE; + } + + @Override + public void execute(Context context) throws Throwable { + if (context.getAccount().getStorageKey() != null) { + context.getStorageHelper().readDataFromStorage(); + } else { + if (!context.getAccount().isMasterDevice()) { + context.getSyncHelper().requestAllSyncData(); + } + } + } +} diff --git a/lib/src/main/java/org/asamk/signal/manager/actions/SendSyncKeysAction.java b/lib/src/main/java/org/asamk/signal/manager/actions/SendSyncKeysAction.java new file mode 100644 index 00000000..fe609291 --- /dev/null +++ b/lib/src/main/java/org/asamk/signal/manager/actions/SendSyncKeysAction.java @@ -0,0 +1,20 @@ +package org.asamk.signal.manager.actions; + +import org.asamk.signal.manager.jobs.Context; + +public class SendSyncKeysAction implements HandleAction { + + private static final SendSyncKeysAction INSTANCE = new SendSyncKeysAction(); + + private SendSyncKeysAction() { + } + + public static SendSyncKeysAction create() { + return INSTANCE; + } + + @Override + public void execute(Context context) throws Throwable { + context.getSyncHelper().sendKeysMessage(); + } +} diff --git a/lib/src/main/java/org/asamk/signal/manager/helper/IncomingMessageHandler.java b/lib/src/main/java/org/asamk/signal/manager/helper/IncomingMessageHandler.java index e6e43478..e46effc0 100644 --- a/lib/src/main/java/org/asamk/signal/manager/helper/IncomingMessageHandler.java +++ b/lib/src/main/java/org/asamk/signal/manager/helper/IncomingMessageHandler.java @@ -8,12 +8,14 @@ import org.asamk.signal.manager.UntrustedIdentityException; import org.asamk.signal.manager.actions.HandleAction; import org.asamk.signal.manager.actions.RenewSessionAction; import org.asamk.signal.manager.actions.RetrieveProfileAction; +import org.asamk.signal.manager.actions.RetrieveStorageDataAction; import org.asamk.signal.manager.actions.SendGroupInfoAction; import org.asamk.signal.manager.actions.SendGroupInfoRequestAction; import org.asamk.signal.manager.actions.SendReceiptAction; import org.asamk.signal.manager.actions.SendSyncBlockedListAction; import org.asamk.signal.manager.actions.SendSyncContactsAction; import org.asamk.signal.manager.actions.SendSyncGroupsAction; +import org.asamk.signal.manager.actions.SendSyncKeysAction; import org.asamk.signal.manager.groups.GroupId; import org.asamk.signal.manager.groups.GroupNotFoundException; import org.asamk.signal.manager.groups.GroupUtils; @@ -237,7 +239,10 @@ public final class IncomingMessageHandler { if (rm.isBlockedListRequest()) { actions.add(SendSyncBlockedListAction.create()); } - // TODO Handle rm.isConfigurationRequest(); rm.isKeysRequest(); + if (rm.isKeysRequest()) { + actions.add(SendSyncKeysAction.create()); + } + // TODO Handle rm.isConfigurationRequest(); } if (syncMessage.getGroups().isPresent()) { logger.warn("Received a group v1 sync message, that can't be handled anymore, ignoring."); @@ -307,7 +312,7 @@ public final class IncomingMessageHandler { case LOCAL_PROFILE: actions.add(new RetrieveProfileAction(account.getSelfRecipientId())); case STORAGE_MANIFEST: - // TODO + actions.add(RetrieveStorageDataAction.create()); } } if (syncMessage.getKeys().isPresent()) { @@ -315,6 +320,7 @@ public final class IncomingMessageHandler { if (keysMessage.getStorageService().isPresent()) { final var storageKey = keysMessage.getStorageService().get(); account.setStorageKey(storageKey); + actions.add(RetrieveStorageDataAction.create()); } } if (syncMessage.getConfiguration().isPresent()) { diff --git a/lib/src/main/java/org/asamk/signal/manager/helper/StorageHelper.java b/lib/src/main/java/org/asamk/signal/manager/helper/StorageHelper.java new file mode 100644 index 00000000..4caab519 --- /dev/null +++ b/lib/src/main/java/org/asamk/signal/manager/helper/StorageHelper.java @@ -0,0 +1,222 @@ +package org.asamk.signal.manager.helper; + +import org.asamk.signal.manager.SignalDependencies; +import org.asamk.signal.manager.TrustLevel; +import org.asamk.signal.manager.groups.GroupId; +import org.asamk.signal.manager.storage.SignalAccount; +import org.asamk.signal.manager.storage.recipients.Contact; +import org.signal.zkgroup.InvalidInputException; +import org.signal.zkgroup.groups.GroupMasterKey; +import org.signal.zkgroup.profiles.ProfileKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.whispersystems.libsignal.IdentityKey; +import org.whispersystems.libsignal.InvalidKeyException; +import org.whispersystems.libsignal.util.guava.Optional; +import org.whispersystems.signalservice.api.storage.SignalAccountRecord; +import org.whispersystems.signalservice.api.storage.SignalStorageManifest; +import org.whispersystems.signalservice.api.storage.SignalStorageRecord; +import org.whispersystems.signalservice.api.storage.StorageId; +import org.whispersystems.signalservice.internal.storage.protos.ManifestRecord; + +import java.io.IOException; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +public class StorageHelper { + + private final static Logger logger = LoggerFactory.getLogger(StorageHelper.class); + + private final SignalAccount account; + private final SignalDependencies dependencies; + private final GroupHelper groupHelper; + + public StorageHelper( + final SignalAccount account, final SignalDependencies dependencies, final GroupHelper groupHelper + ) { + this.account = account; + this.dependencies = dependencies; + this.groupHelper = groupHelper; + } + + public void readDataFromStorage() throws IOException { + logger.debug("Reading data from remote storage"); + Optional manifest; + try { + manifest = dependencies.getAccountManager() + .getStorageManifestIfDifferentVersion(account.getStorageKey(), account.getStorageManifestVersion()); + } catch (InvalidKeyException e) { + logger.warn("Manifest couldn't be decrypted, ignoring."); + return; + } + + if (!manifest.isPresent()) { + logger.debug("Manifest is up to date, does not exist or couldn't be decrypted, ignoring."); + return; + } + + account.setStorageManifestVersion(manifest.get().getVersion()); + + readAccountRecord(manifest.get()); + + final var storageIds = manifest.get() + .getStorageIds() + .stream() + .filter(id -> !id.isUnknown() && id.getType() != ManifestRecord.Identifier.Type.ACCOUNT_VALUE) + .collect(Collectors.toList()); + + for (final var record : getSignalStorageRecords(storageIds)) { + if (record.getType() == ManifestRecord.Identifier.Type.GROUPV2_VALUE) { + readGroupV2Record(record); + } else if (record.getType() == ManifestRecord.Identifier.Type.GROUPV1_VALUE) { + readGroupV1Record(record); + } else if (record.getType() == ManifestRecord.Identifier.Type.CONTACT_VALUE) { + readContactRecord(record); + } + } + } + + private void readContactRecord(final SignalStorageRecord record) { + if (record == null || !record.getContact().isPresent()) { + return; + } + + final var contactRecord = record.getContact().get(); + final var address = contactRecord.getAddress(); + + final var recipientId = account.getRecipientStore().resolveRecipient(address); + final var contact = account.getContactStore().getContact(recipientId); + if (contactRecord.getGivenName().isPresent() || contactRecord.getFamilyName().isPresent() || ( + (contact == null || !contact.isBlocked()) && contactRecord.isBlocked() + )) { + final var newContact = (contact == null ? Contact.newBuilder() : Contact.newBuilder(contact)).withBlocked( + contactRecord.isBlocked()) + .withName((contactRecord.getGivenName().or("") + " " + contactRecord.getFamilyName().or("")).trim()) + .build(); + account.getContactStore().storeContact(recipientId, newContact); + } + + if (contactRecord.getProfileKey().isPresent()) { + try { + final var profileKey = new ProfileKey(contactRecord.getProfileKey().get()); + account.getProfileStore().storeProfileKey(recipientId, profileKey); + } catch (InvalidInputException e) { + logger.warn("Received invalid contact profile key from storage"); + } + } + if (contactRecord.getIdentityKey().isPresent()) { + try { + final var identityKey = new IdentityKey(contactRecord.getIdentityKey().get()); + account.getIdentityKeyStore().saveIdentity(recipientId, identityKey, new Date()); + + final var trustLevel = TrustLevel.fromIdentityState(contactRecord.getIdentityState()); + if (trustLevel != null) { + account.getIdentityKeyStore().setIdentityTrustLevel(recipientId, identityKey, trustLevel); + } + } catch (InvalidKeyException e) { + logger.warn("Received invalid contact identity key from storage"); + } + } + } + + private void readGroupV1Record(final SignalStorageRecord record) { + if (record == null || !record.getGroupV1().isPresent()) { + return; + } + + final var groupV1Record = record.getGroupV1().get(); + final var groupIdV1 = GroupId.v1(groupV1Record.getGroupId()); + + final var group = account.getGroupStore().getGroup(groupIdV1); + if (group == null) { + try { + groupHelper.sendGroupInfoRequest(groupIdV1, account.getSelfRecipientId()); + } catch (Throwable e) { + logger.warn("Failed to send group request", e); + } + } + final var groupV1 = account.getGroupStore().getOrCreateGroupV1(groupIdV1); + if (groupV1.isBlocked() != groupV1Record.isBlocked()) { + groupV1.setBlocked(groupV1Record.isBlocked()); + account.getGroupStore().updateGroup(groupV1); + } + } + + private void readGroupV2Record(final SignalStorageRecord record) { + if (record == null || !record.getGroupV2().isPresent()) { + return; + } + + final var groupV2Record = record.getGroupV2().get(); + if (groupV2Record.isArchived()) { + return; + } + + final GroupMasterKey groupMasterKey; + try { + groupMasterKey = new GroupMasterKey(groupV2Record.getMasterKeyBytes()); + } catch (InvalidInputException e) { + logger.warn("Received invalid group master key from storage"); + return; + } + + final var group = groupHelper.getOrMigrateGroup(groupMasterKey, 0, null); + if (group.isBlocked() != groupV2Record.isBlocked()) { + group.setBlocked(groupV2Record.isBlocked()); + account.getGroupStore().updateGroup(group); + } + } + + private void readAccountRecord(final SignalStorageManifest manifest) throws IOException { + Optional accountId = manifest.getAccountStorageId(); + if (!accountId.isPresent()) { + logger.warn("Manifest has no account record, ignoring."); + return; + } + + SignalStorageRecord record = getSignalStorageRecord(accountId.get()); + if (record == null) { + logger.warn("Could not find account record, even though we had an ID, ignoring."); + return; + } + + SignalAccountRecord accountRecord = record.getAccount().orNull(); + if (accountRecord == null) { + logger.warn("The storage record didn't actually have an account, ignoring."); + return; + } + + if (accountRecord.getProfileKey().isPresent()) { + try { + account.setProfileKey(new ProfileKey(accountRecord.getProfileKey().get())); + } catch (InvalidInputException e) { + logger.warn("Received invalid profile key from storage"); + } + } + } + + private SignalStorageRecord getSignalStorageRecord(final StorageId accountId) throws IOException { + List records; + try { + records = dependencies.getAccountManager() + .readStorageRecords(account.getStorageKey(), Collections.singletonList(accountId)); + } catch (InvalidKeyException e) { + logger.warn("Failed to read storage records, ignoring."); + return null; + } + return records.size() > 0 ? records.get(0) : null; + } + + private List getSignalStorageRecords(final List storageIds) throws IOException { + List records; + try { + records = dependencies.getAccountManager().readStorageRecords(account.getStorageKey(), storageIds); + } catch (InvalidKeyException e) { + logger.warn("Failed to read storage records, ignoring."); + return List.of(); + } + return records; + } +} diff --git a/lib/src/main/java/org/asamk/signal/manager/helper/SyncHelper.java b/lib/src/main/java/org/asamk/signal/manager/helper/SyncHelper.java index bcdf6ab1..6db1ca7d 100644 --- a/lib/src/main/java/org/asamk/signal/manager/helper/SyncHelper.java +++ b/lib/src/main/java/org/asamk/signal/manager/helper/SyncHelper.java @@ -20,6 +20,7 @@ import org.whispersystems.signalservice.api.messages.multidevice.DeviceContactsI import org.whispersystems.signalservice.api.messages.multidevice.DeviceContactsOutputStream; import org.whispersystems.signalservice.api.messages.multidevice.DeviceGroup; import org.whispersystems.signalservice.api.messages.multidevice.DeviceGroupsOutputStream; +import org.whispersystems.signalservice.api.messages.multidevice.KeysMessage; import org.whispersystems.signalservice.api.messages.multidevice.RequestMessage; import org.whispersystems.signalservice.api.messages.multidevice.SignalServiceSyncMessage; import org.whispersystems.signalservice.api.messages.multidevice.VerifiedMessage; @@ -215,6 +216,11 @@ public class SyncHelper { sendHelper.sendSyncMessage(SignalServiceSyncMessage.forVerified(verifiedMessage)); } + public void sendKeysMessage() throws IOException { + var keysMessage = new KeysMessage(Optional.fromNullable(account.getStorageKey())); + sendHelper.sendSyncMessage(SignalServiceSyncMessage.forKeys(keysMessage)); + } + public void handleSyncDeviceContacts(final InputStream input) throws IOException { final var s = new DeviceContactsInputStream(input); DeviceContact c; diff --git a/lib/src/main/java/org/asamk/signal/manager/jobs/Context.java b/lib/src/main/java/org/asamk/signal/manager/jobs/Context.java index 142c148a..beb41969 100644 --- a/lib/src/main/java/org/asamk/signal/manager/jobs/Context.java +++ b/lib/src/main/java/org/asamk/signal/manager/jobs/Context.java @@ -5,6 +5,7 @@ import org.asamk.signal.manager.StickerPackStore; import org.asamk.signal.manager.helper.GroupHelper; import org.asamk.signal.manager.helper.ProfileHelper; import org.asamk.signal.manager.helper.SendHelper; +import org.asamk.signal.manager.helper.StorageHelper; import org.asamk.signal.manager.helper.SyncHelper; import org.asamk.signal.manager.storage.SignalAccount; @@ -17,6 +18,7 @@ public class Context { private final GroupHelper groupHelper; private final SyncHelper syncHelper; private final ProfileHelper profileHelper; + private final StorageHelper storageHelper; public Context( final SignalAccount account, @@ -25,7 +27,8 @@ public class Context { final SendHelper sendHelper, final GroupHelper groupHelper, final SyncHelper syncHelper, - final ProfileHelper profileHelper + final ProfileHelper profileHelper, + final StorageHelper storageHelper ) { this.account = account; this.dependencies = dependencies; @@ -34,6 +37,7 @@ public class Context { this.groupHelper = groupHelper; this.syncHelper = syncHelper; this.profileHelper = profileHelper; + this.storageHelper = storageHelper; } public SignalAccount getAccount() { @@ -63,4 +67,8 @@ public class Context { public ProfileHelper getProfileHelper() { return profileHelper; } + + public StorageHelper getStorageHelper() { + return storageHelper; + } } diff --git a/lib/src/main/java/org/asamk/signal/manager/storage/SignalAccount.java b/lib/src/main/java/org/asamk/signal/manager/storage/SignalAccount.java index efdcf798..fd4ec597 100644 --- a/lib/src/main/java/org/asamk/signal/manager/storage/SignalAccount.java +++ b/lib/src/main/java/org/asamk/signal/manager/storage/SignalAccount.java @@ -84,6 +84,7 @@ public class SignalAccount implements Closeable { private String registrationLockPin; private MasterKey pinMasterKey; private StorageKey storageKey; + private long storageManifestVersion = -1; private ProfileKey profileKey; private int preKeyIdOffset; private int nextSignedPreKeyId; @@ -291,6 +292,9 @@ public class SignalAccount implements Closeable { this.registered = true; this.isMultiDevice = true; this.lastReceiveTimestamp = 0; + this.pinMasterKey = null; + this.storageManifestVersion = -1; + this.storageKey = null; } private void migrateLegacyConfigs() { @@ -432,6 +436,9 @@ public class SignalAccount implements Closeable { if (rootNode.hasNonNull("storageKey")) { storageKey = new StorageKey(Base64.getDecoder().decode(rootNode.get("storageKey").asText())); } + if (rootNode.hasNonNull("storageManifestVersion")) { + storageManifestVersion = rootNode.get("storageManifestVersion").asLong(); + } if (rootNode.hasNonNull("preKeyIdOffset")) { preKeyIdOffset = rootNode.get("preKeyIdOffset").asInt(0); } else { @@ -693,6 +700,7 @@ public class SignalAccount implements Closeable { pinMasterKey == null ? null : Base64.getEncoder().encodeToString(pinMasterKey.serialize())) .put("storageKey", storageKey == null ? null : Base64.getEncoder().encodeToString(storageKey.serialize())) + .put("storageManifestVersion", storageManifestVersion == -1 ? null : storageManifestVersion) .put("preKeyIdOffset", preKeyIdOffset) .put("nextSignedPreKeyId", nextSignedPreKeyId) .put("profileKey", @@ -877,6 +885,18 @@ public class SignalAccount implements Closeable { save(); } + public long getStorageManifestVersion() { + return this.storageManifestVersion; + } + + public void setStorageManifestVersion(final long storageManifestVersion) { + if (storageManifestVersion == this.storageManifestVersion) { + return; + } + this.storageManifestVersion = storageManifestVersion; + save(); + } + public ProfileKey getProfileKey() { return profileKey; } @@ -948,6 +968,8 @@ public class SignalAccount implements Closeable { public void finishRegistration(final UUID uuid, final MasterKey masterKey, final String pin) { this.pinMasterKey = masterKey; + this.storageManifestVersion = -1; + this.storageKey = null; this.encryptedDeviceName = null; this.deviceId = SignalServiceAddress.DEFAULT_DEVICE_ID; this.isMultiDevice = false;