nmode's Git Repositories - signal-cli/blob - lib/src/main/java/org/asamk/signal/manager/helper/StorageHelper.java

   1 package org.asamk.signal.manager.helper;
   2
   3 import org.asamk.signal.manager.api.GroupIdV1;
   4 import org.asamk.signal.manager.api.GroupIdV2;
   5 import org.asamk.signal.manager.api.Profile;
   6 import org.asamk.signal.manager.internal.SignalDependencies;
   7 import org.asamk.signal.manager.storage.SignalAccount;
   8 import org.asamk.signal.manager.storage.recipients.RecipientId;
   9 import org.asamk.signal.manager.syncStorage.AccountRecordProcessor;
  10 import org.asamk.signal.manager.syncStorage.ContactRecordProcessor;
  11 import org.asamk.signal.manager.syncStorage.GroupV1RecordProcessor;
  12 import org.asamk.signal.manager.syncStorage.GroupV2RecordProcessor;
  13 import org.asamk.signal.manager.syncStorage.StorageSyncModels;
  14 import org.asamk.signal.manager.syncStorage.StorageSyncValidations;
  15 import org.asamk.signal.manager.syncStorage.WriteOperationResult;
  16 import org.asamk.signal.manager.util.KeyUtils;
  17 import org.signal.core.util.SetUtil;
  18 import org.signal.libsignal.protocol.InvalidKeyException;
  19 import org.slf4j.Logger;
  20 import org.slf4j.LoggerFactory;
  21 import org.whispersystems.signalservice.api.storage.RecordIkm;
  22 import org.whispersystems.signalservice.api.storage.SignalStorageManifest;
  23 import org.whispersystems.signalservice.api.storage.SignalStorageRecord;
  24 import org.whispersystems.signalservice.api.storage.StorageId;
  25 import org.whispersystems.signalservice.api.storage.StorageKey;
  26 import org.whispersystems.signalservice.api.storage.StorageRecordConvertersKt;
  27 import org.whispersystems.signalservice.api.storage.StorageServiceRepository;
  28 import org.whispersystems.signalservice.api.storage.StorageServiceRepository.ManifestIfDifferentVersionResult;
  29 import org.whispersystems.signalservice.api.storage.StorageServiceRepository.WriteStorageRecordsResult;
  30 import org.whispersystems.signalservice.internal.storage.protos.ManifestRecord;
  31 import org.whispersystems.signalservice.internal.storage.protos.StorageRecord;
  32
  33 import java.io.IOException;
  34 import java.sql.Connection;
  35 import java.sql.SQLException;
  36 import java.util.ArrayList;
  37 import java.util.Base64;
  38 import java.util.Collection;
  39 import java.util.Collections;
  40 import java.util.List;
  41 import java.util.Map;
  42 import java.util.stream.Collectors;
  43
  44 import static org.asamk.signal.manager.util.Utils.handleResponseException;
  45
  46 public class StorageHelper {
  47
  48     private static final Logger logger = LoggerFactory.getLogger(StorageHelper.class);
  49     private static final List<Integer> KNOWN_TYPES = List.of(ManifestRecord.Identifier.Type.CONTACT.getValue(),
  50             ManifestRecord.Identifier.Type.GROUPV1.getValue(),
  51             ManifestRecord.Identifier.Type.GROUPV2.getValue(),
  52             ManifestRecord.Identifier.Type.ACCOUNT.getValue());
  53
  54     private final SignalAccount account;
  55     private final SignalDependencies dependencies;
  56     private final Context context;
  57
  58     public StorageHelper(final Context context) {
  59         this.account = context.getAccount();
  60         this.dependencies = context.getDependencies();
  61         this.context = context;
  62     }
  63
  64     public void syncDataWithStorage() throws IOException {
  65         var storageKey = account.getOrCreateStorageKey();
  66         if (storageKey == null) {
  67             if (!account.isPrimaryDevice()) {
  68                 logger.debug("Storage key unknown, requesting from primary device.");
  69                 context.getSyncHelper().requestSyncKeys();
  70             }
  71             return;
  72         }
  73
  74         logger.trace("Reading manifest from remote storage");
  75         final var localManifestVersion = account.getStorageManifestVersion();
  76         final var localManifest = account.getStorageManifest().orElse(SignalStorageManifest.Companion.getEMPTY());
  77         final var storageServiceRepository = dependencies.getStorageServiceRepository();
  78         final var result = storageServiceRepository.getStorageManifestIfDifferentVersion(storageKey,
  79                 localManifestVersion);
  80
  81         var needsForcePush = false;
  82         final var remoteManifest = switch (result) {
  83             case ManifestIfDifferentVersionResult.DifferentVersion diff -> {
  84                 final var manifest = diff.getManifest();
  85                 storeManifestLocally(manifest);
  86                 yield manifest;
  87             }
  88             case ManifestIfDifferentVersionResult.DecryptionError ignore -> {
  89                 logger.warn("Manifest couldn't be decrypted.");
  90                 if (account.isPrimaryDevice()) {
  91                     needsForcePush = true;
  92                 } else {
  93                     context.getSyncHelper().requestSyncKeys();
  94                 }
  95                 yield null;
  96             }
  97             case ManifestIfDifferentVersionResult.SameVersion ignored -> localManifest;
  98             case ManifestIfDifferentVersionResult.NetworkError e -> throw e.getException();
  99             case ManifestIfDifferentVersionResult.StatusCodeError e -> throw e.getException();
 100             default -> throw new RuntimeException("Unhandled ManifestIfDifferentVersionResult type");
 101         };
 102
 103         if (remoteManifest != null) {
 104             logger.trace("Manifest versions: local {}, remote {}", localManifestVersion, remoteManifest.version);
 105
 106             if (remoteManifest.version > localManifestVersion) {
 107                 logger.trace("Remote version was newer, reading records.");
 108                 needsForcePush = readDataFromStorage(storageKey, localManifest, remoteManifest);
 109             } else if (remoteManifest.version < localManifest.version) {
 110                 logger.debug("Remote storage manifest version was older. User might have switched accounts.");
 111             }
 112             logger.trace("Done reading data from remote storage");
 113
 114             readRecordsWithPreviouslyUnknownTypes(storageKey, remoteManifest);
 115         }
 116
 117         logger.trace("Adding missing storageIds to local data");
 118         account.getRecipientStore().setMissingStorageIds();
 119         account.getGroupStore().setMissingStorageIds();
 120
 121         var needsMultiDeviceSync = false;
 122
 123         if (account.needsStorageKeyMigration()) {
 124             logger.debug("Storage needs force push due to new account entropy pool");
 125             // Set new aep and reset previous master key and storage key
 126             account.setAccountEntropyPool(account.getOrCreateAccountEntropyPool());
 127             storageKey = account.getOrCreateStorageKey();
 128             context.getSyncHelper().sendKeysMessage();
 129             needsForcePush = true;
 130         } else if (remoteManifest == null) {
 131             if (account.isPrimaryDevice()) {
 132                 needsForcePush = true;
 133             }
 134         } else if (remoteManifest.recordIkm == null && account.getSelfRecipientProfile()
 135                 .getCapabilities()
 136                 .contains(Profile.Capability.storageServiceEncryptionV2Capability)) {
 137             logger.debug("The SSRE2 capability is supported, but no recordIkm is set! Force pushing.");
 138             needsForcePush = true;
 139         } else {
 140             try {
 141                 needsMultiDeviceSync = writeToStorage(storageKey, remoteManifest, needsForcePush);
 142             } catch (RetryLaterException e) {
 143                 // TODO retry later
 144                 return;
 145             }
 146         }
 147
 148         if (needsForcePush) {
 149             logger.debug("Doing a force push.");
 150             try {
 151                 forcePushToStorage(storageKey);
 152                 needsMultiDeviceSync = true;
 153             } catch (RetryLaterException e) {
 154                 // TODO retry later
 155                 return;
 156             }
 157         }
 158
 159         if (needsMultiDeviceSync) {
 160             context.getSyncHelper().sendSyncFetchStorageMessage();
 161         }
 162
 163         logger.debug("Done syncing data with remote storage");
 164     }
 165
 166     public void forcePushToStorage() throws IOException {
 167         if (!account.isPrimaryDevice()) {
 168             return;
 169         }
 170
 171         final var storageKey = account.getOrCreateStorageKey();
 172         if (storageKey == null) {
 173             return;
 174         }
 175
 176         try {
 177             forcePushToStorage(storageKey);
 178         } catch (RetryLaterException e) {
 179             // TODO retry later
 180         }
 181     }
 182
 183     private boolean readDataFromStorage(
 184             final StorageKey storageKey,
 185             final SignalStorageManifest localManifest,
 186             final SignalStorageManifest remoteManifest
 187     ) throws IOException {
 188         var needsForcePush = false;
 189         try (final var connection = account.getAccountDatabase().getConnection()) {
 190             connection.setAutoCommit(false);
 191
 192             var idDifference = findIdDifference(remoteManifest.storageIds, localManifest.storageIds);
 193
 194             if (idDifference.hasTypeMismatches() && account.isPrimaryDevice()) {
 195                 logger.debug("Found type mismatches in the ID sets! Scheduling a force push after this sync completes.");
 196                 needsForcePush = true;
 197             }
 198
 199             logger.debug("Pre-Merge ID Difference :: {}", idDifference);
 200
 201             if (!idDifference.isEmpty()) {
 202                 final var remoteOnlyRecords = getSignalStorageRecords(storageKey,
 203                         remoteManifest,
 204                         idDifference.remoteOnlyIds());
 205
 206                 if (remoteOnlyRecords.size() != idDifference.remoteOnlyIds().size()) {
 207                     logger.debug(
 208                             "Could not find all remote-only records! Requested: {}, Found: {}. These stragglers should naturally get deleted during the sync.",
 209                             idDifference.remoteOnlyIds().size(),
 210                             remoteOnlyRecords.size());
 211                 }
 212
 213                 final var unknownInserts = processKnownRecords(connection, remoteOnlyRecords);
 214                 final var unknownDeletes = idDifference.localOnlyIds()
 215                         .stream()
 216                         .filter(id -> !KNOWN_TYPES.contains(id.getType()))
 217                         .toList();
 218
 219                 if (!idDifference.localOnlyIds().isEmpty()) {
 220                     final var updated = account.getRecipientStore()
 221                             .removeStorageIdsFromLocalOnlyUnregisteredRecipients(connection,
 222                                     idDifference.localOnlyIds());
 223
 224                     if (updated > 0) {
 225                         logger.warn(
 226                                 "Found {} records that were deleted remotely but only marked unregistered locally. Removed those from local store.",
 227                                 updated);
 228                     }
 229                 }
 230
 231                 logger.debug("Storage ids with unknown type: {} inserts, {} deletes",
 232                         unknownInserts.size(),
 233                         unknownDeletes.size());
 234
 235                 account.getUnknownStorageIdStore().addUnknownStorageIds(connection, unknownInserts);
 236                 account.getUnknownStorageIdStore().deleteUnknownStorageIds(connection, unknownDeletes);
 237             } else {
 238                 logger.debug("Remote version was newer, but there were no remote-only IDs.");
 239             }
 240             connection.commit();
 241         } catch (SQLException e) {
 242             throw new RuntimeException("Failed to sync remote storage", e);
 243         }
 244         return needsForcePush;
 245     }
 246
 247     private void readRecordsWithPreviouslyUnknownTypes(
 248             final StorageKey storageKey,
 249             final SignalStorageManifest remoteManifest
 250     ) throws IOException {
 251         try (final var connection = account.getAccountDatabase().getConnection()) {
 252             connection.setAutoCommit(false);
 253             final var knownUnknownIds = account.getUnknownStorageIdStore()
 254                     .getUnknownStorageIds(connection, KNOWN_TYPES);
 255
 256             if (!knownUnknownIds.isEmpty()) {
 257                 logger.debug("We have {} unknown records that we can now process.", knownUnknownIds.size());
 258
 259                 final var remote = getSignalStorageRecords(storageKey, remoteManifest, knownUnknownIds);
 260
 261                 logger.debug("Found {} of the known-unknowns remotely.", remote.size());
 262
 263                 processKnownRecords(connection, remote);
 264                 account.getUnknownStorageIdStore()
 265                         .deleteUnknownStorageIds(connection, remote.stream().map(SignalStorageRecord::getId).toList());
 266             }
 267             connection.commit();
 268         } catch (SQLException e) {
 269             throw new RuntimeException("Failed to sync remote storage", e);
 270         }
 271     }
 272
 273     private boolean writeToStorage(
 274             final StorageKey storageKey,
 275             final SignalStorageManifest remoteManifest,
 276             final boolean needsForcePush
 277     ) throws IOException, RetryLaterException {
 278         final WriteOperationResult remoteWriteOperation;
 279         try (final var connection = account.getAccountDatabase().getConnection()) {
 280             connection.setAutoCommit(false);
 281
 282             final var localStorageIds = getAllLocalStorageIds(connection);
 283             final var idDifference = findIdDifference(remoteManifest.storageIds, localStorageIds);
 284             logger.debug("ID Difference :: {}", idDifference);
 285
 286             final var remoteDeletes = idDifference.remoteOnlyIds().stream().map(StorageId::getRaw).toList();
 287             final var remoteInserts = buildLocalStorageRecords(connection, idDifference.localOnlyIds());
 288             // TODO check if local storage record proto matches remote, then reset to remote storage_id
 289
 290             remoteWriteOperation = new WriteOperationResult(new SignalStorageManifest(remoteManifest.version + 1,
 291                     account.getDeviceId(),
 292                     remoteManifest.recordIkm,
 293                     localStorageIds), remoteInserts, remoteDeletes);
 294
 295             connection.commit();
 296         } catch (SQLException e) {
 297             throw new RuntimeException("Failed to sync remote storage", e);
 298         }
 299
 300         if (remoteWriteOperation.isEmpty()) {
 301             logger.debug("No remote writes needed. Still at version: {}", remoteManifest.version);
 302             return false;
 303         }
 304
 305         logger.debug("We have something to write remotely.");
 306         logger.debug("WriteOperationResult :: {}", remoteWriteOperation);
 307
 308         StorageSyncValidations.validate(remoteWriteOperation,
 309                 remoteManifest,
 310                 needsForcePush,
 311                 account.getSelfRecipientAddress());
 312
 313         final var result = dependencies.getStorageServiceRepository()
 314                 .writeStorageRecords(storageKey,
 315                         remoteWriteOperation.manifest(),
 316                         remoteWriteOperation.inserts(),
 317                         remoteWriteOperation.deletes());
 318         switch (result) {
 319             case WriteStorageRecordsResult.ConflictError ignored -> {
 320                 logger.debug("Hit a conflict when trying to resolve the conflict! Retrying.");
 321                 throw new RetryLaterException();
 322             }
 323             case WriteStorageRecordsResult.NetworkError networkError -> throw networkError.getException();
 324             case WriteStorageRecordsResult.StatusCodeError statusCodeError -> throw statusCodeError.getException();
 325             case WriteStorageRecordsResult.Success ignored -> {
 326                 logger.debug("Saved new manifest. Now at version: {}", remoteWriteOperation.manifest().version);
 327                 storeManifestLocally(remoteWriteOperation.manifest());
 328                 return true;
 329             }
 330             default -> throw new IllegalStateException("Unexpected value: " + result);
 331         }
 332     }
 333
 334     private void forcePushToStorage(
 335             final StorageKey storageServiceKey
 336     ) throws IOException, RetryLaterException {
 337         logger.debug("Force pushing local state to remote storage");
 338
 339         final var currentVersion = handleResponseException(dependencies.getStorageServiceRepository()
 340                 .getManifestVersion());
 341         final var newVersion = currentVersion + 1;
 342         final var newStorageRecords = new ArrayList<SignalStorageRecord>();
 343         final Map<RecipientId, StorageId> newContactStorageIds;
 344         final Map<GroupIdV1, StorageId> newGroupV1StorageIds;
 345         final Map<GroupIdV2, StorageId> newGroupV2StorageIds;
 346
 347         try (final var connection = account.getAccountDatabase().getConnection()) {
 348             connection.setAutoCommit(false);
 349
 350             final var recipientIds = account.getRecipientStore().getRecipientIds(connection);
 351             newContactStorageIds = generateContactStorageIds(recipientIds);
 352             for (final var recipientId : recipientIds) {
 353                 final var storageId = newContactStorageIds.get(recipientId);
 354                 if (storageId.getType() == ManifestRecord.Identifier.Type.ACCOUNT.getValue()) {
 355                     final var recipient = account.getRecipientStore().getRecipient(connection, recipientId);
 356                     final var accountRecord = StorageSyncModels.localToRemoteRecord(account.getConfigurationStore(),
 357                             recipient,
 358                             account.getUsernameLink());
 359                     newStorageRecords.add(new SignalStorageRecord(storageId,
 360                             new StorageRecord.Builder().account(accountRecord).build()));
 361                 } else {
 362                     final var recipient = account.getRecipientStore().getRecipient(connection, recipientId);
 363                     final var address = recipient.getAddress().getIdentifier();
 364                     final var identity = account.getIdentityKeyStore().getIdentityInfo(connection, address);
 365                     final var record = StorageSyncModels.localToRemoteRecord(recipient, identity);
 366                     newStorageRecords.add(new SignalStorageRecord(storageId,
 367                             new StorageRecord.Builder().contact(record).build()));
 368                 }
 369             }
 370
 371             final var groupV1Ids = account.getGroupStore().getGroupV1Ids(connection);
 372             newGroupV1StorageIds = generateGroupV1StorageIds(groupV1Ids);
 373             for (final var groupId : groupV1Ids) {
 374                 final var storageId = newGroupV1StorageIds.get(groupId);
 375                 final var group = account.getGroupStore().getGroup(connection, groupId);
 376                 final var record = StorageSyncModels.localToRemoteRecord(group);
 377                 newStorageRecords.add(new SignalStorageRecord(storageId,
 378                         new StorageRecord.Builder().groupV1(record).build()));
 379             }
 380
 381             final var groupV2Ids = account.getGroupStore().getGroupV2Ids(connection);
 382             newGroupV2StorageIds = generateGroupV2StorageIds(groupV2Ids);
 383             for (final var groupId : groupV2Ids) {
 384                 final var storageId = newGroupV2StorageIds.get(groupId);
 385                 final var group = account.getGroupStore().getGroup(connection, groupId);
 386                 final var record = StorageSyncModels.localToRemoteRecord(group);
 387                 newStorageRecords.add(new SignalStorageRecord(storageId,
 388                         new StorageRecord.Builder().groupV2(record).build()));
 389             }
 390
 391             connection.commit();
 392         } catch (SQLException e) {
 393             throw new RuntimeException("Failed to sync remote storage", e);
 394         }
 395         final var newStorageIds = newStorageRecords.stream().map(SignalStorageRecord::getId).toList();
 396
 397         final RecordIkm recordIkm;
 398         if (account.getSelfRecipientProfile()
 399                 .getCapabilities()
 400                 .contains(Profile.Capability.storageServiceEncryptionV2Capability)) {
 401             logger.debug("Generating and including a new recordIkm.");
 402             recordIkm = RecordIkm.Companion.generate();
 403         } else {
 404             logger.debug("SSRE2 not yet supported. Not including recordIkm.");
 405             recordIkm = null;
 406         }
 407
 408         final var manifest = new SignalStorageManifest(newVersion, account.getDeviceId(), recordIkm, newStorageIds);
 409
 410         StorageSyncValidations.validateForcePush(manifest, newStorageRecords, account.getSelfRecipientAddress());
 411
 412         final WriteStorageRecordsResult result;
 413         if (newVersion > 1) {
 414             logger.trace("Force-pushing data. Inserting {} IDs.", newStorageRecords.size());
 415             result = dependencies.getStorageServiceRepository()
 416                     .resetAndWriteStorageRecords(storageServiceKey, manifest, newStorageRecords);
 417         } else {
 418             logger.trace("First version, normal push. Inserting {} IDs.", newStorageRecords.size());
 419             result = dependencies.getStorageServiceRepository()
 420                     .writeStorageRecords(storageServiceKey, manifest, newStorageRecords, Collections.emptyList());
 421         }
 422
 423         switch (result) {
 424             case WriteStorageRecordsResult.ConflictError ignored -> {
 425                 logger.debug("Hit a conflict. Trying again.");
 426                 throw new RetryLaterException();
 427             }
 428             case WriteStorageRecordsResult.NetworkError networkError -> throw networkError.getException();
 429             case WriteStorageRecordsResult.StatusCodeError statusCodeError -> throw statusCodeError.getException();
 430             case WriteStorageRecordsResult.Success ignored -> {
 431                 logger.debug("Force push succeeded. Updating local manifest version to: {}", manifest.version);
 432                 storeManifestLocally(manifest);
 433             }
 434             default -> throw new IllegalStateException("Unexpected value: " + result);
 435         }
 436
 437         try (final var connection = account.getAccountDatabase().getConnection()) {
 438             connection.setAutoCommit(false);
 439             account.getRecipientStore().updateStorageIds(connection, newContactStorageIds);
 440             account.getGroupStore().updateStorageIds(connection, newGroupV1StorageIds, newGroupV2StorageIds);
 441
 442             // delete all unknown storage ids
 443             account.getUnknownStorageIdStore().deleteAllUnknownStorageIds(connection);
 444             connection.commit();
 445         } catch (SQLException e) {
 446             throw new RuntimeException("Failed to sync remote storage", e);
 447         }
 448     }
 449
 450     private Map<RecipientId, StorageId> generateContactStorageIds(List<RecipientId> recipientIds) {
 451         final var selfRecipientId = account.getSelfRecipientId();
 452         return recipientIds.stream().collect(Collectors.toMap(recipientId -> recipientId, recipientId -> {
 453             if (recipientId.equals(selfRecipientId)) {
 454                 return StorageId.forAccount(KeyUtils.createRawStorageId());
 455             } else {
 456                 return StorageId.forContact(KeyUtils.createRawStorageId());
 457             }
 458         }));
 459     }
 460
 461     private Map<GroupIdV1, StorageId> generateGroupV1StorageIds(List<GroupIdV1> groupIds) {
 462         return groupIds.stream()
 463                 .collect(Collectors.toMap(recipientId -> recipientId,
 464                         recipientId -> StorageId.forGroupV1(KeyUtils.createRawStorageId())));
 465     }
 466
 467     private Map<GroupIdV2, StorageId> generateGroupV2StorageIds(List<GroupIdV2> groupIds) {
 468         return groupIds.stream()
 469                 .collect(Collectors.toMap(recipientId -> recipientId,
 470                         recipientId -> StorageId.forGroupV2(KeyUtils.createRawStorageId())));
 471     }
 472
 473     private void storeManifestLocally(
 474             final SignalStorageManifest remoteManifest
 475     ) {
 476         account.setStorageManifestVersion(remoteManifest.version);
 477         account.setStorageManifest(remoteManifest);
 478     }
 479
 480     private List<SignalStorageRecord> getSignalStorageRecords(
 481             final StorageKey storageKey,
 482             final SignalStorageManifest manifest,
 483             final List<StorageId> storageIds
 484     ) throws IOException {
 485         final var result = dependencies.getStorageServiceRepository()
 486                 .readStorageRecords(storageKey, manifest.recordIkm, storageIds);
 487         return switch (result) {
 488             case StorageServiceRepository.StorageRecordResult.DecryptionError decryptionError -> {
 489                 if (decryptionError.getException() instanceof InvalidKeyException) {
 490                     logger.warn("Failed to read storage records, ignoring.");
 491                     yield List.of();
 492                 } else if (decryptionError.getException() instanceof IOException ioe) {
 493                     throw ioe;
 494                 } else {
 495                     throw new IOException(decryptionError.getException());
 496                 }
 497             }
 498             case StorageServiceRepository.StorageRecordResult.NetworkError networkError ->
 499                     throw networkError.getException();
 500             case StorageServiceRepository.StorageRecordResult.StatusCodeError statusCodeError ->
 501                     throw statusCodeError.getException();
 502             case StorageServiceRepository.StorageRecordResult.Success success -> success.getRecords();
 503             default -> throw new IllegalStateException("Unexpected value: " + result);
 504         };
 505     }
 506
 507     private List<StorageId> getAllLocalStorageIds(final Connection connection) throws SQLException {
 508         final var storageIds = new ArrayList<StorageId>();
 509         storageIds.addAll(account.getUnknownStorageIdStore().getUnknownStorageIds(connection));
 510         storageIds.addAll(account.getGroupStore().getStorageIds(connection));
 511         storageIds.addAll(account.getRecipientStore().getStorageIds(connection));
 512         storageIds.add(account.getRecipientStore().getSelfStorageId(connection));
 513         return storageIds;
 514     }
 515
 516     private List<SignalStorageRecord> buildLocalStorageRecords(
 517             final Connection connection,
 518             final List<StorageId> storageIds
 519     ) throws SQLException {
 520         final var records = new ArrayList<SignalStorageRecord>(storageIds.size());
 521         for (final var storageId : storageIds) {
 522             final var record = buildLocalStorageRecord(connection, storageId);
 523             records.add(record);
 524         }
 525         return records;
 526     }
 527
 528     private SignalStorageRecord buildLocalStorageRecord(
 529             Connection connection,
 530             StorageId storageId
 531     ) throws SQLException {
 532         return switch (ManifestRecord.Identifier.Type.fromValue(storageId.getType())) {
 533             case ManifestRecord.Identifier.Type.CONTACT -> {
 534                 final var recipient = account.getRecipientStore().getRecipient(connection, storageId);
 535                 final var address = recipient.getAddress().getIdentifier();
 536                 final var identity = account.getIdentityKeyStore().getIdentityInfo(connection, address);
 537                 final var record = StorageSyncModels.localToRemoteRecord(recipient, identity);
 538                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().contact(record).build());
 539             }
 540             case ManifestRecord.Identifier.Type.GROUPV1 -> {
 541                 final var groupV1 = account.getGroupStore().getGroupV1(connection, storageId);
 542                 final var record = StorageSyncModels.localToRemoteRecord(groupV1);
 543                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().groupV1(record).build());
 544             }
 545             case ManifestRecord.Identifier.Type.GROUPV2 -> {
 546                 final var groupV2 = account.getGroupStore().getGroupV2(connection, storageId);
 547                 final var record = StorageSyncModels.localToRemoteRecord(groupV2);
 548                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().groupV2(record).build());
 549             }
 550             case ManifestRecord.Identifier.Type.ACCOUNT -> {
 551                 final var selfRecipient = account.getRecipientStore()
 552                         .getRecipient(connection, account.getSelfRecipientId());
 553
 554                 final var record = StorageSyncModels.localToRemoteRecord(account.getConfigurationStore(),
 555                         selfRecipient,
 556                         account.getUsernameLink());
 557                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().account(record).build());
 558             }
 559             case null, default -> {
 560                 throw new AssertionError("Got unknown local storage record type: " + storageId);
 561             }
 562         };
 563     }
 564
 565     /**
 566      * Given a list of all the local and remote keys you know about, this will
 567      * return a result telling
 568      * you which keys are exclusively remote and which are exclusively local.
 569      *
 570      * @param remoteIds All remote keys available.
 571      * @param localIds  All local keys available.
 572      * @return An object describing which keys are exclusive to the remote data set
 573      * and which keys are
 574      * exclusive to the local data set.
 575      */
 576     private static IdDifferenceResult findIdDifference(
 577             Collection<StorageId> remoteIds,
 578             Collection<StorageId> localIds
 579     ) {
 580         final var base64Encoder = Base64.getEncoder();
 581         final var remoteByRawId = remoteIds.stream()
 582                 .collect(Collectors.toMap(id -> base64Encoder.encodeToString(id.getRaw()), id -> id));
 583         final var localByRawId = localIds.stream()
 584                 .collect(Collectors.toMap(id -> base64Encoder.encodeToString(id.getRaw()), id -> id));
 585
 586         boolean hasTypeMismatch = remoteByRawId.size() != remoteIds.size() || localByRawId.size() != localIds.size();
 587
 588         final var remoteOnlyRawIds = SetUtil.difference(remoteByRawId.keySet(), localByRawId.keySet());
 589         final var localOnlyRawIds = SetUtil.difference(localByRawId.keySet(), remoteByRawId.keySet());
 590         final var sharedRawIds = SetUtil.intersection(localByRawId.keySet(), remoteByRawId.keySet());
 591
 592         for (String rawId : sharedRawIds) {
 593             final var remote = remoteByRawId.get(rawId);
 594             final var local = localByRawId.get(rawId);
 595
 596             if (remote.getType() != local.getType() && local.getType() != 0) {
 597                 remoteOnlyRawIds.remove(rawId);
 598                 localOnlyRawIds.remove(rawId);
 599                 hasTypeMismatch = true;
 600                 logger.debug("Remote type {} did not match local type {} for {}!",
 601                         remote.getType(),
 602                         local.getType(),
 603                         rawId);
 604             }
 605         }
 606
 607         final var remoteOnlyKeys = remoteOnlyRawIds.stream().map(remoteByRawId::get).toList();
 608         final var localOnlyKeys = localOnlyRawIds.stream().map(localByRawId::get).toList();
 609
 610         return new IdDifferenceResult(remoteOnlyKeys, localOnlyKeys, hasTypeMismatch);
 611     }
 612
 613     private List<StorageId> processKnownRecords(
 614             final Connection connection,
 615             List<SignalStorageRecord> records
 616     ) throws SQLException {
 617         final var unknownRecords = new ArrayList<StorageId>();
 618
 619         final var accountRecordProcessor = new AccountRecordProcessor(account, connection, context.getJobExecutor());
 620         final var contactRecordProcessor = new ContactRecordProcessor(account, connection, context.getJobExecutor());
 621         final var groupV1RecordProcessor = new GroupV1RecordProcessor(account, connection);
 622         final var groupV2RecordProcessor = new GroupV2RecordProcessor(account, connection);
 623
 624         for (final var record : records) {
 625             if (record.getProto().account != null) {
 626                 logger.debug("Reading record {} of type account", record.getId());
 627                 accountRecordProcessor.process(StorageRecordConvertersKt.toSignalAccountRecord(record.getProto().account,
 628                         record.getId()));
 629             } else if (record.getProto().groupV1 != null) {
 630                 logger.debug("Reading record {} of type groupV1", record.getId());
 631                 groupV1RecordProcessor.process(StorageRecordConvertersKt.toSignalGroupV1Record(record.getProto().groupV1,
 632                         record.getId()));
 633             } else if (record.getProto().groupV2 != null) {
 634                 logger.debug("Reading record {} of type groupV2", record.getId());
 635                 groupV2RecordProcessor.process(StorageRecordConvertersKt.toSignalGroupV2Record(record.getProto().groupV2,
 636                         record.getId()));
 637             } else if (record.getProto().contact != null) {
 638                 logger.debug("Reading record {} of type contact", record.getId());
 639                 contactRecordProcessor.process(StorageRecordConvertersKt.toSignalContactRecord(record.getProto().contact,
 640                         record.getId()));
 641             } else {
 642                 unknownRecords.add(record.getId());
 643             }
 644         }
 645
 646         return unknownRecords;
 647     }
 648
 649     /**
 650      * hasTypeMismatches is True if there exist some keys that have matching raw ID's but different types, otherwise false.
 651      */
 652     private record IdDifferenceResult(
 653             List<StorageId> remoteOnlyIds, List<StorageId> localOnlyIds, boolean hasTypeMismatches
 654     ) {
 655
 656         public boolean isEmpty() {
 657             return remoteOnlyIds.isEmpty() && localOnlyIds.isEmpty();
 658         }
 659     }
 660
 661     private static class RetryLaterException extends Throwable {}
 662 }