nmode's Git Repositories - signal-cli/blob - lib/src/main/java/org/asamk/signal/manager/helper/StorageHelper.java

   1 package org.asamk.signal.manager.helper;
   2
   3 import org.asamk.signal.manager.api.GroupIdV1;
   4 import org.asamk.signal.manager.api.GroupIdV2;
   5 import org.asamk.signal.manager.api.Profile;
   6 import org.asamk.signal.manager.internal.SignalDependencies;
   7 import org.asamk.signal.manager.storage.SignalAccount;
   8 import org.asamk.signal.manager.storage.recipients.RecipientId;
   9 import org.asamk.signal.manager.syncStorage.AccountRecordProcessor;
  10 import org.asamk.signal.manager.syncStorage.ContactRecordProcessor;
  11 import org.asamk.signal.manager.syncStorage.GroupV1RecordProcessor;
  12 import org.asamk.signal.manager.syncStorage.GroupV2RecordProcessor;
  13 import org.asamk.signal.manager.syncStorage.StorageSyncModels;
  14 import org.asamk.signal.manager.syncStorage.StorageSyncValidations;
  15 import org.asamk.signal.manager.syncStorage.WriteOperationResult;
  16 import org.asamk.signal.manager.util.KeyUtils;
  17 import org.signal.core.util.SetUtil;
  18 import org.signal.libsignal.protocol.InvalidKeyException;
  19 import org.slf4j.Logger;
  20 import org.slf4j.LoggerFactory;
  21 import org.whispersystems.signalservice.api.storage.RecordIkm;
  22 import org.whispersystems.signalservice.api.storage.SignalStorageManifest;
  23 import org.whispersystems.signalservice.api.storage.SignalStorageRecord;
  24 import org.whispersystems.signalservice.api.storage.StorageId;
  25 import org.whispersystems.signalservice.api.storage.StorageKey;
  26 import org.whispersystems.signalservice.api.storage.StorageRecordConvertersKt;
  27 import org.whispersystems.signalservice.api.storage.StorageServiceRepository;
  28 import org.whispersystems.signalservice.api.storage.StorageServiceRepository.ManifestIfDifferentVersionResult;
  29 import org.whispersystems.signalservice.api.storage.StorageServiceRepository.WriteStorageRecordsResult;
  30 import org.whispersystems.signalservice.internal.storage.protos.ManifestRecord;
  31 import org.whispersystems.signalservice.internal.storage.protos.StorageRecord;
  32
  33 import java.io.IOException;
  34 import java.sql.Connection;
  35 import java.sql.SQLException;
  36 import java.util.ArrayList;
  37 import java.util.Base64;
  38 import java.util.Collection;
  39 import java.util.Collections;
  40 import java.util.List;
  41 import java.util.Map;
  42 import java.util.stream.Collectors;
  43
  44 import static org.asamk.signal.manager.util.Utils.handleResponseException;
  45
  46 public class StorageHelper {
  47
  48     private static final Logger logger = LoggerFactory.getLogger(StorageHelper.class);
  49     private static final List<Integer> KNOWN_TYPES = List.of(ManifestRecord.Identifier.Type.CONTACT.getValue(),
  50             ManifestRecord.Identifier.Type.GROUPV1.getValue(),
  51             ManifestRecord.Identifier.Type.GROUPV2.getValue(),
  52             ManifestRecord.Identifier.Type.ACCOUNT.getValue());
  53
  54     private final SignalAccount account;
  55     private final SignalDependencies dependencies;
  56     private final Context context;
  57
  58     public StorageHelper(final Context context) {
  59         this.account = context.getAccount();
  60         this.dependencies = context.getDependencies();
  61         this.context = context;
  62     }
  63
  64     public void syncDataWithStorage() throws IOException {
  65         var storageKey = account.getOrCreateStorageKey();
  66         if (storageKey == null) {
  67             if (!account.isPrimaryDevice()) {
  68                 logger.debug("Storage key unknown, requesting from primary device.");
  69                 context.getSyncHelper().requestSyncKeys();
  70             }
  71             return;
  72         }
  73
  74         logger.trace("Reading manifest from remote storage");
  75         final var localManifestVersion = account.getStorageManifestVersion();
  76         final var localManifest = account.getStorageManifest().orElse(SignalStorageManifest.Companion.getEMPTY());
  77         final var storageServiceRepository = dependencies.getStorageServiceRepository();
  78         final var result = storageServiceRepository.getStorageManifestIfDifferentVersion(storageKey,
  79                 localManifestVersion);
  80
  81         var needsForcePush = false;
  82         final var remoteManifest = switch (result) {
  83             case ManifestIfDifferentVersionResult.DifferentVersion diff -> {
  84                 final var manifest = diff.getManifest();
  85                 storeManifestLocally(manifest);
  86                 yield manifest;
  87             }
  88             case ManifestIfDifferentVersionResult.DecryptionError ignore -> {
  89                 logger.warn("Manifest couldn't be decrypted.");
  90                 if (account.isPrimaryDevice()) {
  91                     needsForcePush = true;
  92                 } else {
  93                     context.getSyncHelper().requestSyncKeys();
  94                 }
  95                 yield null;
  96             }
  97             case ManifestIfDifferentVersionResult.SameVersion ignored -> localManifest;
  98             case ManifestIfDifferentVersionResult.NetworkError e -> throw e.getException();
  99             case ManifestIfDifferentVersionResult.StatusCodeError e -> throw e.getException();
 100             default -> throw new RuntimeException("Unhandled ManifestIfDifferentVersionResult type");
 101         };
 102
 103         if (remoteManifest != null) {
 104             logger.trace("Manifest versions: local {}, remote {}", localManifestVersion, remoteManifest.version);
 105
 106             if (remoteManifest.version > localManifestVersion) {
 107                 logger.trace("Remote version was newer, reading records.");
 108                 needsForcePush = readDataFromStorage(storageKey, localManifest, remoteManifest);
 109             } else if (remoteManifest.version < localManifest.version) {
 110                 logger.debug("Remote storage manifest version was older. User might have switched accounts.");
 111             }
 112             logger.trace("Done reading data from remote storage");
 113
 114             readRecordsWithPreviouslyUnknownTypes(storageKey, remoteManifest);
 115         }
 116
 117         logger.trace("Adding missing storageIds to local data");
 118         account.getRecipientStore().setMissingStorageIds();
 119         account.getGroupStore().setMissingStorageIds();
 120
 121         var needsMultiDeviceSync = false;
 122
 123         if (account.needsStorageKeyMigration()) {
 124             logger.debug("Storage needs force push due to new account entropy pool");
 125             // Set new aep and reset previous master key and storage key
 126             account.setAccountEntropyPool(account.getOrCreateAccountEntropyPool());
 127             storageKey = account.getOrCreateStorageKey();
 128             context.getSyncHelper().sendKeysMessage();
 129             needsForcePush = true;
 130         } else if (remoteManifest == null) {
 131             if (account.isPrimaryDevice()) {
 132                 needsForcePush = true;
 133             }
 134         } else if (remoteManifest.recordIkm == null && account.getSelfRecipientProfile()
 135                 .getCapabilities()
 136                 .contains(Profile.Capability.storageServiceEncryptionV2Capability)) {
 137             logger.debug("The SSRE2 capability is supported, but no recordIkm is set! Force pushing.");
 138             needsForcePush = true;
 139         } else {
 140             try {
 141                 needsMultiDeviceSync = writeToStorage(storageKey, remoteManifest, needsForcePush);
 142             } catch (RetryLaterException e) {
 143                 // TODO retry later
 144                 return;
 145             }
 146         }
 147
 148         if (needsForcePush) {
 149             logger.debug("Doing a force push.");
 150             try {
 151                 forcePushToStorage(storageKey);
 152                 needsMultiDeviceSync = true;
 153             } catch (RetryLaterException e) {
 154                 // TODO retry later
 155                 return;
 156             }
 157         }
 158
 159         if (needsMultiDeviceSync) {
 160             context.getSyncHelper().sendSyncFetchStorageMessage();
 161         }
 162
 163         logger.debug("Done syncing data with remote storage");
 164     }
 165
 166     public void forcePushToStorage() throws IOException {
 167         if (!account.isPrimaryDevice()) {
 168             return;
 169         }
 170
 171         final var storageKey = account.getOrCreateStorageKey();
 172         if (storageKey == null) {
 173             return;
 174         }
 175
 176         try {
 177             forcePushToStorage(storageKey);
 178         } catch (RetryLaterException e) {
 179             // TODO retry later
 180         }
 181     }
 182
 183     private boolean readDataFromStorage(
 184             final StorageKey storageKey,
 185             final SignalStorageManifest localManifest,
 186             final SignalStorageManifest remoteManifest
 187     ) throws IOException {
 188         var needsForcePush = false;
 189         try (final var connection = account.getAccountDatabase().getConnection()) {
 190             connection.setAutoCommit(false);
 191
 192             var idDifference = findIdDifference(remoteManifest.storageIds, localManifest.storageIds);
 193
 194             if (idDifference.hasTypeMismatches() && account.isPrimaryDevice()) {
 195                 logger.debug("Found type mismatches in the ID sets! Scheduling a force push after this sync completes.");
 196                 needsForcePush = true;
 197             }
 198
 199             logger.debug("Pre-Merge ID Difference :: {}", idDifference);
 200
 201             if (!idDifference.localOnlyIds().isEmpty()) {
 202                 final var updated = account.getRecipientStore()
 203                         .removeStorageIdsFromLocalOnlyUnregisteredRecipients(connection, idDifference.localOnlyIds());
 204
 205                 if (updated > 0) {
 206                     logger.warn(
 207                             "Found {} records that were deleted remotely but only marked unregistered locally. Removed those from local store.",
 208                             updated);
 209                 }
 210             }
 211
 212             if (!idDifference.isEmpty()) {
 213                 final var remoteOnlyRecords = getSignalStorageRecords(storageKey,
 214                         remoteManifest,
 215                         idDifference.remoteOnlyIds());
 216
 217                 if (remoteOnlyRecords.size() != idDifference.remoteOnlyIds().size()) {
 218                     logger.debug(
 219                             "Could not find all remote-only records! Requested: {}, Found: {}. These stragglers should naturally get deleted during the sync.",
 220                             idDifference.remoteOnlyIds().size(),
 221                             remoteOnlyRecords.size());
 222                 }
 223
 224                 final var unknownInserts = processKnownRecords(connection, remoteOnlyRecords);
 225                 final var unknownDeletes = idDifference.localOnlyIds()
 226                         .stream()
 227                         .filter(id -> !KNOWN_TYPES.contains(id.getType()))
 228                         .toList();
 229
 230                 logger.debug("Storage ids with unknown type: {} inserts, {} deletes",
 231                         unknownInserts.size(),
 232                         unknownDeletes.size());
 233
 234                 account.getUnknownStorageIdStore().addUnknownStorageIds(connection, unknownInserts);
 235                 account.getUnknownStorageIdStore().deleteUnknownStorageIds(connection, unknownDeletes);
 236             } else {
 237                 logger.debug("Remote version was newer, but there were no remote-only IDs.");
 238             }
 239             connection.commit();
 240         } catch (SQLException e) {
 241             throw new RuntimeException("Failed to sync remote storage", e);
 242         }
 243         return needsForcePush;
 244     }
 245
 246     private void readRecordsWithPreviouslyUnknownTypes(
 247             final StorageKey storageKey,
 248             final SignalStorageManifest remoteManifest
 249     ) throws IOException {
 250         try (final var connection = account.getAccountDatabase().getConnection()) {
 251             connection.setAutoCommit(false);
 252             final var knownUnknownIds = account.getUnknownStorageIdStore()
 253                     .getUnknownStorageIds(connection, KNOWN_TYPES);
 254
 255             if (!knownUnknownIds.isEmpty()) {
 256                 logger.debug("We have {} unknown records that we can now process.", knownUnknownIds.size());
 257
 258                 final var remote = getSignalStorageRecords(storageKey, remoteManifest, knownUnknownIds);
 259
 260                 logger.debug("Found {} of the known-unknowns remotely.", remote.size());
 261
 262                 processKnownRecords(connection, remote);
 263                 account.getUnknownStorageIdStore()
 264                         .deleteUnknownStorageIds(connection, remote.stream().map(SignalStorageRecord::getId).toList());
 265             }
 266             connection.commit();
 267         } catch (SQLException e) {
 268             throw new RuntimeException("Failed to sync remote storage", e);
 269         }
 270     }
 271
 272     private boolean writeToStorage(
 273             final StorageKey storageKey,
 274             final SignalStorageManifest remoteManifest,
 275             final boolean needsForcePush
 276     ) throws IOException, RetryLaterException {
 277         final WriteOperationResult remoteWriteOperation;
 278         try (final var connection = account.getAccountDatabase().getConnection()) {
 279             connection.setAutoCommit(false);
 280
 281             final var localStorageIds = getAllLocalStorageIds(connection);
 282             final var idDifference = findIdDifference(remoteManifest.storageIds, localStorageIds);
 283             logger.debug("ID Difference :: {}", idDifference);
 284
 285             final var remoteDeletes = idDifference.remoteOnlyIds().stream().map(StorageId::getRaw).toList();
 286             final var remoteInserts = buildLocalStorageRecords(connection, idDifference.localOnlyIds());
 287             // TODO check if local storage record proto matches remote, then reset to remote storage_id
 288
 289             remoteWriteOperation = new WriteOperationResult(new SignalStorageManifest(remoteManifest.version + 1,
 290                     account.getDeviceId(),
 291                     remoteManifest.recordIkm,
 292                     localStorageIds), remoteInserts, remoteDeletes);
 293
 294             connection.commit();
 295         } catch (SQLException e) {
 296             throw new RuntimeException("Failed to sync remote storage", e);
 297         }
 298
 299         if (remoteWriteOperation.isEmpty()) {
 300             logger.debug("No remote writes needed. Still at version: {}", remoteManifest.version);
 301             return false;
 302         }
 303
 304         logger.debug("We have something to write remotely.");
 305         logger.debug("WriteOperationResult :: {}", remoteWriteOperation);
 306
 307         StorageSyncValidations.validate(remoteWriteOperation,
 308                 remoteManifest,
 309                 needsForcePush,
 310                 account.getSelfRecipientAddress());
 311
 312         final var result = dependencies.getStorageServiceRepository()
 313                 .writeStorageRecords(storageKey,
 314                         remoteWriteOperation.manifest(),
 315                         remoteWriteOperation.inserts(),
 316                         remoteWriteOperation.deletes());
 317         switch (result) {
 318             case WriteStorageRecordsResult.ConflictError ignored -> {
 319                 logger.debug("Hit a conflict when trying to resolve the conflict! Retrying.");
 320                 throw new RetryLaterException();
 321             }
 322             case WriteStorageRecordsResult.NetworkError networkError -> throw networkError.getException();
 323             case WriteStorageRecordsResult.StatusCodeError statusCodeError -> throw statusCodeError.getException();
 324             case WriteStorageRecordsResult.Success ignored -> {
 325                 logger.debug("Saved new manifest. Now at version: {}", remoteWriteOperation.manifest().version);
 326                 storeManifestLocally(remoteWriteOperation.manifest());
 327                 return true;
 328             }
 329             default -> throw new IllegalStateException("Unexpected value: " + result);
 330         }
 331     }
 332
 333     private void forcePushToStorage(
 334             final StorageKey storageServiceKey
 335     ) throws IOException, RetryLaterException {
 336         logger.debug("Force pushing local state to remote storage");
 337
 338         final var currentVersion = handleResponseException(dependencies.getStorageServiceRepository()
 339                 .getManifestVersion());
 340         final var newVersion = currentVersion + 1;
 341         final var newStorageRecords = new ArrayList<SignalStorageRecord>();
 342         final Map<RecipientId, StorageId> newContactStorageIds;
 343         final Map<GroupIdV1, StorageId> newGroupV1StorageIds;
 344         final Map<GroupIdV2, StorageId> newGroupV2StorageIds;
 345
 346         try (final var connection = account.getAccountDatabase().getConnection()) {
 347             connection.setAutoCommit(false);
 348
 349             final var recipientIds = account.getRecipientStore().getRecipientIds(connection);
 350             newContactStorageIds = generateContactStorageIds(recipientIds);
 351             for (final var recipientId : recipientIds) {
 352                 final var storageId = newContactStorageIds.get(recipientId);
 353                 if (storageId.getType() == ManifestRecord.Identifier.Type.ACCOUNT.getValue()) {
 354                     final var recipient = account.getRecipientStore().getRecipient(connection, recipientId);
 355                     final var accountRecord = StorageSyncModels.localToRemoteRecord(account.getConfigurationStore(),
 356                             recipient,
 357                             account.getUsernameLink());
 358                     newStorageRecords.add(new SignalStorageRecord(storageId,
 359                             new StorageRecord.Builder().account(accountRecord).build()));
 360                 } else {
 361                     final var recipient = account.getRecipientStore().getRecipient(connection, recipientId);
 362                     final var address = recipient.getAddress().getIdentifier();
 363                     final var identity = account.getIdentityKeyStore().getIdentityInfo(connection, address);
 364                     final var record = StorageSyncModels.localToRemoteRecord(recipient, identity);
 365                     newStorageRecords.add(new SignalStorageRecord(storageId,
 366                             new StorageRecord.Builder().contact(record).build()));
 367                 }
 368             }
 369
 370             final var groupV1Ids = account.getGroupStore().getGroupV1Ids(connection);
 371             newGroupV1StorageIds = generateGroupV1StorageIds(groupV1Ids);
 372             for (final var groupId : groupV1Ids) {
 373                 final var storageId = newGroupV1StorageIds.get(groupId);
 374                 final var group = account.getGroupStore().getGroup(connection, groupId);
 375                 final var record = StorageSyncModels.localToRemoteRecord(group);
 376                 newStorageRecords.add(new SignalStorageRecord(storageId,
 377                         new StorageRecord.Builder().groupV1(record).build()));
 378             }
 379
 380             final var groupV2Ids = account.getGroupStore().getGroupV2Ids(connection);
 381             newGroupV2StorageIds = generateGroupV2StorageIds(groupV2Ids);
 382             for (final var groupId : groupV2Ids) {
 383                 final var storageId = newGroupV2StorageIds.get(groupId);
 384                 final var group = account.getGroupStore().getGroup(connection, groupId);
 385                 final var record = StorageSyncModels.localToRemoteRecord(group);
 386                 newStorageRecords.add(new SignalStorageRecord(storageId,
 387                         new StorageRecord.Builder().groupV2(record).build()));
 388             }
 389
 390             connection.commit();
 391         } catch (SQLException e) {
 392             throw new RuntimeException("Failed to sync remote storage", e);
 393         }
 394         final var newStorageIds = newStorageRecords.stream().map(SignalStorageRecord::getId).toList();
 395
 396         final RecordIkm recordIkm;
 397         if (account.getSelfRecipientProfile()
 398                 .getCapabilities()
 399                 .contains(Profile.Capability.storageServiceEncryptionV2Capability)) {
 400             logger.debug("Generating and including a new recordIkm.");
 401             recordIkm = RecordIkm.Companion.generate();
 402         } else {
 403             logger.debug("SSRE2 not yet supported. Not including recordIkm.");
 404             recordIkm = null;
 405         }
 406
 407         final var manifest = new SignalStorageManifest(newVersion, account.getDeviceId(), recordIkm, newStorageIds);
 408
 409         StorageSyncValidations.validateForcePush(manifest, newStorageRecords, account.getSelfRecipientAddress());
 410
 411         final WriteStorageRecordsResult result;
 412         if (newVersion > 1) {
 413             logger.trace("Force-pushing data. Inserting {} IDs.", newStorageRecords.size());
 414             result = dependencies.getStorageServiceRepository()
 415                     .resetAndWriteStorageRecords(storageServiceKey, manifest, newStorageRecords);
 416         } else {
 417             logger.trace("First version, normal push. Inserting {} IDs.", newStorageRecords.size());
 418             result = dependencies.getStorageServiceRepository()
 419                     .writeStorageRecords(storageServiceKey, manifest, newStorageRecords, Collections.emptyList());
 420         }
 421
 422         switch (result) {
 423             case WriteStorageRecordsResult.ConflictError ignored -> {
 424                 logger.debug("Hit a conflict. Trying again.");
 425                 throw new RetryLaterException();
 426             }
 427             case WriteStorageRecordsResult.NetworkError networkError -> throw networkError.getException();
 428             case WriteStorageRecordsResult.StatusCodeError statusCodeError -> throw statusCodeError.getException();
 429             case WriteStorageRecordsResult.Success ignored -> {
 430                 logger.debug("Force push succeeded. Updating local manifest version to: {}", manifest.version);
 431                 storeManifestLocally(manifest);
 432             }
 433             default -> throw new IllegalStateException("Unexpected value: " + result);
 434         }
 435
 436         try (final var connection = account.getAccountDatabase().getConnection()) {
 437             connection.setAutoCommit(false);
 438             account.getRecipientStore().updateStorageIds(connection, newContactStorageIds);
 439             account.getGroupStore().updateStorageIds(connection, newGroupV1StorageIds, newGroupV2StorageIds);
 440
 441             // delete all unknown storage ids
 442             account.getUnknownStorageIdStore().deleteAllUnknownStorageIds(connection);
 443             connection.commit();
 444         } catch (SQLException e) {
 445             throw new RuntimeException("Failed to sync remote storage", e);
 446         }
 447     }
 448
 449     private Map<RecipientId, StorageId> generateContactStorageIds(List<RecipientId> recipientIds) {
 450         final var selfRecipientId = account.getSelfRecipientId();
 451         return recipientIds.stream().collect(Collectors.toMap(recipientId -> recipientId, recipientId -> {
 452             if (recipientId.equals(selfRecipientId)) {
 453                 return StorageId.forAccount(KeyUtils.createRawStorageId());
 454             } else {
 455                 return StorageId.forContact(KeyUtils.createRawStorageId());
 456             }
 457         }));
 458     }
 459
 460     private Map<GroupIdV1, StorageId> generateGroupV1StorageIds(List<GroupIdV1> groupIds) {
 461         return groupIds.stream()
 462                 .collect(Collectors.toMap(recipientId -> recipientId,
 463                         recipientId -> StorageId.forGroupV1(KeyUtils.createRawStorageId())));
 464     }
 465
 466     private Map<GroupIdV2, StorageId> generateGroupV2StorageIds(List<GroupIdV2> groupIds) {
 467         return groupIds.stream()
 468                 .collect(Collectors.toMap(recipientId -> recipientId,
 469                         recipientId -> StorageId.forGroupV2(KeyUtils.createRawStorageId())));
 470     }
 471
 472     private void storeManifestLocally(
 473             final SignalStorageManifest remoteManifest
 474     ) {
 475         account.setStorageManifestVersion(remoteManifest.version);
 476         account.setStorageManifest(remoteManifest);
 477     }
 478
 479     private List<SignalStorageRecord> getSignalStorageRecords(
 480             final StorageKey storageKey,
 481             final SignalStorageManifest manifest,
 482             final List<StorageId> storageIds
 483     ) throws IOException {
 484         final var result = dependencies.getStorageServiceRepository()
 485                 .readStorageRecords(storageKey, manifest.recordIkm, storageIds);
 486         return switch (result) {
 487             case StorageServiceRepository.StorageRecordResult.DecryptionError decryptionError -> {
 488                 if (decryptionError.getException() instanceof InvalidKeyException) {
 489                     logger.warn("Failed to read storage records, ignoring.");
 490                     yield List.of();
 491                 } else if (decryptionError.getException() instanceof IOException ioe) {
 492                     throw ioe;
 493                 } else {
 494                     throw new IOException(decryptionError.getException());
 495                 }
 496             }
 497             case StorageServiceRepository.StorageRecordResult.NetworkError networkError ->
 498                     throw networkError.getException();
 499             case StorageServiceRepository.StorageRecordResult.StatusCodeError statusCodeError ->
 500                     throw statusCodeError.getException();
 501             case StorageServiceRepository.StorageRecordResult.Success success -> success.getRecords();
 502             default -> throw new IllegalStateException("Unexpected value: " + result);
 503         };
 504     }
 505
 506     private List<StorageId> getAllLocalStorageIds(final Connection connection) throws SQLException {
 507         final var storageIds = new ArrayList<StorageId>();
 508         storageIds.addAll(account.getUnknownStorageIdStore().getUnknownStorageIds(connection));
 509         storageIds.addAll(account.getGroupStore().getStorageIds(connection));
 510         storageIds.addAll(account.getRecipientStore().getStorageIds(connection));
 511         storageIds.add(account.getRecipientStore().getSelfStorageId(connection));
 512         return storageIds;
 513     }
 514
 515     private List<SignalStorageRecord> buildLocalStorageRecords(
 516             final Connection connection,
 517             final List<StorageId> storageIds
 518     ) throws SQLException {
 519         final var records = new ArrayList<SignalStorageRecord>(storageIds.size());
 520         for (final var storageId : storageIds) {
 521             final var record = buildLocalStorageRecord(connection, storageId);
 522             records.add(record);
 523         }
 524         return records;
 525     }
 526
 527     private SignalStorageRecord buildLocalStorageRecord(
 528             Connection connection,
 529             StorageId storageId
 530     ) throws SQLException {
 531         return switch (ManifestRecord.Identifier.Type.fromValue(storageId.getType())) {
 532             case ManifestRecord.Identifier.Type.CONTACT -> {
 533                 final var recipient = account.getRecipientStore().getRecipient(connection, storageId);
 534                 final var address = recipient.getAddress().getIdentifier();
 535                 final var identity = account.getIdentityKeyStore().getIdentityInfo(connection, address);
 536                 final var record = StorageSyncModels.localToRemoteRecord(recipient, identity);
 537                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().contact(record).build());
 538             }
 539             case ManifestRecord.Identifier.Type.GROUPV1 -> {
 540                 final var groupV1 = account.getGroupStore().getGroupV1(connection, storageId);
 541                 final var record = StorageSyncModels.localToRemoteRecord(groupV1);
 542                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().groupV1(record).build());
 543             }
 544             case ManifestRecord.Identifier.Type.GROUPV2 -> {
 545                 final var groupV2 = account.getGroupStore().getGroupV2(connection, storageId);
 546                 final var record = StorageSyncModels.localToRemoteRecord(groupV2);
 547                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().groupV2(record).build());
 548             }
 549             case ManifestRecord.Identifier.Type.ACCOUNT -> {
 550                 final var selfRecipient = account.getRecipientStore()
 551                         .getRecipient(connection, account.getSelfRecipientId());
 552
 553                 final var record = StorageSyncModels.localToRemoteRecord(account.getConfigurationStore(),
 554                         selfRecipient,
 555                         account.getUsernameLink());
 556                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().account(record).build());
 557             }
 558             case null, default -> {
 559                 throw new AssertionError("Got unknown local storage record type: " + storageId);
 560             }
 561         };
 562     }
 563
 564     /**
 565      * Given a list of all the local and remote keys you know about, this will
 566      * return a result telling
 567      * you which keys are exclusively remote and which are exclusively local.
 568      *
 569      * @param remoteIds All remote keys available.
 570      * @param localIds  All local keys available.
 571      * @return An object describing which keys are exclusive to the remote data set
 572      * and which keys are
 573      * exclusive to the local data set.
 574      */
 575     private static IdDifferenceResult findIdDifference(
 576             Collection<StorageId> remoteIds,
 577             Collection<StorageId> localIds
 578     ) {
 579         final var base64Encoder = Base64.getEncoder();
 580         final var remoteByRawId = remoteIds.stream()
 581                 .collect(Collectors.toMap(id -> base64Encoder.encodeToString(id.getRaw()), id -> id));
 582         final var localByRawId = localIds.stream()
 583                 .collect(Collectors.toMap(id -> base64Encoder.encodeToString(id.getRaw()), id -> id));
 584
 585         boolean hasTypeMismatch = remoteByRawId.size() != remoteIds.size() || localByRawId.size() != localIds.size();
 586
 587         final var remoteOnlyRawIds = SetUtil.difference(remoteByRawId.keySet(), localByRawId.keySet());
 588         final var localOnlyRawIds = SetUtil.difference(localByRawId.keySet(), remoteByRawId.keySet());
 589         final var sharedRawIds = SetUtil.intersection(localByRawId.keySet(), remoteByRawId.keySet());
 590
 591         for (String rawId : sharedRawIds) {
 592             final var remote = remoteByRawId.get(rawId);
 593             final var local = localByRawId.get(rawId);
 594
 595             if (remote.getType() != local.getType() && local.getType() != 0) {
 596                 remoteOnlyRawIds.remove(rawId);
 597                 localOnlyRawIds.remove(rawId);
 598                 hasTypeMismatch = true;
 599                 logger.debug("Remote type {} did not match local type {} for {}!",
 600                         remote.getType(),
 601                         local.getType(),
 602                         rawId);
 603             }
 604         }
 605
 606         final var remoteOnlyKeys = remoteOnlyRawIds.stream().map(remoteByRawId::get).toList();
 607         final var localOnlyKeys = localOnlyRawIds.stream().map(localByRawId::get).toList();
 608
 609         return new IdDifferenceResult(remoteOnlyKeys, localOnlyKeys, hasTypeMismatch);
 610     }
 611
 612     private List<StorageId> processKnownRecords(
 613             final Connection connection,
 614             List<SignalStorageRecord> records
 615     ) throws SQLException {
 616         final var unknownRecords = new ArrayList<StorageId>();
 617
 618         final var accountRecordProcessor = new AccountRecordProcessor(account, connection, context.getJobExecutor());
 619         final var contactRecordProcessor = new ContactRecordProcessor(account, connection, context.getJobExecutor());
 620         final var groupV1RecordProcessor = new GroupV1RecordProcessor(account, connection);
 621         final var groupV2RecordProcessor = new GroupV2RecordProcessor(account, connection);
 622
 623         for (final var record : records) {
 624             if (record.getProto().account != null) {
 625                 logger.debug("Reading record {} of type account", record.getId());
 626                 accountRecordProcessor.process(StorageRecordConvertersKt.toSignalAccountRecord(record.getProto().account,
 627                         record.getId()));
 628             } else if (record.getProto().groupV1 != null) {
 629                 logger.debug("Reading record {} of type groupV1", record.getId());
 630                 groupV1RecordProcessor.process(StorageRecordConvertersKt.toSignalGroupV1Record(record.getProto().groupV1,
 631                         record.getId()));
 632             } else if (record.getProto().groupV2 != null) {
 633                 logger.debug("Reading record {} of type groupV2", record.getId());
 634                 groupV2RecordProcessor.process(StorageRecordConvertersKt.toSignalGroupV2Record(record.getProto().groupV2,
 635                         record.getId()));
 636             } else if (record.getProto().contact != null) {
 637                 logger.debug("Reading record {} of type contact", record.getId());
 638                 contactRecordProcessor.process(StorageRecordConvertersKt.toSignalContactRecord(record.getProto().contact,
 639                         record.getId()));
 640             } else {
 641                 unknownRecords.add(record.getId());
 642             }
 643         }
 644
 645         return unknownRecords;
 646     }
 647
 648     /**
 649      * hasTypeMismatches is True if there exist some keys that have matching raw ID's but different types, otherwise false.
 650      */
 651     private record IdDifferenceResult(
 652             List<StorageId> remoteOnlyIds, List<StorageId> localOnlyIds, boolean hasTypeMismatches
 653     ) {
 654
 655         public boolean isEmpty() {
 656             return remoteOnlyIds.isEmpty() && localOnlyIds.isEmpty();
 657         }
 658     }
 659
 660     private static class RetryLaterException extends Throwable {}
 661 }