nmode's Git Repositories - signal-cli/blob - lib/src/main/java/org/asamk/signal/manager/helper/StorageHelper.java

   1 package org.asamk.signal.manager.helper;
   2
   3 import org.asamk.signal.manager.api.GroupIdV1;
   4 import org.asamk.signal.manager.api.GroupIdV2;
   5 import org.asamk.signal.manager.api.Profile;
   6 import org.asamk.signal.manager.internal.SignalDependencies;
   7 import org.asamk.signal.manager.storage.SignalAccount;
   8 import org.asamk.signal.manager.storage.recipients.RecipientId;
   9 import org.asamk.signal.manager.syncStorage.AccountRecordProcessor;
  10 import org.asamk.signal.manager.syncStorage.ContactRecordProcessor;
  11 import org.asamk.signal.manager.syncStorage.GroupV1RecordProcessor;
  12 import org.asamk.signal.manager.syncStorage.GroupV2RecordProcessor;
  13 import org.asamk.signal.manager.syncStorage.StorageSyncModels;
  14 import org.asamk.signal.manager.syncStorage.StorageSyncValidations;
  15 import org.asamk.signal.manager.syncStorage.WriteOperationResult;
  16 import org.asamk.signal.manager.util.KeyUtils;
  17 import org.signal.core.util.SetUtil;
  18 import org.signal.libsignal.protocol.InvalidKeyException;
  19 import org.slf4j.Logger;
  20 import org.slf4j.LoggerFactory;
  21 import org.whispersystems.signalservice.api.storage.RecordIkm;
  22 import org.whispersystems.signalservice.api.storage.SignalStorageManifest;
  23 import org.whispersystems.signalservice.api.storage.SignalStorageRecord;
  24 import org.whispersystems.signalservice.api.storage.StorageId;
  25 import org.whispersystems.signalservice.api.storage.StorageKey;
  26 import org.whispersystems.signalservice.api.storage.StorageRecordConvertersKt;
  27 import org.whispersystems.signalservice.api.storage.StorageServiceRepository;
  28 import org.whispersystems.signalservice.api.storage.StorageServiceRepository.ManifestIfDifferentVersionResult;
  29 import org.whispersystems.signalservice.api.storage.StorageServiceRepository.WriteStorageRecordsResult;
  30 import org.whispersystems.signalservice.internal.storage.protos.ManifestRecord;
  31 import org.whispersystems.signalservice.internal.storage.protos.StorageRecord;
  32
  33 import java.io.IOException;
  34 import java.sql.Connection;
  35 import java.sql.SQLException;
  36 import java.util.ArrayList;
  37 import java.util.Base64;
  38 import java.util.Collection;
  39 import java.util.Collections;
  40 import java.util.List;
  41 import java.util.Map;
  42 import java.util.stream.Collectors;
  43
  44 import static org.asamk.signal.manager.util.Utils.handleResponseException;
  45
  46 public class StorageHelper {
  47
  48     private static final Logger logger = LoggerFactory.getLogger(StorageHelper.class);
  49     private static final List<Integer> KNOWN_TYPES = List.of(ManifestRecord.Identifier.Type.CONTACT.getValue(),
  50             ManifestRecord.Identifier.Type.GROUPV1.getValue(),
  51             ManifestRecord.Identifier.Type.GROUPV2.getValue(),
  52             ManifestRecord.Identifier.Type.ACCOUNT.getValue());
  53
  54     private final SignalAccount account;
  55     private final SignalDependencies dependencies;
  56     private final Context context;
  57
  58     public StorageHelper(final Context context) {
  59         this.account = context.getAccount();
  60         this.dependencies = context.getDependencies();
  61         this.context = context;
  62     }
  63
  64     public void syncDataWithStorage() throws IOException {
  65         var storageKey = account.getOrCreateStorageKey();
  66         if (storageKey == null) {
  67             if (!account.isPrimaryDevice()) {
  68                 logger.debug("Storage key unknown, requesting from primary device.");
  69                 context.getSyncHelper().requestSyncKeys();
  70             }
  71             return;
  72         }
  73
  74         logger.trace("Reading manifest from remote storage");
  75         final var localManifestVersion = account.getStorageManifestVersion();
  76         final var localManifest = account.getStorageManifest().orElse(SignalStorageManifest.Companion.getEMPTY());
  77         final var storageServiceRepository = dependencies.getStorageServiceRepository();
  78         final var result = storageServiceRepository.getStorageManifestIfDifferentVersion(storageKey,
  79                 localManifestVersion);
  80
  81         var needsForcePush = false;
  82         final var remoteManifest = switch (result) {
  83             case ManifestIfDifferentVersionResult.DifferentVersion diff -> {
  84                 final var manifest = diff.getManifest();
  85                 storeManifestLocally(manifest);
  86                 yield manifest;
  87             }
  88             case ManifestIfDifferentVersionResult.DecryptionError ignore -> {
  89                 logger.warn("Manifest couldn't be decrypted.");
  90                 if (account.isPrimaryDevice()) {
  91                     needsForcePush = true;
  92                 } else {
  93                     context.getSyncHelper().requestSyncKeys();
  94                 }
  95                 yield null;
  96             }
  97             case ManifestIfDifferentVersionResult.SameVersion ignored -> localManifest;
  98             case ManifestIfDifferentVersionResult.NetworkError e -> throw e.getException();
  99             case ManifestIfDifferentVersionResult.StatusCodeError e -> throw e.getException();
 100             default -> throw new RuntimeException("Unhandled ManifestIfDifferentVersionResult type");
 101         };
 102
 103         if (remoteManifest != null) {
 104             logger.trace("Manifest versions: local {}, remote {}", localManifestVersion, remoteManifest.version);
 105
 106             if (remoteManifest.version > localManifestVersion) {
 107                 logger.trace("Remote version was newer, reading records.");
 108                 needsForcePush = readDataFromStorage(storageKey, localManifest, remoteManifest);
 109             } else if (remoteManifest.version < localManifest.version) {
 110                 logger.debug("Remote storage manifest version was older. User might have switched accounts.");
 111             }
 112             logger.trace("Done reading data from remote storage");
 113
 114             readRecordsWithPreviouslyUnknownTypes(storageKey, remoteManifest);
 115         }
 116
 117         logger.trace("Adding missing storageIds to local data");
 118         account.getRecipientStore().setMissingStorageIds();
 119         account.getGroupStore().setMissingStorageIds();
 120
 121         var needsMultiDeviceSync = false;
 122
 123         if (account.needsStorageKeyMigration()) {
 124             logger.debug("Storage needs force push due to new account entropy pool");
 125             // Set new aep and reset previous master key and storage key
 126             account.setAccountEntropyPool(account.getOrCreateAccountEntropyPool());
 127             storageKey = account.getOrCreateStorageKey();
 128             context.getSyncHelper().sendKeysMessage();
 129             needsForcePush = true;
 130         } else if (remoteManifest == null) {
 131             if (account.isPrimaryDevice()) {
 132                 needsForcePush = true;
 133             }
 134         } else if (remoteManifest.recordIkm == null && account.getSelfRecipientProfile()
 135                 .getCapabilities()
 136                 .contains(Profile.Capability.storageServiceEncryptionV2Capability)) {
 137             logger.debug("The SSRE2 capability is supported, but no recordIkm is set! Force pushing.");
 138             needsForcePush = true;
 139         } else {
 140             try {
 141                 needsMultiDeviceSync = writeToStorage(storageKey, remoteManifest, needsForcePush);
 142             } catch (RetryLaterException e) {
 143                 // TODO retry later
 144                 return;
 145             }
 146         }
 147
 148         if (needsForcePush) {
 149             logger.debug("Doing a force push.");
 150             try {
 151                 forcePushToStorage(storageKey);
 152                 needsMultiDeviceSync = true;
 153             } catch (RetryLaterException e) {
 154                 // TODO retry later
 155                 return;
 156             }
 157         }
 158
 159         if (needsMultiDeviceSync) {
 160             context.getSyncHelper().sendSyncFetchStorageMessage();
 161         }
 162
 163         logger.debug("Done syncing data with remote storage");
 164     }
 165
 166     public void forcePushToStorage() throws IOException {
 167         if (!account.isPrimaryDevice()) {
 168             return;
 169         }
 170
 171         final var storageKey = account.getOrCreateStorageKey();
 172         if (storageKey == null) {
 173             return;
 174         }
 175
 176         try {
 177             forcePushToStorage(storageKey);
 178         } catch (RetryLaterException e) {
 179             // TODO retry later
 180         }
 181     }
 182
 183     private boolean readDataFromStorage(
 184             final StorageKey storageKey,
 185             final SignalStorageManifest localManifest,
 186             final SignalStorageManifest remoteManifest
 187     ) throws IOException {
 188         var needsForcePush = false;
 189         try (final var connection = account.getAccountDatabase().getConnection()) {
 190             connection.setAutoCommit(false);
 191
 192             var idDifference = findIdDifference(remoteManifest.storageIds, localManifest.storageIds);
 193
 194             if (idDifference.hasTypeMismatches() && account.isPrimaryDevice()) {
 195                 logger.debug("Found type mismatches in the ID sets! Scheduling a force push after this sync completes.");
 196                 needsForcePush = true;
 197             }
 198
 199             logger.debug("Pre-Merge ID Difference :: {}", idDifference);
 200
 201             if (!idDifference.isEmpty()) {
 202                 final var remoteOnlyRecords = getSignalStorageRecords(storageKey,
 203                         remoteManifest,
 204                         idDifference.remoteOnlyIds());
 205
 206                 if (remoteOnlyRecords.size() != idDifference.remoteOnlyIds().size()) {
 207                     logger.debug(
 208                             "Could not find all remote-only records! Requested: {}, Found: {}. These stragglers should naturally get deleted during the sync.",
 209                             idDifference.remoteOnlyIds().size(),
 210                             remoteOnlyRecords.size());
 211                 }
 212
 213                 final var unknownInserts = processKnownRecords(connection, remoteOnlyRecords);
 214                 final var unknownDeletes = idDifference.localOnlyIds()
 215                         .stream()
 216                         .filter(id -> !KNOWN_TYPES.contains(id.getType()))
 217                         .toList();
 218
 219                 if (!idDifference.localOnlyIds().isEmpty()) {
 220                     final var updated = account.getRecipientStore()
 221                             .removeStorageIdsFromLocalOnlyUnregisteredRecipients(connection,
 222                                     idDifference.localOnlyIds());
 223
 224                     if (updated > 0) {
 225                         logger.warn(
 226                                 "Found {} records that were deleted remotely but only marked unregistered locally. Removed those from local store.",
 227                                 updated);
 228                     }
 229                 }
 230
 231                 logger.debug("Storage ids with unknown type: {} inserts, {} deletes",
 232                         unknownInserts.size(),
 233                         unknownDeletes.size());
 234
 235                 account.getUnknownStorageIdStore().addUnknownStorageIds(connection, unknownInserts);
 236                 account.getUnknownStorageIdStore().deleteUnknownStorageIds(connection, unknownDeletes);
 237             } else {
 238                 logger.debug("Remote version was newer, but there were no remote-only IDs.");
 239             }
 240             connection.commit();
 241         } catch (SQLException e) {
 242             throw new RuntimeException("Failed to sync remote storage", e);
 243         }
 244         return needsForcePush;
 245     }
 246
 247     private void readRecordsWithPreviouslyUnknownTypes(
 248             final StorageKey storageKey,
 249             final SignalStorageManifest remoteManifest
 250     ) throws IOException {
 251         try (final var connection = account.getAccountDatabase().getConnection()) {
 252             connection.setAutoCommit(false);
 253             final var knownUnknownIds = account.getUnknownStorageIdStore()
 254                     .getUnknownStorageIds(connection, KNOWN_TYPES);
 255
 256             if (!knownUnknownIds.isEmpty()) {
 257                 logger.debug("We have {} unknown records that we can now process.", knownUnknownIds.size());
 258
 259                 final var remote = getSignalStorageRecords(storageKey, remoteManifest, knownUnknownIds);
 260
 261                 logger.debug("Found {} of the known-unknowns remotely.", remote.size());
 262
 263                 processKnownRecords(connection, remote);
 264                 account.getUnknownStorageIdStore()
 265                         .deleteUnknownStorageIds(connection, remote.stream().map(SignalStorageRecord::getId).toList());
 266             }
 267             connection.commit();
 268         } catch (SQLException e) {
 269             throw new RuntimeException("Failed to sync remote storage", e);
 270         }
 271     }
 272
 273     private boolean writeToStorage(
 274             final StorageKey storageKey,
 275             final SignalStorageManifest remoteManifest,
 276             final boolean needsForcePush
 277     ) throws IOException, RetryLaterException {
 278         final WriteOperationResult remoteWriteOperation;
 279         try (final var connection = account.getAccountDatabase().getConnection()) {
 280             connection.setAutoCommit(false);
 281
 282             final var localStorageIds = getAllLocalStorageIds(connection);
 283             final var idDifference = findIdDifference(remoteManifest.storageIds, localStorageIds);
 284             logger.debug("ID Difference :: {}", idDifference);
 285
 286             final var remoteDeletes = idDifference.remoteOnlyIds().stream().map(StorageId::getRaw).toList();
 287             final var remoteInserts = buildLocalStorageRecords(connection, idDifference.localOnlyIds());
 288             // TODO check if local storage record proto matches remote, then reset to remote storage_id
 289
 290             remoteWriteOperation = new WriteOperationResult(new SignalStorageManifest(remoteManifest.version + 1,
 291                     account.getDeviceId(),
 292                     remoteManifest.recordIkm,
 293                     localStorageIds), remoteInserts, remoteDeletes);
 294
 295             connection.commit();
 296         } catch (SQLException e) {
 297             throw new RuntimeException("Failed to sync remote storage", e);
 298         }
 299
 300         if (remoteWriteOperation.isEmpty()) {
 301             logger.debug("No remote writes needed. Still at version: {}", remoteManifest.version);
 302             return false;
 303         }
 304
 305         logger.debug("We have something to write remotely.");
 306         logger.debug("WriteOperationResult :: {}", remoteWriteOperation);
 307
 308         StorageSyncValidations.validate(remoteWriteOperation,
 309                 remoteManifest,
 310                 needsForcePush,
 311                 account.getSelfRecipientAddress());
 312
 313         final var result = dependencies.getStorageServiceRepository()
 314                 .writeStorageRecords(storageKey,
 315                         remoteWriteOperation.manifest(),
 316                         remoteWriteOperation.inserts(),
 317                         remoteWriteOperation.deletes());
 318         switch (result) {
 319             case WriteStorageRecordsResult.ConflictError ignored -> {
 320                 logger.debug("Hit a conflict when trying to resolve the conflict! Retrying.");
 321                 throw new RetryLaterException();
 322             }
 323             case WriteStorageRecordsResult.NetworkError networkError -> throw networkError.getException();
 324             case WriteStorageRecordsResult.StatusCodeError statusCodeError -> throw statusCodeError.getException();
 325             case WriteStorageRecordsResult.Success ignored -> {
 326                 logger.debug("Saved new manifest. Now at version: {}", remoteWriteOperation.manifest().version);
 327                 storeManifestLocally(remoteWriteOperation.manifest());
 328                 return true;
 329             }
 330             default -> throw new IllegalStateException("Unexpected value: " + result);
 331         }
 332     }
 333
 334     private void forcePushToStorage(
 335             final StorageKey storageServiceKey
 336     ) throws IOException, RetryLaterException {
 337         logger.debug("Force pushing local state to remote storage");
 338
 339         final var currentVersion = handleResponseException(dependencies.getStorageServiceRepository()
 340                 .getManifestVersion());
 341         final var newVersion = currentVersion + 1;
 342         final var newStorageRecords = new ArrayList<SignalStorageRecord>();
 343         final Map<RecipientId, StorageId> newContactStorageIds;
 344         final Map<GroupIdV1, StorageId> newGroupV1StorageIds;
 345         final Map<GroupIdV2, StorageId> newGroupV2StorageIds;
 346
 347         try (final var connection = account.getAccountDatabase().getConnection()) {
 348             connection.setAutoCommit(false);
 349
 350             final var recipientIds = account.getRecipientStore().getRecipientIds(connection);
 351             newContactStorageIds = generateContactStorageIds(recipientIds);
 352             for (final var recipientId : recipientIds) {
 353                 final var storageId = newContactStorageIds.get(recipientId);
 354                 if (storageId.getType() == ManifestRecord.Identifier.Type.ACCOUNT.getValue()) {
 355                     final var recipient = account.getRecipientStore().getRecipient(connection, recipientId);
 356                     final var accountRecord = StorageSyncModels.localToRemoteRecord(connection,
 357                             account.getConfigurationStore(),
 358                             recipient,
 359                             account.getUsernameLink());
 360                     newStorageRecords.add(new SignalStorageRecord(storageId,
 361                             new StorageRecord.Builder().account(accountRecord).build()));
 362                 } else {
 363                     final var recipient = account.getRecipientStore().getRecipient(connection, recipientId);
 364                     final var address = recipient.getAddress().getIdentifier();
 365                     final var identity = account.getIdentityKeyStore().getIdentityInfo(connection, address);
 366                     final var record = StorageSyncModels.localToRemoteRecord(recipient, identity);
 367                     newStorageRecords.add(new SignalStorageRecord(storageId,
 368                             new StorageRecord.Builder().contact(record).build()));
 369                 }
 370             }
 371
 372             final var groupV1Ids = account.getGroupStore().getGroupV1Ids(connection);
 373             newGroupV1StorageIds = generateGroupV1StorageIds(groupV1Ids);
 374             for (final var groupId : groupV1Ids) {
 375                 final var storageId = newGroupV1StorageIds.get(groupId);
 376                 final var group = account.getGroupStore().getGroup(connection, groupId);
 377                 final var record = StorageSyncModels.localToRemoteRecord(group);
 378                 newStorageRecords.add(new SignalStorageRecord(storageId,
 379                         new StorageRecord.Builder().groupV1(record).build()));
 380             }
 381
 382             final var groupV2Ids = account.getGroupStore().getGroupV2Ids(connection);
 383             newGroupV2StorageIds = generateGroupV2StorageIds(groupV2Ids);
 384             for (final var groupId : groupV2Ids) {
 385                 final var storageId = newGroupV2StorageIds.get(groupId);
 386                 final var group = account.getGroupStore().getGroup(connection, groupId);
 387                 final var record = StorageSyncModels.localToRemoteRecord(group);
 388                 newStorageRecords.add(new SignalStorageRecord(storageId,
 389                         new StorageRecord.Builder().groupV2(record).build()));
 390             }
 391
 392             connection.commit();
 393         } catch (SQLException e) {
 394             throw new RuntimeException("Failed to sync remote storage", e);
 395         }
 396         final var newStorageIds = newStorageRecords.stream().map(SignalStorageRecord::getId).toList();
 397
 398         final RecordIkm recordIkm;
 399         if (account.getSelfRecipientProfile()
 400                 .getCapabilities()
 401                 .contains(Profile.Capability.storageServiceEncryptionV2Capability)) {
 402             logger.debug("Generating and including a new recordIkm.");
 403             recordIkm = RecordIkm.Companion.generate();
 404         } else {
 405             logger.debug("SSRE2 not yet supported. Not including recordIkm.");
 406             recordIkm = null;
 407         }
 408
 409         final var manifest = new SignalStorageManifest(newVersion, account.getDeviceId(), recordIkm, newStorageIds);
 410
 411         StorageSyncValidations.validateForcePush(manifest, newStorageRecords, account.getSelfRecipientAddress());
 412
 413         final WriteStorageRecordsResult result;
 414         if (newVersion > 1) {
 415             logger.trace("Force-pushing data. Inserting {} IDs.", newStorageRecords.size());
 416             result = dependencies.getStorageServiceRepository()
 417                     .resetAndWriteStorageRecords(storageServiceKey, manifest, newStorageRecords);
 418         } else {
 419             logger.trace("First version, normal push. Inserting {} IDs.", newStorageRecords.size());
 420             result = dependencies.getStorageServiceRepository()
 421                     .writeStorageRecords(storageServiceKey, manifest, newStorageRecords, Collections.emptyList());
 422         }
 423
 424         switch (result) {
 425             case WriteStorageRecordsResult.ConflictError ignored -> {
 426                 logger.debug("Hit a conflict. Trying again.");
 427                 throw new RetryLaterException();
 428             }
 429             case WriteStorageRecordsResult.NetworkError networkError -> throw networkError.getException();
 430             case WriteStorageRecordsResult.StatusCodeError statusCodeError -> throw statusCodeError.getException();
 431             case WriteStorageRecordsResult.Success ignored -> {
 432                 logger.debug("Force push succeeded. Updating local manifest version to: {}", manifest.version);
 433                 storeManifestLocally(manifest);
 434             }
 435             default -> throw new IllegalStateException("Unexpected value: " + result);
 436         }
 437
 438         try (final var connection = account.getAccountDatabase().getConnection()) {
 439             connection.setAutoCommit(false);
 440             account.getRecipientStore().updateStorageIds(connection, newContactStorageIds);
 441             account.getGroupStore().updateStorageIds(connection, newGroupV1StorageIds, newGroupV2StorageIds);
 442
 443             // delete all unknown storage ids
 444             account.getUnknownStorageIdStore().deleteAllUnknownStorageIds(connection);
 445             connection.commit();
 446         } catch (SQLException e) {
 447             throw new RuntimeException("Failed to sync remote storage", e);
 448         }
 449     }
 450
 451     private Map<RecipientId, StorageId> generateContactStorageIds(List<RecipientId> recipientIds) {
 452         final var selfRecipientId = account.getSelfRecipientId();
 453         return recipientIds.stream().collect(Collectors.toMap(recipientId -> recipientId, recipientId -> {
 454             if (recipientId.equals(selfRecipientId)) {
 455                 return StorageId.forAccount(KeyUtils.createRawStorageId());
 456             } else {
 457                 return StorageId.forContact(KeyUtils.createRawStorageId());
 458             }
 459         }));
 460     }
 461
 462     private Map<GroupIdV1, StorageId> generateGroupV1StorageIds(List<GroupIdV1> groupIds) {
 463         return groupIds.stream()
 464                 .collect(Collectors.toMap(recipientId -> recipientId,
 465                         recipientId -> StorageId.forGroupV1(KeyUtils.createRawStorageId())));
 466     }
 467
 468     private Map<GroupIdV2, StorageId> generateGroupV2StorageIds(List<GroupIdV2> groupIds) {
 469         return groupIds.stream()
 470                 .collect(Collectors.toMap(recipientId -> recipientId,
 471                         recipientId -> StorageId.forGroupV2(KeyUtils.createRawStorageId())));
 472     }
 473
 474     private void storeManifestLocally(
 475             final SignalStorageManifest remoteManifest
 476     ) {
 477         account.setStorageManifestVersion(remoteManifest.version);
 478         account.setStorageManifest(remoteManifest);
 479     }
 480
 481     private List<SignalStorageRecord> getSignalStorageRecords(
 482             final StorageKey storageKey,
 483             final SignalStorageManifest manifest,
 484             final List<StorageId> storageIds
 485     ) throws IOException {
 486         final var result = dependencies.getStorageServiceRepository()
 487                 .readStorageRecords(storageKey, manifest.recordIkm, storageIds);
 488         return switch (result) {
 489             case StorageServiceRepository.StorageRecordResult.DecryptionError decryptionError -> {
 490                 if (decryptionError.getException() instanceof InvalidKeyException) {
 491                     logger.warn("Failed to read storage records, ignoring.");
 492                     yield List.of();
 493                 } else if (decryptionError.getException() instanceof IOException ioe) {
 494                     throw ioe;
 495                 } else {
 496                     throw new IOException(decryptionError.getException());
 497                 }
 498             }
 499             case StorageServiceRepository.StorageRecordResult.NetworkError networkError ->
 500                     throw networkError.getException();
 501             case StorageServiceRepository.StorageRecordResult.StatusCodeError statusCodeError ->
 502                     throw statusCodeError.getException();
 503             case StorageServiceRepository.StorageRecordResult.Success success -> success.getRecords();
 504             default -> throw new IllegalStateException("Unexpected value: " + result);
 505         };
 506     }
 507
 508     private List<StorageId> getAllLocalStorageIds(final Connection connection) throws SQLException {
 509         final var storageIds = new ArrayList<StorageId>();
 510         storageIds.addAll(account.getUnknownStorageIdStore().getUnknownStorageIds(connection));
 511         storageIds.addAll(account.getGroupStore().getStorageIds(connection));
 512         storageIds.addAll(account.getRecipientStore().getStorageIds(connection));
 513         storageIds.add(account.getRecipientStore().getSelfStorageId(connection));
 514         return storageIds;
 515     }
 516
 517     private List<SignalStorageRecord> buildLocalStorageRecords(
 518             final Connection connection,
 519             final List<StorageId> storageIds
 520     ) throws SQLException {
 521         final var records = new ArrayList<SignalStorageRecord>(storageIds.size());
 522         for (final var storageId : storageIds) {
 523             final var record = buildLocalStorageRecord(connection, storageId);
 524             records.add(record);
 525         }
 526         return records;
 527     }
 528
 529     private SignalStorageRecord buildLocalStorageRecord(
 530             Connection connection,
 531             StorageId storageId
 532     ) throws SQLException {
 533         return switch (ManifestRecord.Identifier.Type.fromValue(storageId.getType())) {
 534             case ManifestRecord.Identifier.Type.CONTACT -> {
 535                 final var recipient = account.getRecipientStore().getRecipient(connection, storageId);
 536                 final var address = recipient.getAddress().getIdentifier();
 537                 final var identity = account.getIdentityKeyStore().getIdentityInfo(connection, address);
 538                 final var record = StorageSyncModels.localToRemoteRecord(recipient, identity);
 539                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().contact(record).build());
 540             }
 541             case ManifestRecord.Identifier.Type.GROUPV1 -> {
 542                 final var groupV1 = account.getGroupStore().getGroupV1(connection, storageId);
 543                 final var record = StorageSyncModels.localToRemoteRecord(groupV1);
 544                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().groupV1(record).build());
 545             }
 546             case ManifestRecord.Identifier.Type.GROUPV2 -> {
 547                 final var groupV2 = account.getGroupStore().getGroupV2(connection, storageId);
 548                 final var record = StorageSyncModels.localToRemoteRecord(groupV2);
 549                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().groupV2(record).build());
 550             }
 551             case ManifestRecord.Identifier.Type.ACCOUNT -> {
 552                 final var selfRecipient = account.getRecipientStore()
 553                         .getRecipient(connection, account.getSelfRecipientId());
 554
 555                 final var record = StorageSyncModels.localToRemoteRecord(connection,
 556                         account.getConfigurationStore(),
 557                         selfRecipient,
 558                         account.getUsernameLink());
 559                 yield new SignalStorageRecord(storageId, new StorageRecord.Builder().account(record).build());
 560             }
 561             case null, default -> {
 562                 throw new AssertionError("Got unknown local storage record type: " + storageId);
 563             }
 564         };
 565     }
 566
 567     /**
 568      * Given a list of all the local and remote keys you know about, this will
 569      * return a result telling
 570      * you which keys are exclusively remote and which are exclusively local.
 571      *
 572      * @param remoteIds All remote keys available.
 573      * @param localIds  All local keys available.
 574      * @return An object describing which keys are exclusive to the remote data set
 575      * and which keys are
 576      * exclusive to the local data set.
 577      */
 578     private static IdDifferenceResult findIdDifference(
 579             Collection<StorageId> remoteIds,
 580             Collection<StorageId> localIds
 581     ) {
 582         final var base64Encoder = Base64.getEncoder();
 583         final var remoteByRawId = remoteIds.stream()
 584                 .collect(Collectors.toMap(id -> base64Encoder.encodeToString(id.getRaw()), id -> id));
 585         final var localByRawId = localIds.stream()
 586                 .collect(Collectors.toMap(id -> base64Encoder.encodeToString(id.getRaw()), id -> id));
 587
 588         boolean hasTypeMismatch = remoteByRawId.size() != remoteIds.size() || localByRawId.size() != localIds.size();
 589
 590         final var remoteOnlyRawIds = SetUtil.difference(remoteByRawId.keySet(), localByRawId.keySet());
 591         final var localOnlyRawIds = SetUtil.difference(localByRawId.keySet(), remoteByRawId.keySet());
 592         final var sharedRawIds = SetUtil.intersection(localByRawId.keySet(), remoteByRawId.keySet());
 593
 594         for (String rawId : sharedRawIds) {
 595             final var remote = remoteByRawId.get(rawId);
 596             final var local = localByRawId.get(rawId);
 597
 598             if (remote.getType() != local.getType() && local.getType() != 0) {
 599                 remoteOnlyRawIds.remove(rawId);
 600                 localOnlyRawIds.remove(rawId);
 601                 hasTypeMismatch = true;
 602                 logger.debug("Remote type {} did not match local type {} for {}!",
 603                         remote.getType(),
 604                         local.getType(),
 605                         rawId);
 606             }
 607         }
 608
 609         final var remoteOnlyKeys = remoteOnlyRawIds.stream().map(remoteByRawId::get).toList();
 610         final var localOnlyKeys = localOnlyRawIds.stream().map(localByRawId::get).toList();
 611
 612         return new IdDifferenceResult(remoteOnlyKeys, localOnlyKeys, hasTypeMismatch);
 613     }
 614
 615     private List<StorageId> processKnownRecords(
 616             final Connection connection,
 617             List<SignalStorageRecord> records
 618     ) throws SQLException {
 619         final var unknownRecords = new ArrayList<StorageId>();
 620
 621         final var accountRecordProcessor = new AccountRecordProcessor(account, connection, context.getJobExecutor());
 622         final var contactRecordProcessor = new ContactRecordProcessor(account, connection, context.getJobExecutor());
 623         final var groupV1RecordProcessor = new GroupV1RecordProcessor(account, connection);
 624         final var groupV2RecordProcessor = new GroupV2RecordProcessor(account, connection);
 625
 626         for (final var record : records) {
 627             if (record.getProto().account != null) {
 628                 logger.debug("Reading record {} of type account", record.getId());
 629                 accountRecordProcessor.process(StorageRecordConvertersKt.toSignalAccountRecord(record.getProto().account,
 630                         record.getId()));
 631             } else if (record.getProto().groupV1 != null) {
 632                 logger.debug("Reading record {} of type groupV1", record.getId());
 633                 groupV1RecordProcessor.process(StorageRecordConvertersKt.toSignalGroupV1Record(record.getProto().groupV1,
 634                         record.getId()));
 635             } else if (record.getProto().groupV2 != null) {
 636                 logger.debug("Reading record {} of type groupV2", record.getId());
 637                 groupV2RecordProcessor.process(StorageRecordConvertersKt.toSignalGroupV2Record(record.getProto().groupV2,
 638                         record.getId()));
 639             } else if (record.getProto().contact != null) {
 640                 logger.debug("Reading record {} of type contact", record.getId());
 641                 contactRecordProcessor.process(StorageRecordConvertersKt.toSignalContactRecord(record.getProto().contact,
 642                         record.getId()));
 643             } else {
 644                 unknownRecords.add(record.getId());
 645             }
 646         }
 647
 648         return unknownRecords;
 649     }
 650
 651     /**
 652      * hasTypeMismatches is True if there exist some keys that have matching raw ID's but different types, otherwise false.
 653      */
 654     private record IdDifferenceResult(
 655             List<StorageId> remoteOnlyIds, List<StorageId> localOnlyIds, boolean hasTypeMismatches
 656     ) {
 657
 658         public boolean isEmpty() {
 659             return remoteOnlyIds.isEmpty() && localOnlyIds.isEmpty();
 660         }
 661     }
 662
 663     private static class RetryLaterException extends Throwable {}
 664 }