From: AsamK Date: Tue, 7 Jun 2022 13:09:10 +0000 (+0200) Subject: Move sticker store to database X-Git-Tag: v0.11.0~25 X-Git-Url: https://git.nmode.ca/signal-cli/commitdiff_plain/9a698929f47d47d5cb63ad74ec4c1196f0cd91df?hp=862c2fec8707f87076233b0991e47e6c0b37dfad Move sticker store to database --- diff --git a/graalvm-config-dir/reflect-config.json b/graalvm-config-dir/reflect-config.json index 1ccbb0fd..62364e13 100644 --- a/graalvm-config-dir/reflect-config.json +++ b/graalvm-config-dir/reflect-config.json @@ -1210,23 +1210,25 @@ ] }, { - "name":"org.asamk.signal.manager.storage.stickers.StickerStore", + "name":"org.asamk.signal.manager.storage.stickers.LegacyStickerStore$Storage", "allDeclaredFields":true, - "allDeclaredMethods":true, - "allDeclaredConstructors":true, - "fields":[{"name":"stickers", "allowWrite":true}] + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true, + "methods":[{"name":"","parameterTypes":["java.util.List"] }] }, { - "name":"org.asamk.signal.manager.storage.stickers.StickerStore$Storage", + "name":"org.asamk.signal.manager.storage.stickers.LegacyStickerStore$Storage$Sticker", "allDeclaredFields":true, - "allDeclaredMethods":true, - "allDeclaredConstructors":true + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true, + "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","boolean"] }] }, { - "name":"org.asamk.signal.manager.storage.stickers.StickerStore$Storage$Sticker", + "name":"org.asamk.signal.manager.storage.stickers.StickerStore", "allDeclaredFields":true, "allDeclaredMethods":true, - "allDeclaredConstructors":true + "allDeclaredConstructors":true, + "fields":[{"name":"stickers", "allowWrite":true}] }, { "name":"org.asamk.signal.util.SecurityProvider$DefaultRandom", diff --git a/lib/src/main/java/org/asamk/signal/manager/ManagerImpl.java b/lib/src/main/java/org/asamk/signal/manager/ManagerImpl.java index 7b09aa3e..21d43fa5 100644 --- a/lib/src/main/java/org/asamk/signal/manager/ManagerImpl.java +++ b/lib/src/main/java/org/asamk/signal/manager/ManagerImpl.java @@ -32,7 +32,6 @@ import org.asamk.signal.manager.api.RecipientIdentifier; import org.asamk.signal.manager.api.SendGroupMessageResults; import org.asamk.signal.manager.api.SendMessageResult; import org.asamk.signal.manager.api.SendMessageResults; -import org.asamk.signal.manager.api.StickerPack; import org.asamk.signal.manager.api.StickerPackId; import org.asamk.signal.manager.api.StickerPackInvalidException; import org.asamk.signal.manager.api.StickerPackUrl; @@ -58,7 +57,7 @@ import org.asamk.signal.manager.storage.recipients.Recipient; import org.asamk.signal.manager.storage.recipients.RecipientId; import org.asamk.signal.manager.storage.stickerPacks.JsonStickerPack; import org.asamk.signal.manager.storage.stickerPacks.StickerPackStore; -import org.asamk.signal.manager.storage.stickers.Sticker; +import org.asamk.signal.manager.storage.stickers.StickerPack; import org.asamk.signal.manager.util.AttachmentUtils; import org.asamk.signal.manager.util.KeyUtils; import org.asamk.signal.manager.util.StickerUtils; @@ -580,7 +579,7 @@ class ManagerImpl implements Manager { if (stickerPack == null) { throw new InvalidStickerException("Sticker pack not found"); } - final var manifest = context.getStickerHelper().getOrRetrieveStickerPack(packId, stickerPack.getPackKey()); + final var manifest = context.getStickerHelper().getOrRetrieveStickerPack(packId, stickerPack.packKey()); if (manifest.stickers().size() <= stickerId) { throw new InvalidStickerException("Sticker id not part of this pack"); } @@ -590,7 +589,7 @@ class ManagerImpl implements Manager { throw new InvalidStickerException("Missing local sticker file"); } messageBuilder.withSticker(new SignalServiceDataMessage.Sticker(packId.serialize(), - stickerPack.getPackKey(), + stickerPack.packKey(), stickerId, manifestSticker.emoji(), AttachmentUtils.createAttachmentStream(streamDetails, Optional.empty()))); @@ -796,21 +795,21 @@ class ManagerImpl implements Manager { var packIdString = messageSender.uploadStickerManifest(manifest, packKey); var packId = StickerPackId.deserialize(Hex.fromStringCondensed(packIdString)); - var sticker = new Sticker(packId, packKey); - account.getStickerStore().updateSticker(sticker); + var sticker = new StickerPack(packId, packKey); + account.getStickerStore().addStickerPack(sticker); return new StickerPackUrl(packId, packKey); } @Override - public List getStickerPacks() { + public List getStickerPacks() { final var stickerPackStore = context.getStickerPackStore(); return account.getStickerStore().getStickerPacks().stream().map(pack -> { - if (stickerPackStore.existsStickerPack(pack.getPackId())) { + if (stickerPackStore.existsStickerPack(pack.packId())) { try { - final var manifest = stickerPackStore.retrieveManifest(pack.getPackId()); - return new StickerPack(pack.getPackId(), - new StickerPackUrl(pack.getPackId(), pack.getPackKey()), + final var manifest = stickerPackStore.retrieveManifest(pack.packId()); + return new org.asamk.signal.manager.api.StickerPack(pack.packId(), + new StickerPackUrl(pack.packId(), pack.packKey()), pack.isInstalled(), manifest.title(), manifest.author(), @@ -821,7 +820,7 @@ class ManagerImpl implements Manager { } } - return new StickerPack(pack.getPackId(), pack.getPackKey(), pack.isInstalled()); + return new org.asamk.signal.manager.api.StickerPack(pack.packId(), pack.packKey(), pack.isInstalled()); }).toList(); } diff --git a/lib/src/main/java/org/asamk/signal/manager/helper/IncomingMessageHandler.java b/lib/src/main/java/org/asamk/signal/manager/helper/IncomingMessageHandler.java index d89c5177..579d76e4 100644 --- a/lib/src/main/java/org/asamk/signal/manager/helper/IncomingMessageHandler.java +++ b/lib/src/main/java/org/asamk/signal/manager/helper/IncomingMessageHandler.java @@ -34,7 +34,7 @@ import org.asamk.signal.manager.storage.SignalAccount; import org.asamk.signal.manager.storage.groups.GroupInfoV1; import org.asamk.signal.manager.storage.recipients.Profile; import org.asamk.signal.manager.storage.recipients.RecipientId; -import org.asamk.signal.manager.storage.stickers.Sticker; +import org.asamk.signal.manager.storage.stickers.StickerPack; import org.asamk.signal.manager.util.KeyUtils; import org.signal.libsignal.metadata.ProtocolInvalidKeyException; import org.signal.libsignal.metadata.ProtocolInvalidKeyIdException; @@ -439,7 +439,8 @@ public final class IncomingMessageHandler { var sticker = account.getStickerStore().getStickerPack(stickerPackId); if (m.getPackKey().isPresent()) { if (sticker == null) { - sticker = new Sticker(stickerPackId, m.getPackKey().get()); + sticker = new StickerPack(-1, stickerPackId, m.getPackKey().get(), installed); + account.getStickerStore().addStickerPack(sticker); } if (installed) { context.getJobExecutor() @@ -447,9 +448,8 @@ public final class IncomingMessageHandler { } } - if (sticker != null) { - sticker.setInstalled(installed); - account.getStickerStore().updateSticker(sticker); + if (sticker != null && sticker.isInstalled() != installed) { + account.getStickerStore().updateStickerPackInstalled(sticker.packId(), installed); } } } @@ -703,8 +703,8 @@ public final class IncomingMessageHandler { final var stickerPackId = StickerPackId.deserialize(messageSticker.getPackId()); var sticker = account.getStickerStore().getStickerPack(stickerPackId); if (sticker == null) { - sticker = new Sticker(stickerPackId, messageSticker.getPackKey()); - account.getStickerStore().updateSticker(sticker); + sticker = new StickerPack(stickerPackId, messageSticker.getPackKey()); + account.getStickerStore().addStickerPack(sticker); } context.getJobExecutor().enqueueJob(new RetrieveStickerPackJob(stickerPackId, messageSticker.getPackKey())); } diff --git a/lib/src/main/java/org/asamk/signal/manager/storage/AccountDatabase.java b/lib/src/main/java/org/asamk/signal/manager/storage/AccountDatabase.java index 9c504b47..df7f3019 100644 --- a/lib/src/main/java/org/asamk/signal/manager/storage/AccountDatabase.java +++ b/lib/src/main/java/org/asamk/signal/manager/storage/AccountDatabase.java @@ -4,6 +4,7 @@ import com.zaxxer.hikari.HikariDataSource; import org.asamk.signal.manager.storage.recipients.RecipientStore; import org.asamk.signal.manager.storage.sendLog.MessageSendLogStore; +import org.asamk.signal.manager.storage.stickers.StickerStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,7 +15,7 @@ import java.sql.SQLException; public class AccountDatabase extends Database { private final static Logger logger = LoggerFactory.getLogger(AccountDatabase.class); - private static final long DATABASE_VERSION = 2; + private static final long DATABASE_VERSION = 3; private AccountDatabase(final HikariDataSource dataSource) { super(logger, DATABASE_VERSION, dataSource); @@ -28,6 +29,7 @@ public class AccountDatabase extends Database { protected void createDatabase(final Connection connection) throws SQLException { RecipientStore.createSql(connection); MessageSendLogStore.createSql(connection); + StickerStore.createSql(connection); } @Override @@ -65,5 +67,18 @@ public class AccountDatabase extends Database { """); } } + if (oldVersion < 3) { + logger.debug("Updating database: Creating sticker table"); + try (final var statement = connection.createStatement()) { + statement.executeUpdate(""" + CREATE TABLE sticker ( + _id INTEGER PRIMARY KEY, + pack_id BLOB UNIQUE NOT NULL, + pack_key BLOB NOT NULL, + installed BOOLEAN NOT NULL DEFAULT FALSE + ); + """); + } + } } } diff --git a/lib/src/main/java/org/asamk/signal/manager/storage/SignalAccount.java b/lib/src/main/java/org/asamk/signal/manager/storage/SignalAccount.java index 3761c5c4..5e10f724 100644 --- a/lib/src/main/java/org/asamk/signal/manager/storage/SignalAccount.java +++ b/lib/src/main/java/org/asamk/signal/manager/storage/SignalAccount.java @@ -36,6 +36,7 @@ import org.asamk.signal.manager.storage.recipients.RecipientTrustedResolver; import org.asamk.signal.manager.storage.sendLog.MessageSendLogStore; import org.asamk.signal.manager.storage.senderKeys.SenderKeyStore; import org.asamk.signal.manager.storage.sessions.SessionStore; +import org.asamk.signal.manager.storage.stickers.LegacyStickerStore; import org.asamk.signal.manager.storage.stickers.StickerStore; import org.asamk.signal.manager.storage.threads.LegacyJsonThreadStore; import org.asamk.signal.manager.util.IOUtils; @@ -146,7 +147,6 @@ public class SignalAccount implements Closeable { private GroupStore.Storage groupStoreStorage; private RecipientStore recipientStore; private StickerStore stickerStore; - private StickerStore.Storage stickerStoreStorage; private ConfigurationStore configurationStore; private ConfigurationStore.Storage configurationStoreStorage; @@ -216,7 +216,6 @@ public class SignalAccount implements Closeable { signalAccount.groupStore = new GroupStore(getGroupCachePath(dataPath, accountPath), signalAccount.getRecipientResolver(), signalAccount::saveGroupStore); - signalAccount.stickerStore = new StickerStore(signalAccount::saveStickerStore); signalAccount.configurationStore = new ConfigurationStore(signalAccount::saveConfigurationStore); signalAccount.registered = false; @@ -341,7 +340,6 @@ public class SignalAccount implements Closeable { signalAccount.groupStore = new GroupStore(getGroupCachePath(dataPath, accountPath), signalAccount.getRecipientResolver(), signalAccount::saveGroupStore); - signalAccount.stickerStore = new StickerStore(signalAccount::saveStickerStore); signalAccount.configurationStore = new ConfigurationStore(signalAccount::saveConfigurationStore); signalAccount.getRecipientTrustedResolver() @@ -659,10 +657,10 @@ public class SignalAccount implements Closeable { } if (rootNode.hasNonNull("stickerStore")) { - stickerStoreStorage = jsonProcessor.convertValue(rootNode.get("stickerStore"), StickerStore.Storage.class); - stickerStore = StickerStore.fromStorage(stickerStoreStorage, this::saveStickerStore); - } else { - stickerStore = new StickerStore(this::saveStickerStore); + final var storage = jsonProcessor.convertValue(rootNode.get("stickerStore"), + LegacyStickerStore.Storage.class); + LegacyStickerStore.migrate(storage, getStickerStore()); + migratedLegacyConfig = true; } if (rootNode.hasNonNull("configurationStore")) { @@ -853,11 +851,6 @@ public class SignalAccount implements Closeable { return false; } - private void saveStickerStore(StickerStore.Storage storage) { - this.stickerStoreStorage = storage; - save(); - } - private void saveGroupStore(GroupStore.Storage storage) { this.groupStoreStorage = storage; save(); @@ -910,7 +903,6 @@ public class SignalAccount implements Closeable { profileKey == null ? null : Base64.getEncoder().encodeToString(profileKey.serialize())) .put("registered", registered) .putPOJO("groupStore", groupStoreStorage) - .putPOJO("stickerStore", stickerStoreStorage) .putPOJO("configurationStore", configurationStoreStorage); try { try (var output = new ByteArrayOutputStream()) { @@ -1147,7 +1139,7 @@ public class SignalAccount implements Closeable { } public StickerStore getStickerStore() { - return stickerStore; + return getOrCreate(() -> stickerStore, () -> stickerStore = new StickerStore(getAccountDatabase())); } public SenderKeyStore getSenderKeyStore() { diff --git a/lib/src/main/java/org/asamk/signal/manager/storage/stickers/LegacyStickerStore.java b/lib/src/main/java/org/asamk/signal/manager/storage/stickers/LegacyStickerStore.java new file mode 100644 index 00000000..dd044536 --- /dev/null +++ b/lib/src/main/java/org/asamk/signal/manager/storage/stickers/LegacyStickerStore.java @@ -0,0 +1,35 @@ +package org.asamk.signal.manager.storage.stickers; + +import org.asamk.signal.manager.api.StickerPackId; + +import java.util.Base64; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; + +public class LegacyStickerStore { + + public static void migrate(Storage storage, StickerStore stickerStore) { + final var packIds = new HashSet(); + final var stickers = storage.stickers.stream().map(s -> { + var packId = StickerPackId.deserialize(Base64.getDecoder().decode(s.packId)); + if (packIds.contains(packId)) { + // Remove legacy duplicate packIds ... + return null; + } + packIds.add(packId); + var packKey = Base64.getDecoder().decode(s.packKey); + var installed = s.installed; + return new StickerPack(-1, packId, packKey, installed); + }).filter(Objects::nonNull).toList(); + + stickerStore.addLegacyStickers(stickers); + } + + public record Storage(List stickers) { + + private record Sticker(String packId, String packKey, boolean installed) { + + } + } +} diff --git a/lib/src/main/java/org/asamk/signal/manager/storage/stickers/Sticker.java b/lib/src/main/java/org/asamk/signal/manager/storage/stickers/Sticker.java deleted file mode 100644 index f101e474..00000000 --- a/lib/src/main/java/org/asamk/signal/manager/storage/stickers/Sticker.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.asamk.signal.manager.storage.stickers; - -import org.asamk.signal.manager.api.StickerPackId; - -public class Sticker { - - private final StickerPackId packId; - private final byte[] packKey; - private boolean installed; - - public Sticker(final StickerPackId packId, final byte[] packKey) { - this.packId = packId; - this.packKey = packKey; - } - - public Sticker(final StickerPackId packId, final byte[] packKey, final boolean installed) { - this.packId = packId; - this.packKey = packKey; - this.installed = installed; - } - - public StickerPackId getPackId() { - return packId; - } - - public byte[] getPackKey() { - return packKey; - } - - public boolean isInstalled() { - return installed; - } - - public void setInstalled(final boolean installed) { - this.installed = installed; - } -} diff --git a/lib/src/main/java/org/asamk/signal/manager/storage/stickers/StickerPack.java b/lib/src/main/java/org/asamk/signal/manager/storage/stickers/StickerPack.java new file mode 100644 index 00000000..a7e1adc0 --- /dev/null +++ b/lib/src/main/java/org/asamk/signal/manager/storage/stickers/StickerPack.java @@ -0,0 +1,10 @@ +package org.asamk.signal.manager.storage.stickers; + +import org.asamk.signal.manager.api.StickerPackId; + +public record StickerPack(long internalId, StickerPackId packId, byte[] packKey, boolean isInstalled) { + + public StickerPack(final StickerPackId packId, final byte[] packKey) { + this(-1, packId, packKey, false); + } +} diff --git a/lib/src/main/java/org/asamk/signal/manager/storage/stickers/StickerStore.java b/lib/src/main/java/org/asamk/signal/manager/storage/stickers/StickerStore.java index 2ebb7078..aeed8ccd 100644 --- a/lib/src/main/java/org/asamk/signal/manager/storage/stickers/StickerStore.java +++ b/lib/src/main/java/org/asamk/signal/manager/storage/stickers/StickerStore.java @@ -1,86 +1,149 @@ package org.asamk.signal.manager.storage.stickers; import org.asamk.signal.manager.api.StickerPackId; - -import java.util.Base64; +import org.asamk.signal.manager.storage.Database; +import org.asamk.signal.manager.storage.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; public class StickerStore { - private final Map stickers; - - private final Saver saver; - - public StickerStore(final Saver saver) { - this.saver = saver; - stickers = new HashMap<>(); + private final static Logger logger = LoggerFactory.getLogger(StickerStore.class); + private static final String TABLE_STICKER = "sticker"; + + private final Database database; + + public static void createSql(Connection connection) throws SQLException { + // When modifying the CREATE statement here, also add a migration in AccountDatabase.java + try (final var statement = connection.createStatement()) { + statement.executeUpdate(""" + CREATE TABLE sticker ( + _id INTEGER PRIMARY KEY, + pack_id BLOB UNIQUE NOT NULL, + pack_key BLOB NOT NULL, + installed BOOLEAN NOT NULL DEFAULT FALSE + ); + """); + } } - public StickerStore(final Map stickers, final Saver saver) { - this.stickers = stickers; - this.saver = saver; + public StickerStore(final Database database) { + this.database = database; } - public static StickerStore fromStorage(Storage storage, Saver saver) { - final var packIds = new HashSet(); - final var stickers = storage.stickers.stream().map(s -> { - var packId = StickerPackId.deserialize(Base64.getDecoder().decode(s.packId)); - if (packIds.contains(packId)) { - // Remove legacy duplicate packIds ... - return null; + public Collection getStickerPacks() { + final var sql = ( + """ + SELECT s._id, s.pack_id, s.pack_key, s.installed + FROM %s s + """ + ).formatted(TABLE_STICKER); + try (final var connection = database.getConnection()) { + try (final var statement = connection.prepareStatement(sql)) { + try (var result = Utils.executeQueryForStream(statement, this::getStickerPackFromResultSet)) { + return result.toList(); + } } - packIds.add(packId); - var packKey = Base64.getDecoder().decode(s.packKey); - var installed = s.installed; - return new Sticker(packId, packKey, installed); - }).filter(Objects::nonNull).collect(Collectors.toMap(Sticker::getPackId, s -> s)); - - return new StickerStore(stickers, saver); - } - - public Collection getStickerPacks() { - return stickers.values(); + } catch (SQLException e) { + throw new RuntimeException("Failed read from sticker store", e); + } } - public Sticker getStickerPack(StickerPackId packId) { - synchronized (stickers) { - return stickers.get(packId); + public StickerPack getStickerPack(StickerPackId packId) { + final var sql = ( + """ + SELECT s._id, s.pack_id, s.pack_key, s.installed + FROM %s s + WHERE s.pack_id = ? + """ + ).formatted(TABLE_STICKER); + try (final var connection = database.getConnection()) { + try (final var statement = connection.prepareStatement(sql)) { + statement.setBytes(1, packId.serialize()); + return Utils.executeQueryForOptional(statement, this::getStickerPackFromResultSet).orElse(null); + } + } catch (SQLException e) { + throw new RuntimeException("Failed read from sticker store", e); } } - public void updateSticker(Sticker sticker) { - Storage storage; - synchronized (stickers) { - stickers.put(sticker.getPackId(), sticker); - storage = toStorageLocked(); + public void addStickerPack(StickerPack stickerPack) { + final var sql = ( + """ + INSERT INTO %s (pack_id, pack_key, installed) + VALUES (?, ?, ?) + """ + ).formatted(TABLE_STICKER); + try (final var connection = database.getConnection()) { + try (final var statement = connection.prepareStatement(sql)) { + statement.setBytes(1, stickerPack.packId().serialize()); + statement.setBytes(2, stickerPack.packKey()); + statement.setBoolean(3, stickerPack.isInstalled()); + statement.executeUpdate(); + } + } catch (SQLException e) { + throw new RuntimeException("Failed update sticker store", e); } - saver.save(storage); } - private Storage toStorageLocked() { - return new Storage(stickers.values() - .stream() - .map(s -> new Storage.Sticker(Base64.getEncoder().encodeToString(s.getPackId().serialize()), - Base64.getEncoder().encodeToString(s.getPackKey()), - s.isInstalled())) - .toList()); + public void updateStickerPackInstalled(StickerPackId stickerPackId, boolean installed) { + final var sql = ( + """ + UPDATE %s + SET installed = ? + WHERE pack_id = ? + """ + ).formatted(TABLE_STICKER); + try (final var connection = database.getConnection()) { + try (final var statement = connection.prepareStatement(sql)) { + statement.setBytes(1, stickerPackId.serialize()); + statement.setBoolean(2, installed); + statement.executeUpdate(); + } + } catch (SQLException e) { + throw new RuntimeException("Failed update sticker store", e); + } } - public record Storage(List stickers) { - - private record Sticker(String packId, String packKey, boolean installed) { - + void addLegacyStickers(Collection stickerPacks) { + logger.debug("Migrating legacy stickers to database"); + long start = System.nanoTime(); + final var sql = ( + """ + INSERT INTO %s (pack_id, pack_key, installed) + VALUES (?, ?, ?) + """ + ).formatted(TABLE_STICKER); + try (final var connection = database.getConnection()) { + connection.setAutoCommit(false); + try (final var statement = connection.prepareStatement("DELETE FROM %s".formatted(TABLE_STICKER))) { + statement.executeUpdate(); + } + try (final var statement = connection.prepareStatement(sql)) { + for (final var sticker : stickerPacks) { + statement.setBytes(1, sticker.packId().serialize()); + statement.setBytes(2, sticker.packKey()); + statement.setBoolean(3, sticker.isInstalled()); + statement.executeUpdate(); + } + } + connection.commit(); + } catch (SQLException e) { + throw new RuntimeException("Failed update sticker store", e); } + logger.debug("Stickers migration took {}ms", (System.nanoTime() - start) / 1000000); } - public interface Saver { - - void save(Storage storage); + private StickerPack getStickerPackFromResultSet(ResultSet resultSet) throws SQLException { + final var internalId = resultSet.getLong("_id"); + final var packId = resultSet.getBytes("pack_id"); + final var packKey = resultSet.getBytes("pack_key"); + final var installed = resultSet.getBoolean("installed"); + return new StickerPack(internalId, StickerPackId.deserialize(packId), packKey, installed); } }