From f423d31423aee21ee425c2a2179e70cf251bfdfe Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Tue, 22 Mar 2022 18:52:23 +0600 Subject: [PATCH] implement content caching --- cache/{media => content/common}/build.gradle | 10 ++ .../common/DefaultMessageContentCache.kt | 119 ++++++++++++++++++ .../cache/media/common/MessageContentCache.kt | 12 ++ .../cache/media/common/MessagesFilesCache.kt | 49 ++++++++ .../media/common/InFilesMessagesFilesCache.kt | 75 +++++++++++ .../common/src/main/AndroidManifest.xml | 1 + cache/content/micro_utils/build.gradle | 19 +++ .../SimpleKeyValueMessageContentCache.kt | 84 +++++++++++++ .../micro_utils/src/main/AndroidManifest.xml | 1 + cache/media/src/main/AndroidManifest.xml | 1 - gradle.properties | 6 +- gradle/wrapper/gradle-wrapper.properties | 2 +- settings.gradle | 8 +- 13 files changed, 379 insertions(+), 8 deletions(-) rename cache/{media => content/common}/build.gradle (52%) create mode 100644 cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/DefaultMessageContentCache.kt create mode 100644 cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/MessageContentCache.kt create mode 100644 cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/MessagesFilesCache.kt create mode 100644 cache/content/common/src/jvmMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/InFilesMessagesFilesCache.kt create mode 100644 cache/content/common/src/main/AndroidManifest.xml create mode 100644 cache/content/micro_utils/build.gradle create mode 100644 cache/content/micro_utils/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/micro_utils/SimpleKeyValueMessageContentCache.kt create mode 100644 cache/content/micro_utils/src/main/AndroidManifest.xml delete mode 100644 cache/media/src/main/AndroidManifest.xml diff --git a/cache/media/build.gradle b/cache/content/common/build.gradle similarity index 52% rename from cache/media/build.gradle rename to cache/content/common/build.gradle index 8a22926..1c05fde 100644 --- a/cache/media/build.gradle +++ b/cache/content/common/build.gradle @@ -6,3 +6,13 @@ plugins { apply from: "$mppProjectWithSerializationPresetPath" +kotlin { + sourceSets { + commonMain { + dependencies { + api "dev.inmo:tgbotapi.core:$tgbotapi_version" + } + } + } +} + diff --git a/cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/DefaultMessageContentCache.kt b/cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/DefaultMessageContentCache.kt new file mode 100644 index 0000000..94ce7f2 --- /dev/null +++ b/cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/DefaultMessageContentCache.kt @@ -0,0 +1,119 @@ +package dev.inmo.tgbotapi.libraries.cache.media.common + +import dev.inmo.tgbotapi.bot.TelegramBot +import dev.inmo.tgbotapi.requests.DownloadFileStream +import dev.inmo.tgbotapi.requests.abstracts.MultipartFile +import dev.inmo.tgbotapi.requests.get.GetFile +import dev.inmo.tgbotapi.requests.send.media.* +import dev.inmo.tgbotapi.types.ChatId +import dev.inmo.tgbotapi.types.InputMedia.* +import dev.inmo.tgbotapi.types.MessageIdentifier +import dev.inmo.tgbotapi.types.message.content.abstracts.MediaContent +import dev.inmo.tgbotapi.types.message.content.abstracts.MessageContent +import io.ktor.utils.io.cancel + +class DefaultMessageContentCache( + private val bot: TelegramBot, + private val simpleMessageContentCache: MessageContentCache, + private val messagesFilesCache: MessagesFilesCache, + private val filesRefreshingChatId: ChatId +) : MessageContentCache { + override suspend fun save(chatId: ChatId, messageId: MessageIdentifier, content: MessageContent): Boolean { + runCatching { + if (content is MediaContent) { + val extendedInfo = bot.execute( + GetFile(content.media.fileId) + ) + val allocator = bot.execute( + DownloadFileStream( + extendedInfo.filePath + ) + ) + messagesFilesCache.set(chatId, messageId, extendedInfo.fileName, allocator) + } + }.onFailure { + return false + } + + return simpleMessageContentCache.save( + chatId, messageId, content + ) + } + + override suspend fun get(chatId: ChatId, messageId: MessageIdentifier): MessageContent? { + val savedSimpleContent = simpleMessageContentCache.get(chatId, messageId) ?: return null + + if (savedSimpleContent is MediaContent) { + runCatching { + val streamAllocator = bot.execute( + DownloadFileStream( + bot.execute( + GetFile( + savedSimpleContent.media.fileId + ) + ).filePath + ) + ) + + streamAllocator().apply { + readByte() + cancel() + } + }.onFailure { + val savedFileContentAllocator = messagesFilesCache.get(chatId, messageId) ?: error("Unexpected absence of $chatId:$messageId file for content ($simpleMessageContentCache)") + val newContent = bot.execute( + when (savedSimpleContent.asInputMedia()) { + is InputMediaAnimation -> SendAnimation( + filesRefreshingChatId, + MultipartFile( + savedFileContentAllocator + ), + disableNotification = true + ) + is InputMediaAudio -> SendAudio( + filesRefreshingChatId, + MultipartFile( + savedFileContentAllocator + ), + disableNotification = true + ) + is InputMediaVideo -> SendVideo( + filesRefreshingChatId, + MultipartFile( + savedFileContentAllocator + ), + disableNotification = true + ) + is InputMediaDocument -> SendDocument( + filesRefreshingChatId, + MultipartFile( + savedFileContentAllocator + ), + disableNotification = true + ) + is InputMediaPhoto -> SendPhoto( + filesRefreshingChatId, + MultipartFile( + savedFileContentAllocator + ), + disableNotification = true + ) + } + ) + + simpleMessageContentCache.save(chatId, messageId, newContent.content) + return newContent.content + } + } + return savedSimpleContent + } + + override suspend fun contains(chatId: ChatId, messageId: MessageIdentifier): Boolean { + return simpleMessageContentCache.contains(chatId, messageId) + } + + override suspend fun remove(chatId: ChatId, messageId: MessageIdentifier) { + simpleMessageContentCache.remove(chatId, messageId) + messagesFilesCache.remove(chatId, messageId) + } +} diff --git a/cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/MessageContentCache.kt b/cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/MessageContentCache.kt new file mode 100644 index 0000000..48fc69d --- /dev/null +++ b/cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/MessageContentCache.kt @@ -0,0 +1,12 @@ +package dev.inmo.tgbotapi.libraries.cache.media.common + +import dev.inmo.tgbotapi.types.ChatId +import dev.inmo.tgbotapi.types.MessageIdentifier +import dev.inmo.tgbotapi.types.message.content.abstracts.MessageContent + +interface MessageContentCache { + suspend fun save(chatId: ChatId, messageId: MessageIdentifier, content: MessageContent): Boolean + suspend fun get(chatId: ChatId, messageId: MessageIdentifier): MessageContent? + suspend fun contains(chatId: ChatId, messageId: MessageIdentifier): Boolean + suspend fun remove(chatId: ChatId, messageId: MessageIdentifier) +} diff --git a/cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/MessagesFilesCache.kt b/cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/MessagesFilesCache.kt new file mode 100644 index 0000000..5308cfd --- /dev/null +++ b/cache/content/common/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/MessagesFilesCache.kt @@ -0,0 +1,49 @@ +package dev.inmo.tgbotapi.libraries.cache.media.common + +import dev.inmo.tgbotapi.types.ChatId +import dev.inmo.tgbotapi.types.MessageIdentifier +import dev.inmo.tgbotapi.utils.ByteReadChannelAllocator +import dev.inmo.tgbotapi.utils.StorageFile +import io.ktor.util.toByteArray +import io.ktor.utils.io.ByteReadChannel + +interface MessagesFilesCache { + suspend fun set(chatId: ChatId, messageIdentifier: MessageIdentifier, filename: String, byteReadChannelAllocator: ByteReadChannelAllocator) + suspend fun get(chatId: ChatId, messageIdentifier: MessageIdentifier): StorageFile? + suspend fun remove(chatId: ChatId, messageIdentifier: MessageIdentifier) + suspend fun contains(chatId: ChatId, messageIdentifier: MessageIdentifier): Boolean +} + +/** + * It is not recommended to use in production realization of [MessagesFilesCache] which has been created for fast + * start of application creation with usage of [MessageContentCache] with aim to replace this realization by some + * disks-oriented one + */ +class InMemoryMessagesFilesCache : MessagesFilesCache { + private val map = mutableMapOf, StorageFile>() + + override suspend fun set( + chatId: ChatId, + messageIdentifier: MessageIdentifier, + filename: String, + byteReadChannelAllocator: ByteReadChannelAllocator + ) { + map[chatId to messageIdentifier] = StorageFile( + filename, + byteReadChannelAllocator.invoke().toByteArray() + ) + } + + override suspend fun get(chatId: ChatId, messageIdentifier: MessageIdentifier): StorageFile? { + return map[chatId to messageIdentifier] + } + + override suspend fun remove(chatId: ChatId, messageIdentifier: MessageIdentifier) { + map.remove(chatId to messageIdentifier) + } + + override suspend fun contains(chatId: ChatId, messageIdentifier: MessageIdentifier): Boolean { + return map.contains(chatId to messageIdentifier) + } + +} diff --git a/cache/content/common/src/jvmMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/InFilesMessagesFilesCache.kt b/cache/content/common/src/jvmMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/InFilesMessagesFilesCache.kt new file mode 100644 index 0000000..c893888 --- /dev/null +++ b/cache/content/common/src/jvmMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/common/InFilesMessagesFilesCache.kt @@ -0,0 +1,75 @@ +package dev.inmo.tgbotapi.libraries.cache.media.common + +import dev.inmo.tgbotapi.types.ChatId +import dev.inmo.tgbotapi.types.MessageIdentifier +import dev.inmo.tgbotapi.utils.* +import io.ktor.utils.io.core.copyTo +import io.ktor.utils.io.streams.asInput +import io.ktor.utils.io.streams.asOutput +import java.io.File + +class InFilesMessagesFilesCache( + private val folderFile: File +) : MessagesFilesCache { + private val Pair.storageFile: StorageFile? + get() { + val prefix = filePrefix(first, second) + val filename = folderFile.list() ?.firstOrNull { it.startsWith(prefix) } ?: return null + val file = File(folderFile, filename) + val storageFileFilename = file.name.removePrefix("$prefix ") + + return StorageFile( + StorageFileInfo(storageFileFilename) + ) { + file.inputStream().asInput() + } + } + + init { + require(!folderFile.isFile) { "Folder of messages files cache can't be file, but was $folderFile" } + folderFile.mkdirs() + } + + private fun filePrefix(chatId: ChatId, messageIdentifier: MessageIdentifier): String { + return "${chatId.chatId} $messageIdentifier" + } + + private fun fileName(chatId: ChatId, messageIdentifier: MessageIdentifier, filename: String): String { + return "${chatId.chatId} $messageIdentifier $filename" + } + + override suspend fun set( + chatId: ChatId, + messageIdentifier: MessageIdentifier, + filename: String, + byteReadChannelAllocator: ByteReadChannelAllocator + ) { + val fullFileName = fileName(chatId, messageIdentifier, filename) + val file = File(folderFile, fullFileName).apply { + delete() + } + byteReadChannelAllocator.invoke().asInput().use { input -> + file.outputStream().asOutput().use { output -> + input.copyTo(output) + } + } + } + + override suspend fun get(chatId: ChatId, messageIdentifier: MessageIdentifier): StorageFile? { + return (chatId to messageIdentifier).storageFile + } + + override suspend fun remove(chatId: ChatId, messageIdentifier: MessageIdentifier) { + val prefix = filePrefix(chatId, messageIdentifier) + folderFile.listFiles() ?.forEach { + if (it.name.startsWith(prefix)) { + it.delete() + } + } + } + + override suspend fun contains(chatId: ChatId, messageIdentifier: MessageIdentifier): Boolean { + val prefix = filePrefix(chatId, messageIdentifier) + return folderFile.list() ?.any { it.startsWith(prefix) } == true + } +} diff --git a/cache/content/common/src/main/AndroidManifest.xml b/cache/content/common/src/main/AndroidManifest.xml new file mode 100644 index 0000000..e663aab --- /dev/null +++ b/cache/content/common/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/cache/content/micro_utils/build.gradle b/cache/content/micro_utils/build.gradle new file mode 100644 index 0000000..0dcdc86 --- /dev/null +++ b/cache/content/micro_utils/build.gradle @@ -0,0 +1,19 @@ +plugins { + id "org.jetbrains.kotlin.multiplatform" + id "org.jetbrains.kotlin.plugin.serialization" + id "com.android.library" +} + +apply from: "$mppProjectWithSerializationPresetPath" + +kotlin { + sourceSets { + commonMain { + dependencies { + api "dev.inmo:micro_utils.repos.common:$micro_utils_version" + api project(":tgbotapi.libraries.cache.content.common") + } + } + } +} + diff --git a/cache/content/micro_utils/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/micro_utils/SimpleKeyValueMessageContentCache.kt b/cache/content/micro_utils/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/micro_utils/SimpleKeyValueMessageContentCache.kt new file mode 100644 index 0000000..86f6f6d --- /dev/null +++ b/cache/content/micro_utils/src/commonMain/kotlin/dev/inmo/tgbotapi/libraries/cache/media/micro_utils/SimpleKeyValueMessageContentCache.kt @@ -0,0 +1,84 @@ +package dev.inmo.tgbotapi.libraries.cache.media.micro_utils + +import dev.inmo.micro_utils.repos.* +import dev.inmo.micro_utils.repos.mappers.withMapper +import dev.inmo.tgbotapi.libraries.cache.media.common.MessageContentCache +import dev.inmo.tgbotapi.types.ChatId +import dev.inmo.tgbotapi.types.MessageIdentifier +import dev.inmo.tgbotapi.types.message.content.abstracts.MessageContent +import kotlinx.serialization.* +import kotlinx.serialization.builtins.PairSerializer +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.json.Json +import kotlinx.serialization.modules.SerializersModule +import kotlin.js.JsName +import kotlin.jvm.JvmName + +class SimpleKeyValueMessageContentCache( + private val keyValueRepo: KeyValueRepo, MessageContent> +) : MessageContentCache { + override suspend fun save(chatId: ChatId, messageId: MessageIdentifier, content: MessageContent): Boolean { + return keyValueRepo.runCatching { + set(chatId to messageId, content) + }.isSuccess + } + + override suspend fun get(chatId: ChatId, messageId: MessageIdentifier): MessageContent? { + return keyValueRepo.get(chatId to messageId) + } + + override suspend fun contains(chatId: ChatId, messageId: MessageIdentifier): Boolean { + return keyValueRepo.contains(chatId to messageId) + } + + override suspend fun remove(chatId: ChatId, messageId: MessageIdentifier) { + keyValueRepo.unset(chatId to messageId) + } +} + +val chatIdToMessageIdentifierSerializer = PairSerializer( + ChatId.serializer(), + MessageIdentifier.serializer() +) + +val messageContentSerializer = PolymorphicSerializer(MessageContent::class) + +inline fun KeyValueRepo.asMessageContentCache( + serialFormatCreator: (SerializersModule) -> StringFormat = { Json { serializersModule = it } } +): StandardKeyValueRepo, MessageContent> { + val serialFormat = serialFormatCreator(MessageContent.serializationModule()) + return withMapper, MessageContent, String, String>( + { serialFormat.encodeToString(chatIdToMessageIdentifierSerializer, this) }, + { serialFormat.encodeToString(messageContentSerializer, this) }, + { serialFormat.decodeFromString(chatIdToMessageIdentifierSerializer, this) }, + { serialFormat.decodeFromString(messageContentSerializer, this) }, + ) +} + +@JvmName("stringsKeyValueAsHexMessageContentCache") +@JsName("stringsKeyValueAsHexMessageContentCache") +inline fun KeyValueRepo.asMessageContentCache( + serialFormatCreator: (SerializersModule) -> BinaryFormat +): StandardKeyValueRepo, MessageContent> { + val serialFormat = serialFormatCreator(MessageContent.serializationModule()) + return withMapper, MessageContent, String, String>( + { serialFormat.encodeToHexString(chatIdToMessageIdentifierSerializer, this) }, + { serialFormat.encodeToHexString(messageContentSerializer, this) }, + { serialFormat.decodeFromHexString(chatIdToMessageIdentifierSerializer, this) }, + { serialFormat.decodeFromHexString(messageContentSerializer, this) }, + ) +} + +@JvmName("bytesKeyValueAsMessageContentCache") +@JsName("bytesKeyValueAsMessageContentCache") +inline fun KeyValueRepo.asMessageContentCache( + serialFormatCreator: (SerializersModule) -> BinaryFormat +): StandardKeyValueRepo, MessageContent> { + val serialFormat = serialFormatCreator(MessageContent.serializationModule()) + return withMapper, MessageContent, ByteArray, ByteArray>( + { serialFormat.encodeToByteArray(chatIdToMessageIdentifierSerializer, this) }, + { serialFormat.encodeToByteArray(messageContentSerializer, this) }, + { serialFormat.decodeFromByteArray(chatIdToMessageIdentifierSerializer, this) }, + { serialFormat.decodeFromByteArray(messageContentSerializer, this) }, + ) +} diff --git a/cache/content/micro_utils/src/main/AndroidManifest.xml b/cache/content/micro_utils/src/main/AndroidManifest.xml new file mode 100644 index 0000000..77c705e --- /dev/null +++ b/cache/content/micro_utils/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/cache/media/src/main/AndroidManifest.xml b/cache/media/src/main/AndroidManifest.xml deleted file mode 100644 index 24d7792..0000000 --- a/cache/media/src/main/AndroidManifest.xml +++ /dev/null @@ -1 +0,0 @@ - diff --git a/gradle.properties b/gradle.properties index 02f4c27..833bdda 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,9 +11,9 @@ kotlin_serialisation_core_version=1.3.2 github_release_plugin_version=2.2.12 -tgbotapi_version=0.38.4 -micro_utils_version=0.9.5 -exposed_version=0.37.2 +tgbotapi_version=0.38.9 +micro_utils_version=0.9.16 +exposed_version=0.37.3 plagubot_version=0.5.1 # ANDROID diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e750102..00e33ed 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/settings.gradle b/settings.gradle index 40732cb..ccfc849 100644 --- a/settings.gradle +++ b/settings.gradle @@ -4,13 +4,15 @@ String[] includes = [ ":cache:admins:common", ":cache:admins:micro_utils", ":cache:admins:plagubot", - ":cache:media" + + ":cache:content:common", + ":cache:content:micro_utils", ] includes.each { originalName -> - String projectDirectory = "${rootProject.projectDir.getAbsolutePath()}${originalName.replaceAll(":", File.separator)}" - String projectName = "${rootProject.name}${originalName.replaceAll(":", ".")}" + String projectDirectory = "${rootProject.projectDir.getAbsolutePath()}${originalName.replace(":", File.separator)}" + String projectName = "${rootProject.name}${originalName.replace(":", ".")}" String projectIdentifier = ":${projectName}" include projectIdentifier ProjectDescriptor project = project(projectIdentifier)