From b603fa8822a3a905d324b7863c900bbc235f7296 Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Wed, 18 Jan 2023 23:22:49 +0600 Subject: [PATCH] add caches --- common/build.gradle | 2 + common/src/commonMain/kotlin/UseCache.kt | 16 +++++ common/src/jvmMain/kotlin/CommonPlugin.kt | 4 ++ posts/panel/src/jvmMain/kotlin/Plugin.kt | 11 +++- posts/src/jvmMain/kotlin/Plugin.kt | 18 ++++-- .../jvmMain/kotlin/cached/CachedPostsRepo.kt | 49 +++++++++++++++ .../repos/CachedPollsToMessagesInfoRepo.kt | 15 +++++ .../kotlin/repos/CachedPollsToPostsIdsRepo.kt | 15 +++++ ratings/source/src/jvmMain/kotlin/Plugin.kt | 25 +++++++- .../kotlin/repo/CachedRatingsRepo.kt | 61 +++++++++++++++++++ ratings/src/jvmMain/kotlin/Plugin.kt | 17 ++++-- 11 files changed, 220 insertions(+), 13 deletions(-) create mode 100644 common/src/commonMain/kotlin/UseCache.kt create mode 100644 posts/src/jvmMain/kotlin/cached/CachedPostsRepo.kt create mode 100644 ratings/source/src/commonMain/kotlin/repos/CachedPollsToMessagesInfoRepo.kt create mode 100644 ratings/source/src/commonMain/kotlin/repos/CachedPollsToPostsIdsRepo.kt create mode 100644 ratings/src/commonMain/kotlin/repo/CachedRatingsRepo.kt diff --git a/common/build.gradle b/common/build.gradle index 127edfe..1b4c6b6 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -11,7 +11,9 @@ kotlin { dependencies { api libs.tgbotapi api libs.microutils.repos.common + api libs.microutils.repos.cache api libs.kslog + api libs.microutils.koin } } jvmMain { diff --git a/common/src/commonMain/kotlin/UseCache.kt b/common/src/commonMain/kotlin/UseCache.kt new file mode 100644 index 0000000..2c15e75 --- /dev/null +++ b/common/src/commonMain/kotlin/UseCache.kt @@ -0,0 +1,16 @@ +package dev.inmo.plaguposter.common + +import org.koin.core.Koin +import org.koin.core.module.Module +import org.koin.core.qualifier.named +import org.koin.core.scope.Scope + +val Scope.useCache: Boolean + get() = getOrNull(named("useCache")) ?: false + +val Koin.useCache: Boolean + get() = getOrNull(named("useCache")) ?: false + +fun Module.useCache(useCache: Boolean) { + single(named("useCache")) { useCache } +} diff --git a/common/src/jvmMain/kotlin/CommonPlugin.kt b/common/src/jvmMain/kotlin/CommonPlugin.kt index dd8995c..9f55aa6 100644 --- a/common/src/jvmMain/kotlin/CommonPlugin.kt +++ b/common/src/jvmMain/kotlin/CommonPlugin.kt @@ -10,6 +10,8 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull import org.jetbrains.exposed.sql.Database import org.koin.core.Koin import org.koin.core.module.Module @@ -18,6 +20,8 @@ object CommonPlugin : Plugin { private val Log = logger override fun Module.setupDI(database: Database, params: JsonObject) { single { CoroutineScope(Dispatchers.Default + SupervisorJob()) } + val useCache = (params["useCache"] as? JsonPrimitive) ?.booleanOrNull ?: true + useCache(useCache) } override suspend fun BehaviourContext.setupBotPlugin(koin: Koin) { diff --git a/posts/panel/src/jvmMain/kotlin/Plugin.kt b/posts/panel/src/jvmMain/kotlin/Plugin.kt index a267893..059f9d4 100644 --- a/posts/panel/src/jvmMain/kotlin/Plugin.kt +++ b/posts/panel/src/jvmMain/kotlin/Plugin.kt @@ -4,6 +4,9 @@ import com.benasher44.uuid.uuid4 import dev.inmo.micro_utils.coroutines.runCatchingSafely import dev.inmo.micro_utils.coroutines.subscribeSafelyWithoutExceptions import dev.inmo.micro_utils.koin.getAllDistinct +import dev.inmo.micro_utils.repos.cache.cache.FullKVCache +import dev.inmo.micro_utils.repos.cache.cached +import dev.inmo.micro_utils.repos.cache.full.cached import dev.inmo.micro_utils.repos.deleteById import dev.inmo.micro_utils.repos.id import dev.inmo.micro_utils.repos.set @@ -12,6 +15,7 @@ import dev.inmo.micro_utils.repos.value import dev.inmo.plagubot.Plugin import dev.inmo.plaguposter.common.ChatConfig import dev.inmo.plaguposter.common.UnsuccessfulSymbol +import dev.inmo.plaguposter.common.useCache import dev.inmo.plaguposter.posts.models.PostId import dev.inmo.plaguposter.posts.panel.repos.PostsMessages import dev.inmo.plaguposter.posts.repo.PostsRepo @@ -93,7 +97,12 @@ object Plugin : Plugin { val chatsConfig = koin.get() val config = koin.getOrNull() ?: Config() val api = koin.get() - val postsMessages = PostsMessages(koin.get(), koin.get()) + val basePostsMessages = PostsMessages(koin.get(), koin.get()) + val postsMessages = if (koin.useCache) { + basePostsMessages.cached(FullKVCache(), koin.get()) + } else { + basePostsMessages + } postsRepo.newObjectsFlow.subscribeSafelyWithoutExceptions(this) { val firstContent = it.content.first() diff --git a/posts/src/jvmMain/kotlin/Plugin.kt b/posts/src/jvmMain/kotlin/Plugin.kt index c56f832..047094a 100644 --- a/posts/src/jvmMain/kotlin/Plugin.kt +++ b/posts/src/jvmMain/kotlin/Plugin.kt @@ -4,6 +4,7 @@ import dev.inmo.kslog.common.logger import dev.inmo.kslog.common.w import dev.inmo.micro_utils.coroutines.runCatchingSafely import dev.inmo.micro_utils.coroutines.subscribeSafelyWithoutExceptions +import dev.inmo.micro_utils.koin.singleWithBinds import dev.inmo.micro_utils.repos.deleteById import dev.inmo.plagubot.Plugin import dev.inmo.plaguposter.common.SuccessfulSymbol @@ -13,6 +14,8 @@ import dev.inmo.plaguposter.common.ChatConfig import dev.inmo.plagubot.plugins.inline.queries.models.Format import dev.inmo.plagubot.plugins.inline.queries.models.OfferTemplate import dev.inmo.plagubot.plugins.inline.queries.repos.InlineTemplatesRepo +import dev.inmo.plaguposter.common.useCache +import dev.inmo.plaguposter.posts.cached.CachedPostsRepo import dev.inmo.plaguposter.posts.repo.* import dev.inmo.plaguposter.posts.sending.PostPublisher import dev.inmo.tgbotapi.extensions.api.delete @@ -44,11 +47,16 @@ object Plugin : Plugin { } single { get().decodeFromJsonElement(Config.serializer(), configJson) } single { get().chats } - single { ExposedPostsRepo(database) } binds arrayOf( - PostsRepo::class, - ReadPostsRepo::class, - WritePostsRepo::class, - ) + single { ExposedPostsRepo(database) } + singleWithBinds { + val base = get() + + if (useCache) { + CachedPostsRepo(base, get()) + } else { + base + } + } single { val config = get() PostPublisher(get(), get(), config.chats.cacheChatId, config.chats.targetChatId, config.deleteAfterPublishing) diff --git a/posts/src/jvmMain/kotlin/cached/CachedPostsRepo.kt b/posts/src/jvmMain/kotlin/cached/CachedPostsRepo.kt new file mode 100644 index 0000000..305f310 --- /dev/null +++ b/posts/src/jvmMain/kotlin/cached/CachedPostsRepo.kt @@ -0,0 +1,49 @@ +package dev.inmo.plaguposter.posts.cached + +import com.soywiz.klock.DateTime +import dev.inmo.micro_utils.pagination.FirstPagePagination +import dev.inmo.micro_utils.pagination.firstPageWithOneElementPagination +import dev.inmo.micro_utils.pagination.utils.doForAllWithNextPaging +import dev.inmo.micro_utils.repos.CRUDRepo +import dev.inmo.micro_utils.repos.cache.cache.FullKVCache +import dev.inmo.micro_utils.repos.cache.full.FullCRUDCacheRepo +import dev.inmo.plaguposter.posts.models.NewPost +import dev.inmo.plaguposter.posts.models.PostContentInfo +import dev.inmo.plaguposter.posts.models.PostId +import dev.inmo.plaguposter.posts.models.RegisteredPost +import dev.inmo.plaguposter.posts.repo.PostsRepo +import dev.inmo.tgbotapi.types.IdChatIdentifier +import dev.inmo.tgbotapi.types.MessageIdentifier +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.Flow + +class CachedPostsRepo( + private val parentRepo: PostsRepo, + private val scope: CoroutineScope, + private val kvCache: FullKVCache = FullKVCache() +) : PostsRepo, CRUDRepo by FullCRUDCacheRepo( + parentRepo, + kvCache, + scope, + { it.id } +) { + override val removedPostsFlow: Flow by parentRepo::removedPostsFlow + + override suspend fun getIdByChatAndMessage(chatId: IdChatIdentifier, messageId: MessageIdentifier): PostId? { + doForAllWithNextPaging(firstPageWithOneElementPagination) { + kvCache.values(it).also { + it.results.forEach { + return it.takeIf { + it.content.any { it.chatId == chatId && it.messageId == messageId } + } ?.id ?: return@forEach + } + } + } + + return null + } + + override suspend fun getPostCreationTime(postId: PostId): DateTime? = kvCache.get(postId) ?.created + + override suspend fun getFirstMessageInfo(postId: PostId): PostContentInfo? = kvCache.get(postId) ?.content ?.firstOrNull() +} diff --git a/ratings/source/src/commonMain/kotlin/repos/CachedPollsToMessagesInfoRepo.kt b/ratings/source/src/commonMain/kotlin/repos/CachedPollsToMessagesInfoRepo.kt new file mode 100644 index 0000000..dce3942 --- /dev/null +++ b/ratings/source/src/commonMain/kotlin/repos/CachedPollsToMessagesInfoRepo.kt @@ -0,0 +1,15 @@ +package dev.inmo.plaguposter.ratings.source.repos + +import dev.inmo.micro_utils.repos.KeyValueRepo +import dev.inmo.micro_utils.repos.cache.KeyValueCacheRepo +import dev.inmo.micro_utils.repos.cache.cache.FullKVCache +import dev.inmo.micro_utils.repos.cache.full.cached +import dev.inmo.plaguposter.common.ShortMessageInfo +import dev.inmo.tgbotapi.types.PollIdentifier +import kotlinx.coroutines.CoroutineScope + +class CachedPollsToMessagesInfoRepo( + private val repo: PollsToMessagesInfoRepo, + private val scope: CoroutineScope, + private val kvCache: FullKVCache = FullKVCache() +) : PollsToMessagesInfoRepo, KeyValueRepo by repo.cached(kvCache, scope) diff --git a/ratings/source/src/commonMain/kotlin/repos/CachedPollsToPostsIdsRepo.kt b/ratings/source/src/commonMain/kotlin/repos/CachedPollsToPostsIdsRepo.kt new file mode 100644 index 0000000..220eb51 --- /dev/null +++ b/ratings/source/src/commonMain/kotlin/repos/CachedPollsToPostsIdsRepo.kt @@ -0,0 +1,15 @@ +package dev.inmo.plaguposter.ratings.source.repos + +import dev.inmo.micro_utils.repos.KeyValueRepo +import dev.inmo.micro_utils.repos.cache.cache.FullKVCache +import dev.inmo.micro_utils.repos.cache.full.cached +import dev.inmo.plaguposter.common.ShortMessageInfo +import dev.inmo.plaguposter.posts.models.PostId +import dev.inmo.tgbotapi.types.PollIdentifier +import kotlinx.coroutines.CoroutineScope + +class CachedPollsToPostsIdsRepo( + private val repo: PollsToPostsIdsRepo, + private val scope: CoroutineScope, + private val kvCache: FullKVCache = FullKVCache() +) : PollsToPostsIdsRepo, KeyValueRepo by repo.cached(kvCache, scope) diff --git a/ratings/source/src/jvmMain/kotlin/Plugin.kt b/ratings/source/src/jvmMain/kotlin/Plugin.kt index d48c6b5..dcd3b31 100644 --- a/ratings/source/src/jvmMain/kotlin/Plugin.kt +++ b/ratings/source/src/jvmMain/kotlin/Plugin.kt @@ -67,8 +67,29 @@ object Plugin : Plugin { get().decodeFromJsonElement(Config.serializer(), params["ratingsPolls"] ?: error("Unable to load config for rating polls in $params")) } single(ratingVariantsQualifier) { get().variants } - single { ExposedPollsToPostsIdsRepo(database) } - single { ExposedPollsToMessagesInfoRepo(database) } + + single { ExposedPollsToPostsIdsRepo(database) } + single { + val base = get() + + if (useCache) { + CachedPollsToPostsIdsRepo(base, get()) + } else { + base + } + } + + single { ExposedPollsToMessagesInfoRepo(database) } + single { + val base = get() + + if (useCache) { + CachedPollsToMessagesInfoRepo(base, get()) + } else { + base + } + } + single { val ratingsSettings = get(ratingVariantsQualifier) VariantTransformer { diff --git a/ratings/src/commonMain/kotlin/repo/CachedRatingsRepo.kt b/ratings/src/commonMain/kotlin/repo/CachedRatingsRepo.kt new file mode 100644 index 0000000..115efbd --- /dev/null +++ b/ratings/src/commonMain/kotlin/repo/CachedRatingsRepo.kt @@ -0,0 +1,61 @@ +package dev.inmo.plaguposter.ratings.repo + +import dev.inmo.micro_utils.pagination.utils.doForAllWithNextPaging +import dev.inmo.micro_utils.repos.KeyValueRepo +import dev.inmo.micro_utils.repos.cache.cache.FullKVCache +import dev.inmo.micro_utils.repos.cache.full.FullKeyValueCacheRepo +import dev.inmo.plaguposter.posts.models.PostId +import dev.inmo.plaguposter.ratings.models.Rating +import kotlinx.coroutines.CoroutineScope + +class CachedRatingsRepo( + private val base: RatingsRepo, + private val scope: CoroutineScope, + private val kvCache: FullKVCache = FullKVCache() +) : RatingsRepo, KeyValueRepo by FullKeyValueCacheRepo(base, kvCache, scope) { + override suspend fun getPosts( + range: ClosedRange, + reversed: Boolean, + count: Int?, + exclude: List + ): Map { + val result = mutableMapOf() + + doForAllWithNextPaging { + kvCache.keys(it).also { + it.results.forEach { + val rating = kvCache.get(it) ?: return@forEach + if (it !in exclude && rating in range) { + result[it] = rating + } + } + } + } + + return result.toMap() + } + + override suspend fun getPostsWithRatingGreaterEq( + then: Rating, + reversed: Boolean, + count: Int?, + exclude: List + ): Map = getPosts( + then .. Rating(Double.MAX_VALUE), + reversed, + count, + exclude + ) + + override suspend fun getPostsWithRatingLessEq( + then: Rating, + reversed: Boolean, + count: Int?, + exclude: List + ): Map = getPosts( + Rating(Double.MIN_VALUE) .. then, + reversed, + count, + exclude + ) +} diff --git a/ratings/src/jvmMain/kotlin/Plugin.kt b/ratings/src/jvmMain/kotlin/Plugin.kt index f603605..de859d7 100644 --- a/ratings/src/jvmMain/kotlin/Plugin.kt +++ b/ratings/src/jvmMain/kotlin/Plugin.kt @@ -1,8 +1,10 @@ package dev.inmo.plaguposter.ratings import dev.inmo.micro_utils.coroutines.subscribeSafelyWithoutExceptions +import dev.inmo.micro_utils.koin.singleWithBinds import dev.inmo.micro_utils.repos.unset import dev.inmo.plagubot.Plugin +import dev.inmo.plaguposter.common.useCache import dev.inmo.plaguposter.posts.exposed.ExposedPostsRepo import dev.inmo.plaguposter.posts.repo.PostsRepo import dev.inmo.plaguposter.ratings.exposed.ExposedRatingsRepo @@ -16,11 +18,16 @@ import org.koin.dsl.binds object Plugin : Plugin { override fun Module.setupDI(database: Database, params: JsonObject) { - single { ExposedRatingsRepo(database) } binds arrayOf( - RatingsRepo::class, - ReadRatingsRepo::class, - WriteRatingsRepo::class, - ) + single { ExposedRatingsRepo(database) } + singleWithBinds { + val base = get() + + if (useCache) { + CachedRatingsRepo(base, get()) + } else { + base + } + } } override suspend fun BehaviourContext.setupBotPlugin(koin: Koin) {