huge update with crud caching, common doForAll/getAll and deprecations in old ones doForAll

CHANGELOG.md

@@ -4,11 +4,28 @@
 
 * `Versions`:
     * `Kotlin`: `1.4.31` -> `1.4.32`
-* Add subproject `repos.cache`
+* `Pagination`:
+    * New extensions `PaginationResult.changeResultsUnchecked` and `PaginationResult.changeResults` for mapping results
+    with the same parameters, but different data
+    * Extension `PaginationResult.thisPageIfNotEmpty` now is typed and will return `PaginationResult?` with the same
+    generic type as income `PaginationResult`
+    * New extension `PaginationResult.currentPageIfNotEmpty` - shortcut for `PaginationResult.thisPageIfNotEmpty`
 * `Coroutines`:
     * Rewrite `subscribeSafelyWithoutExceptions`
         * Now `subscribeSafelyWithoutExceptions` will use default handler instead of skipping
     * New extension `subscribeSafelySkippingExceptions`
+* `Repos`
+    * New subproject `repos.cache` - this subproject will contain repos with data caching mechanisms
+    * `Pagination`
+        * New common functions. They were created as replacements for currently available for more comfortable work
+        with repos pagination:
+            * `doForAll`
+            * `doForAllWithNextPaging`
+            * `doForAllWithCurrentPaging`
+            * `getAll`
+            * `getAllWithNextPaging`
+            * `getAllWithCurrentPaging`
+        * Most old `doForAll` methods have been deprecated
 
 ## 0.4.30
 

pagination/common/src/commonMain/kotlin/dev/inmo/micro_utils/pagination/PaginationResult.kt

@@ -12,6 +12,22 @@ data class PaginationResult<T>(
 
 fun <T> emptyPaginationResult() = PaginationResult<T>(0, 0, emptyList(), 0)
 
+/**
+ * @return New [PaginationResult] with [data] without checking of data sizes equality
+ */
+fun <I, O> PaginationResult<I>.changeResultsUnchecked(
+    data: List<O>
+): PaginationResult<O> = PaginationResult(page, pagesNumber, data, size)
+/**
+ * @return New [PaginationResult] with [data] <b>with</b> checking of data sizes equality
+ */
+fun <I, O> PaginationResult<I>.changeResults(
+    data: List<O>
+): PaginationResult<O> {
+    require(data.size == results.size)
+    return changeResultsUnchecked(data)
+}
+
 fun <T> List<T>.createPaginationResult(
     pagination: Pagination,
     commonObjectsNumber: Long

pagination/common/src/commonMain/kotlin/dev/inmo/micro_utils/pagination/WalkPagination.kt

@@ -21,8 +21,10 @@ inline fun PaginationResult<*>.nextPageIfNotEmpty() = if (results.isNotEmpty())
 }
 
 @Suppress("NOTHING_TO_INLINE")
-inline fun PaginationResult<*>.thisPageIfNotEmpty(): Pagination? = if (results.isNotEmpty()) {
+inline fun <T> PaginationResult<T>.thisPageIfNotEmpty(): PaginationResult<T>? = if (results.isNotEmpty()) {
     this
 } else {
     null
 }
+
+inline fun <T> PaginationResult<T>.currentPageIfNotEmpty() = thisPageIfNotEmpty()

repos/cache/build.gradle

@@ -11,6 +11,7 @@ kotlin {
         commonMain {
             dependencies {
                 api internalProject("micro_utils.repos.common")
+                api internalProject("micro_utils.repos.inmemory")
             }
         }
     }

repos/cache/src/commonMain/kotlin/dev/inmo/micro_utils/repos/cache/CRUDCacheRepo.kt

@@ -0,0 +1,24 @@
+package dev.inmo.micro_utils.repos.cache
+
+import dev.inmo.micro_utils.repos.*
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.flow.launchIn
+import kotlinx.coroutines.flow.onEach
+
+open class CRUDCacheRepo<ObjectType, IdType, InputValueType>(
+    protected val parentRepo: CRUDRepo<ObjectType, IdType, InputValueType>,
+    protected val kvCache: KVCache<IdType, ObjectType>,
+    scope: CoroutineScope = CoroutineScope(Dispatchers.Default),
+    protected val idGetter: (ObjectType) -> IdType
+) : CRUDRepo<ObjectType, IdType, InputValueType> by parentRepo {
+    protected val onNewJob = parentRepo.newObjectsFlow.onEach { kvCache.set(idGetter(it), it) }.launchIn(scope)
+    protected val onUpdatedJob = parentRepo.updatedObjectsFlow.onEach { kvCache.set(idGetter(it), it) }.launchIn(scope)
+    protected val onRemoveJob = parentRepo.deletedObjectsIdsFlow.onEach { kvCache.unset(it) }.launchIn(scope)
+
+    override suspend fun getById(id: IdType): ObjectType? = kvCache.get(id) ?: (parentRepo.getById(id) ?.also {
+        kvCache.set(id, it)
+    })
+
+    override suspend fun contains(id: IdType): Boolean = kvCache.contains(id) || parentRepo.contains(id)
+}

repos/cache/src/commonMain/kotlin/dev/inmo/micro_utils/repos/cache/KVCache.kt

@@ -0,0 +1,41 @@
+package dev.inmo.micro_utils.repos.cache
+
+import dev.inmo.micro_utils.repos.*
+import dev.inmo.micro_utils.repos.pagination.getAllWithNextPaging
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+
+interface KVCache<K, V> : KeyValueRepo<K, V>
+
+open class SimpleKVCache<K, V>(
+    protected val cachedValuesCount: Int,
+    private val kvParent: KeyValueRepo<K, V> = MapKeyValueRepo<K, V>()
+) : KVCache<K, V>, KeyValueRepo<K, V> by kvParent {
+    protected open val cacheStack = ArrayList<K>(cachedValuesCount)
+    protected val syncMutex = Mutex()
+
+    protected suspend fun makeUnset(toUnset: List<K>) {
+        cacheStack.removeAll(toUnset)
+        kvParent.unset(toUnset)
+    }
+
+    override suspend fun set(toSet: Map<K, V>) {
+        syncMutex.withLock {
+            if (toSet.size > cachedValuesCount) {
+                cacheStack.clear()
+
+                kvParent.unset(getAllWithNextPaging { kvParent.keys(it) })
+                val keysToInclude = toSet.keys.drop(toSet.size - cachedValuesCount)
+
+                cacheStack.addAll(keysToInclude)
+                kvParent.set(keysToInclude.associateWith { toSet.getValue(it) })
+            } else {
+                makeUnset(cacheStack.take(toSet.size))
+            }
+        }
+    }
+
+    override suspend fun unset(toUnset: List<K>) {
+        syncMutex.withLock { makeUnset(toUnset) }
+    }
+}

repos/cache/src/commonMain/kotlin/dev/inmo/micro_utils/repos/cache/KeyValueCacheRepo.kt

@@ -9,34 +9,12 @@ import kotlinx.coroutines.sync.withLock
 
 open class KeyValueCacheRepo<Key,Value>(
     protected val parentRepo: KeyValueRepo<Key, Value>,
-    protected val cachedValuesCount: Int,
+    protected val kvCache: KVCache<Key, Value>,
     scope: CoroutineScope = CoroutineScope(Dispatchers.Default)
 ) : KeyValueRepo<Key,Value> by parentRepo {
-    protected open val cache = mutableMapOf<Key,Value>()
+    protected val onNewJob = parentRepo.onNewValue.onEach { kvCache.set(it.first, it.second) }.launchIn(scope)
-    protected open val cacheStack = ArrayList<Key>(cachedValuesCount)
+    protected val onRemoveJob = parentRepo.onValueRemoved.onEach { kvCache.unset(it) }.launchIn(scope)
-    protected val syncMutex = Mutex()
-    protected val onNewJob = parentRepo.onNewValue.onEach { putCacheValue(it.first, it.second) }.launchIn(scope)
-    protected val onRemoveJob = parentRepo.onValueRemoved.onEach { removeCacheValue(it) }.launchIn(scope)
 
-    protected suspend fun putCacheValue(k: Key, v: Value) = syncMutex.withLock {
+    override suspend fun get(k: Key): Value? = kvCache.get(k) ?: parentRepo.get(k) ?.also { kvCache.set(k, it) }
-        if (cache.size >= cachedValuesCount) {
+    override suspend fun contains(key: Key): Boolean = kvCache.contains(key) || parentRepo.contains(key)
-            val key = cacheStack.removeAt(0)
-            cache.remove(key)
-        }
-        cacheStack.add(k)
-        cache[k] = v
-    }
-
-    protected suspend fun removeCacheValue(k: Key) = syncMutex.withLock {
-        val i = cacheStack.indexOf(k)
-        if (i >= 0) {
-            val key = cacheStack.removeAt(i)
-            cache.remove(key)
-        }
-    }
-
-    override suspend fun get(k: Key): Value? = syncMutex.withLock {
-        cache[k] ?: parentRepo.get(k) ?.also { cache[k] = it }
-    }
-    override suspend fun contains(key: Key): Boolean = cache.containsKey(key) || parentRepo.contains(key)
 }

repos/cache/src/commonMain/kotlin/dev/inmo/micro_utils/repos/cache/KeyValuesCacheRepo.kt

@@ -13,62 +13,25 @@ import kotlinx.coroutines.sync.withLock
 
 open class KeyValuesCacheRepo<Key,Value>(
     protected val parentRepo: KeyValuesRepo<Key, Value>,
-    protected val cachedValuesCount: Int,
+    protected val kvCache: KVCache<Key, List<Value>>,
     scope: CoroutineScope = CoroutineScope(Dispatchers.Default)
 ) : KeyValuesRepo<Key,Value> by parentRepo {
-    protected open val cache = mutableMapOf<Key, List<Value>>()
+    protected val onNewJob = parentRepo.onNewValue.onEach { kvCache.set(it.first, kvCache.get(it.first) ?.plus(it.second) ?: listOf(it.second)) }.launchIn(scope)
-    protected open val cacheStack = ArrayList<Key>(cachedValuesCount)
+    protected val onRemoveJob = parentRepo.onValueRemoved.onEach { kvCache.set(it.first, kvCache.get(it.first) ?.minus(it.second) ?: return@onEach) }.launchIn(scope)
-    protected val syncMutex = Mutex()
+    protected val onDataClearedJob = parentRepo.onDataCleared.onEach { kvCache.unset(it) }.launchIn(scope)
-    protected val onNewJob = parentRepo.onNewValue.onEach { putCacheValue(it.first, it.second) }.launchIn(scope)
-    protected val onRemoveJob = parentRepo.onValueRemoved.onEach { removeCacheValue(it.first, it.second) }.launchIn(scope)
-    protected val onDataClearedJob = parentRepo.onDataCleared.onEach { clearCacheValues(it) }.launchIn(scope)
-
-    protected suspend fun putCacheValues(k: Key, v: List<Value>) = syncMutex.withLock {
-        if (cache.size >= cachedValuesCount) {
-            val key = cacheStack.removeAt(0)
-            cache.remove(key)
-        }
-        cacheStack.add(k)
-        cache[k] = v
-    }
-    protected suspend fun putCacheValue(k: Key, v: Value) = syncMutex.withLock {
-        cache[k] ?.let {
-            cache[k] = it + v
-        }
-    } ?: putCacheValues(k, listOf(v))
-
-    protected suspend fun removeCacheValue(k: Key, v: Value) = syncMutex.withLock {
-        cache[k] ?.let {
-            val newList = it - v
-            if (newList.isEmpty()) {
-                cache.remove(k)
-                cacheStack.remove(k)
-            } else {
-                cache[k] = newList
-            }
-        }
-    }
-
-    protected suspend fun clearCacheValues(k: Key) = syncMutex.withLock {
-        val i = cacheStack.indexOf(k)
-        if (i >= 0) {
-            val key = cacheStack.removeAt(i)
-            cache.remove(key)
-        }
-    }
 
     override suspend fun get(k: Key, pagination: Pagination, reversed: Boolean): PaginationResult<Value> {
-        return cache[k] ?.paginate(
+        return kvCache.get(k) ?.paginate(
             pagination.let { if (reversed) it.reverse(count(k)) else it }
         ) ?.let {
             if (reversed) it.copy(results = it.results.reversed()) else it
         } ?: parentRepo.get(k, pagination, reversed)
     }
     override suspend fun getAll(k: Key, reversed: Boolean): List<Value> {
-        return cache[k] ?.let {
+        return kvCache.get(k) ?.let {
             if (reversed) it.reversed() else it
         } ?: parentRepo.getAll(k, reversed)
     }
-    override suspend fun contains(k: Key, v: Value): Boolean = cache[k] ?.contains(v) ?: parentRepo.contains(k, v)
+    override suspend fun contains(k: Key, v: Value): Boolean = kvCache.get(k) ?.contains(v) ?: parentRepo.contains(k, v)
-    override suspend fun contains(k: Key): Boolean = cache.containsKey(k) || parentRepo.contains(k)
+    override suspend fun contains(k: Key): Boolean = kvCache.contains(k) || parentRepo.contains(k)
 }

repos/common/src/commonMain/kotlin/dev/inmo/micro_utils/repos/OneToManyKeyValueRepo.kt

@@ -1,6 +1,7 @@
 package dev.inmo.micro_utils.repos
 
 import dev.inmo.micro_utils.pagination.*
+import dev.inmo.micro_utils.repos.pagination.getAllWithNextPaging
 import kotlinx.coroutines.flow.Flow
 
 interface ReadOneToManyKeyValueRepo<Key, Value> : Repo {
@@ -12,14 +13,12 @@ interface ReadOneToManyKeyValueRepo<Key, Value> : Repo {
     suspend fun count(k: Key): Long
     suspend fun count(): Long
 
-    suspend fun getAll(k: Key, reversed: Boolean = false): List<Value> = mutableListOf<Value>().also { list ->
+    suspend fun getAll(k: Key, reversed: Boolean = false): List<Value> {
-        doWithPagination {
+        val results = getAllWithNextPaging { get(k, it) }
-            get(k, it).also {
+        return if (reversed) {
-                list.addAll(it.results)
+            results.reversed()
-            }.nextPageIfNotEmpty()
+        } else {
-        }
+            results
-        if (reversed) {
-            list.reverse()
         }
     }
 

repos/common/src/commonMain/kotlin/dev/inmo/micro_utils/repos/StandartKeyValueRepo.kt

@@ -1,6 +1,7 @@
 package dev.inmo.micro_utils.repos
 
 import dev.inmo.micro_utils.pagination.*
+import dev.inmo.micro_utils.repos.pagination.doAllWithCurrentPaging
 import kotlinx.coroutines.flow.Flow
 
 interface ReadStandardKeyValueRepo<Key, Value> : Repo {
@@ -41,7 +42,7 @@ suspend inline fun <Key, Value> WriteStandardKeyValueRepo<Key, Value>.unsetWithV
 
 interface StandardKeyValueRepo<Key, Value> : ReadStandardKeyValueRepo<Key, Value>, WriteStandardKeyValueRepo<Key, Value> {
     override suspend fun unsetWithValues(toUnset: List<Value>) = toUnset.forEach { v ->
-        doWithPagination {
+        doAllWithCurrentPaging {
             keys(v, it).also {
                 unset(it.results)
             }

repos/common/src/commonMain/kotlin/dev/inmo/micro_utils/repos/pagination/CRUDPaginationExtensions.kt

@@ -3,29 +3,29 @@ package dev.inmo.micro_utils.repos.pagination
 import dev.inmo.micro_utils.pagination.*
 import dev.inmo.micro_utils.repos.ReadStandardCRUDRepo
 
+@Deprecated("Will be removed soon due to redundancy. Can be replaced with other doForAll extensions")
 suspend inline fun <T, ID, REPO : ReadStandardCRUDRepo<T, ID>> REPO.doForAll(
     @Suppress("REDUNDANT_INLINE_SUSPEND_FUNCTION_TYPE")
-    methodCaller: suspend REPO.(Pagination) -> PaginationResult<T>,
+    crossinline methodCaller: suspend REPO.(Pagination) -> PaginationResult<T>,
-    block: (List<T>) -> Unit
+    crossinline block: (List<T>) -> Unit
 ) {
-    doWithPagination {
+    doForAllWithNextPaging {
         methodCaller(it).also {
             block(it.results)
-        }.nextPageIfNotEmpty()
+        }
     }
 }
 
+@Deprecated("Will be removed soon due to redundancy. Can be replaced with other doForAll extensions")
 suspend inline fun <T, ID, REPO : ReadStandardCRUDRepo<T, ID>> REPO.doForAll(
-    block: (List<T>) -> Unit
+    crossinline block: (List<T>) -> Unit
-) = doForAll({ getByPagination(it) }, block)
+) = doForAllWithNextPaging {
+    getByPagination(it).also { block(it.results) }
+}
 
 suspend inline fun <T, ID, REPO : ReadStandardCRUDRepo<T, ID>> REPO.getAll(
     @Suppress("REDUNDANT_INLINE_SUSPEND_FUNCTION_TYPE")
-    methodCaller: suspend REPO.(Pagination) -> PaginationResult<T>
+    crossinline methodCaller: suspend REPO.(Pagination) -> PaginationResult<T>
-): List<T> {
+): List<T> = getAllWithNextPaging {
-    val resultList = mutableListOf<T>()
+    methodCaller(this, it)
-    doForAll(methodCaller) {
-        resultList.addAll(it)
-    }
-    return resultList
 }

repos/common/src/commonMain/kotlin/dev/inmo/micro_utils/repos/pagination/CommonPaginationExtensions.kt

@@ -0,0 +1,67 @@
+package dev.inmo.micro_utils.repos.pagination
+
+import dev.inmo.micro_utils.pagination.*
+
+suspend fun <T> doForAll(
+    initialPagination: Pagination = FirstPagePagination(),
+    paginationMapper: (PaginationResult<T>) -> Pagination?,
+    block: suspend (Pagination) -> PaginationResult<T>
+) {
+    doWithPagination(initialPagination) {
+        block(it).let(paginationMapper)
+    }
+}
+
+suspend fun <T> doForAllWithNextPaging(
+    initialPagination: Pagination = FirstPagePagination(),
+    block: suspend (Pagination) -> PaginationResult<T>
+) {
+    doForAll(
+        initialPagination,
+        { it.nextPageIfNotEmpty() },
+        block
+    )
+}
+
+suspend fun <T> doAllWithCurrentPaging(
+    initialPagination: Pagination = FirstPagePagination(),
+    block: suspend (Pagination) -> PaginationResult<T>
+) {
+    doForAll(
+        initialPagination,
+        { it.thisPageIfNotEmpty() },
+        block
+    )
+}
+
+suspend fun <T> getAll(
+    initialPagination: Pagination = FirstPagePagination(),
+    paginationMapper: (PaginationResult<T>) -> Pagination?,
+    block: suspend (Pagination) -> PaginationResult<T>
+): List<T> {
+    val results = mutableListOf<T>()
+    doForAll(initialPagination, paginationMapper) {
+        block(it).also {
+            results.addAll(it.results)
+        }
+    }
+    return results.toList()
+}
+
+suspend fun <T> getAllWithNextPaging(
+    initialPagination: Pagination = FirstPagePagination(),
+    block: suspend (Pagination) -> PaginationResult<T>
+): List<T> = getAll(
+    initialPagination,
+    { it.nextPageIfNotEmpty() },
+    block
+)
+
+suspend fun <T> getAllWithCurrentPaging(
+    initialPagination: Pagination = FirstPagePagination(),
+    block: suspend (Pagination) -> PaginationResult<T>
+): List<T> = getAll(
+    initialPagination,
+    { it.thisPageIfNotEmpty() },
+    block
+)

repos/common/src/commonMain/kotlin/dev/inmo/micro_utils/repos/pagination/KeyValuePaginationExtensions.kt

@@ -3,6 +3,7 @@ package dev.inmo.micro_utils.repos.pagination
 import dev.inmo.micro_utils.pagination.*
 import dev.inmo.micro_utils.repos.ReadStandardKeyValueRepo
 
+@Deprecated("Will be removed soon due to redundancy")
 suspend inline fun <Key, Value, REPO : ReadStandardKeyValueRepo<Key, Value>> REPO.doForAll(
     @Suppress("REDUNDANT_INLINE_SUSPEND_FUNCTION_TYPE")
     methodCaller: suspend REPO.(Pagination) -> PaginationResult<Key>,
@@ -15,17 +16,17 @@ suspend inline fun <Key, Value, REPO : ReadStandardKeyValueRepo<Key, Value>> REP
     }
 }
 
+@Deprecated("Will be removed soon due to redundancy")
 suspend inline fun <Key, Value, REPO : ReadStandardKeyValueRepo<Key, Value>> REPO.doForAll(
     block: (List<Pair<Key, Value>>) -> Unit
 ) = doForAll({ keys(it, false) }, block)
 
 suspend inline fun <Key, Value, REPO : ReadStandardKeyValueRepo<Key, Value>> REPO.getAll(
     @Suppress("REDUNDANT_INLINE_SUSPEND_FUNCTION_TYPE")
-    methodCaller: suspend REPO.(Pagination) -> PaginationResult<Key>
+    crossinline methodCaller: suspend REPO.(Pagination) -> PaginationResult<Key>
-): List<Pair<Key, Value>> {
+): List<Pair<Key, Value>> = getAllWithNextPaging {
-    val resultList = mutableListOf<Pair<Key, Value>>()
+    val result = methodCaller(it)
-    doForAll(methodCaller) {
+    result.changeResultsUnchecked(
-        resultList.addAll(it)
+        result.results.mapNotNull { it to (get(it) ?: return@mapNotNull null) }
-    }
+    )
-    return resultList
 }

repos/common/src/commonMain/kotlin/dev/inmo/micro_utils/repos/pagination/OneToManyPaginationExtensions.kt

@@ -3,6 +3,7 @@ package dev.inmo.micro_utils.repos.pagination
 import dev.inmo.micro_utils.pagination.*
 import dev.inmo.micro_utils.repos.ReadOneToManyKeyValueRepo
 
+@Deprecated("Will be removed soon due to redundancy")
 suspend inline fun <Key, Value, REPO : ReadOneToManyKeyValueRepo<Key, Value>> REPO.doForAll(
     @Suppress("REDUNDANT_INLINE_SUSPEND_FUNCTION_TYPE")
     methodCaller: suspend REPO.(Pagination) -> PaginationResult<Key>,
@@ -25,17 +26,19 @@ suspend inline fun <Key, Value, REPO : ReadOneToManyKeyValueRepo<Key, Value>> RE
     }
 }
 
+@Deprecated("Will be removed soon due to redundancy")
 suspend inline fun <Key, Value, REPO : ReadOneToManyKeyValueRepo<Key, Value>> REPO.doForAll(
     block: (List<Pair<Key, List<Value>>>) -> Unit
 ) = doForAll({ keys(it, false) }, block)
 
 suspend inline fun <Key, Value, REPO : ReadOneToManyKeyValueRepo<Key, Value>> REPO.getAll(
     @Suppress("REDUNDANT_INLINE_SUSPEND_FUNCTION_TYPE")
-    methodCaller: suspend REPO.(Pagination) -> PaginationResult<Key>
+    crossinline methodCaller: suspend REPO.(Pagination) -> PaginationResult<Key>
-): List<Pair<Key, List<Value>>> {
+): List<Pair<Key, List<Value>>> = getAllWithNextPaging {
-    val resultList = mutableListOf<Pair<Key, List<Value>>>()
+    val keysResult = methodCaller(it)
-    doForAll(methodCaller) {
+    keysResult.changeResultsUnchecked(
-        resultList.addAll(it)
+        keysResult.results.map { k ->
-    }
+            k to getAll(k)
-    return resultList
+        }
+    )
 }