add storing of content

This commit is contained in:
InsanusMokrassar 2022-01-07 23:38:55 +06:00
parent 3661c1ca73
commit 92ab01ee9d
18 changed files with 377 additions and 37 deletions

View File

@ -3,8 +3,8 @@ package dev.inmo.postssystem.client
import dev.inmo.postssystem.client.ui.fsm.*
import dev.inmo.postssystem.features.auth.client.installClientAuthenticator
import dev.inmo.postssystem.features.auth.common.*
import dev.inmo.postssystem.features.files.client.ClientFilesStorage
import dev.inmo.postssystem.features.files.common.storage.FilesStorage
import dev.inmo.postssystem.features.files.client.ClientReadFilesStorage
import dev.inmo.postssystem.features.files.common.storage.ReadFilesStorage
import dev.inmo.postssystem.features.roles.common.Role
import dev.inmo.postssystem.features.roles.common.RolesStorage
import dev.inmo.postssystem.features.roles.client.ClientRolesStorage
@ -18,7 +18,6 @@ import dev.inmo.micro_utils.fsm.common.StatesMachine
import dev.inmo.micro_utils.fsm.common.dsl.FSMBuilder
import dev.inmo.micro_utils.fsm.common.managers.DefaultStatesManagerRepo
import dev.inmo.micro_utils.ktor.client.UnifiedRequester
import dev.inmo.micro_utils.ktor.common.standardKtorSerialFormat
import dev.inmo.micro_utils.repos.KeyValueRepo
import dev.inmo.postssystem.client.settings.DefaultSettings
import dev.inmo.postssystem.client.settings.Settings
@ -131,7 +130,7 @@ fun getAuthorizedFeaturesDIModule(
single { StatusFeatureClient(get(serverUrlQualifier), get()) }
single<FilesStorage> { ClientFilesStorage(get(serverUrlQualifier), get(), get()) }
single<ReadFilesStorage> { ClientReadFilesStorage(get(serverUrlQualifier), get(), get()) }
single<ReadUsersStorage> { UsersStorageKtorClient(get(serverUrlQualifier), get()) }
single<RolesStorage<Role>> { ClientRolesStorage(get(serverUrlQualifier), get(), Role.serializer()) }
}

View File

@ -10,7 +10,8 @@ kotlin {
commonMain {
dependencies {
api project(":postssystem.features.content.binary.common")
api project(":postssystem.features.common.server")
api project(":postssystem.features.content.server")
api project(":postssystem.features.files.server")
}
}
}

View File

@ -0,0 +1,85 @@
package dev.inmo.postssystem.features.content.binary.server
import dev.inmo.micro_utils.pagination.*
import dev.inmo.micro_utils.repos.UpdatedValuePair
import dev.inmo.postssystem.features.content.binary.common.BinaryContent
import dev.inmo.postssystem.features.content.common.*
import dev.inmo.postssystem.features.content.server.ServerContentStorage
import dev.inmo.postssystem.features.files.common.*
import dev.inmo.postssystem.features.files.common.storage.FilesStorage
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.map
class BinaryServerContentStorage(
private val filesStorage: FilesStorage
) : ServerContentStorage<BinaryContent> {
private val FileId.asContentId
get() = ContentId(string)
private val ContentId.asFileId
get() = FileId(string)
private val FullFileInfoStorageWrapper.asRegisteredContent
get() = RegisteredContent(
id.asContentId,
BinaryContent(
fileInfo.name,
fileInfo.mimeType,
fileInfo.byteArrayAllocator
)
)
private val BinaryContent.asFullFileInfo
get() = FullFileInfo(
filename,
mimeType,
bytesAllocator
)
override val deletedObjectsIdsFlow: Flow<ContentId> = filesStorage.deletedObjectsIdsFlow.map { it.asContentId }
override val newObjectsFlow: Flow<RegisteredContent> = filesStorage.newObjectsFlow.map { it.asRegisteredContent }
override val updatedObjectsFlow: Flow<RegisteredContent> = filesStorage.updatedObjectsFlow.map { it.asRegisteredContent }
override suspend fun create(values: List<BinaryContent>): List<RegisteredContent> {
return filesStorage.create(
values.map { it.asFullFileInfo }
).map { it.asRegisteredContent }
}
override suspend fun deleteById(ids: List<ContentId>) {
filesStorage.deleteById(ids.map { it.asFileId })
}
override suspend fun update(id: ContentId, value: BinaryContent): RegisteredContent? {
return filesStorage.update(
id.asFileId,
value.asFullFileInfo
) ?.asRegisteredContent
}
override suspend fun update(values: List<UpdatedValuePair<ContentId, BinaryContent>>): List<RegisteredContent> {
return filesStorage.update(
values.map { (id, content) ->
id.asFileId to content.asFullFileInfo
}
).map {
it.asRegisteredContent
}
}
override suspend fun contains(id: ContentId): Boolean {
return filesStorage.contains(id.asFileId)
}
override suspend fun count(): Long = filesStorage.count()
override suspend fun getById(id: ContentId): RegisteredContent? {
return filesStorage.getFullFileInfo(id.asFileId) ?.asRegisteredContent
}
override suspend fun getByPagination(pagination: Pagination): PaginationResult<RegisteredContent> {
return filesStorage.getByPagination(pagination).let {
it.changeResultsUnchecked(
it.results.mapNotNull {
filesStorage.getFullFileInfo(it.id) ?.asRegisteredContent
}
)
}
}
}

View File

@ -11,6 +11,7 @@ kotlin {
dependencies {
api project(":postssystem.features.content.common")
api project(":postssystem.features.common.server")
api project(":postssystem.features.content.server")
}
}
}

View File

@ -3,4 +3,4 @@ package dev.inmo.postssystem.features.content.server
import dev.inmo.micro_utils.repos.CRUDRepo
import dev.inmo.postssystem.features.content.common.*
interface ServerContentStorage : ServerReadContentStorage, ServerWriteContentStorage, CRUDRepo<RegisteredContent, ContentId, Content>
interface ServerContentStorage<T: Content> : ServerReadContentStorage, ServerWriteContentStorage<T>, CRUDRepo<RegisteredContent, ContentId, T>

View File

@ -0,0 +1,117 @@
package dev.inmo.postssystem.features.content.server
import dev.inmo.micro_utils.pagination.*
import dev.inmo.micro_utils.repos.UpdatedValuePair
import dev.inmo.postssystem.features.content.common.*
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.Flow
class ServerContentStorageAggregator(
private val storages: List<ServerContentStorageWrapper<*>>,
private val scope: CoroutineScope
) : ServerContentStorage<Content> {
override val deletedObjectsIdsFlow: Flow<ContentId>
get() = TODO("Not yet implemented")
override val newObjectsFlow: Flow<RegisteredContent>
get() = TODO("Not yet implemented")
override val updatedObjectsFlow: Flow<RegisteredContent>
get() = TODO("Not yet implemented")
override suspend fun create(values: List<Content>): List<RegisteredContent> {
return values.groupBy {
storages.firstOrNull { storage -> storage.mayHandle(it) }
}.let {
(it - null) as Map<ServerContentStorageWrapper<*>, List<Content>>
}.flatMap { (storage, content) ->
storage.create(content)
}
}
override suspend fun deleteById(ids: List<ContentId>) {
storages.map {
scope.launch { it.deleteById(ids) }
}.joinAll()
}
override suspend fun update(id: ContentId, value: Content): RegisteredContent? {
return storages.mapNotNull {
it.takeIf { it.mayHandle(value) }
}.firstNotNullOfOrNull { it.update(id, value) }
}
override suspend fun update(values: List<UpdatedValuePair<ContentId, Content>>): List<RegisteredContent> {
return values.groupBy { (_, content) ->
storages.firstOrNull { storage -> storage.mayHandle(content) }
}.let {
(it - null) as Map<ServerContentStorageWrapper<*>, List<UpdatedValuePair<ContentId, Content>>>
}.flatMap { (storage, values) ->
storage.update(values)
}
}
override suspend fun contains(id: ContentId): Boolean {
val contains = CompletableDeferred<Boolean>()
storages.map {
scope.launch {
if (it.contains(id)) {
contains.complete(true)
}
}.also { job ->
contains.invokeOnCompletion { job.cancel() }
}
}.joinAll()
return if (contains.isCompleted) {
contains.getCompleted()
} else {
return false
}
}
override suspend fun count(): Long {
return storages.map {
scope.async {
it.count()
}
}.awaitAll().sum()
}
override suspend fun getById(id: ContentId): RegisteredContent? {
val result = CompletableDeferred<RegisteredContent>()
storages.map {
scope.launch {
val content = it.getById(id)
if (content != null) {
result.complete(content)
}
}.also { job ->
result.invokeOnCompletion { job.cancel() }
}
}.joinAll()
return if (result.isCompleted) {
result.getCompleted()
} else {
return null
}
}
override suspend fun getByPagination(
pagination: Pagination
): PaginationResult<RegisteredContent> {
val currentResults = mutableListOf<RegisteredContent>()
for (it in storages) {
val currentPagination = PaginationByIndexes((currentResults.size + pagination.firstIndex), pagination.lastIndex)
val wrapperResults = it.getByPagination(currentPagination)
currentResults.addAll(wrapperResults.results)
if (currentResults.size >= pagination.size) {
break
}
}
return currentResults.createPaginationResult(pagination, count())
}
}

View File

@ -0,0 +1,47 @@
package dev.inmo.postssystem.features.content.server
import dev.inmo.micro_utils.repos.UpdatedValuePair
import dev.inmo.postssystem.features.content.common.*
import kotlinx.coroutines.flow.Flow
import kotlin.reflect.KClass
class ServerContentStorageWrapper<T: Content>(
private val originalStorage: ServerContentStorage<T>,
private val klass: KClass<T>
): ServerContentStorage<Content>, ServerReadContentStorage by originalStorage {
override val deletedObjectsIdsFlow: Flow<ContentId> by originalStorage::deletedObjectsIdsFlow
override val newObjectsFlow: Flow<RegisteredContent> by originalStorage::newObjectsFlow
override val updatedObjectsFlow: Flow<RegisteredContent> by originalStorage::updatedObjectsFlow
fun mayHandle(content: Content) = klass.isInstance(content)
@Suppress("UNCHECKED_CAST")
private fun asHandlableContent(content: Content) = if (mayHandle(content)) content as T else null
override suspend fun create(values: List<Content>): List<RegisteredContent> {
return originalStorage.create(
values.mapNotNull {
asHandlableContent(it)
}
)
}
override suspend fun deleteById(ids: List<ContentId>) = originalStorage.deleteById(ids)
override suspend fun update(id: ContentId, value: Content): RegisteredContent? {
return originalStorage.update(id, asHandlableContent(value) ?: return null)
}
override suspend fun update(values: List<UpdatedValuePair<ContentId, Content>>): List<RegisteredContent> {
return originalStorage.update(
values.mapNotNull { (id, content) ->
id to (asHandlableContent(content) ?: return@mapNotNull null)
}
)
}
}
inline fun <reified T: Content> ServerContentStorage<T>.wrap() = ServerContentStorageWrapper(
this,
T::class
)

View File

@ -3,4 +3,4 @@ package dev.inmo.postssystem.features.content.server
import dev.inmo.micro_utils.repos.WriteCRUDRepo
import dev.inmo.postssystem.features.content.common.*
interface ServerWriteContentStorage : WriteCRUDRepo<RegisteredContent, ContentId, Content>
interface ServerWriteContentStorage<T: Content> : WriteCRUDRepo<RegisteredContent, ContentId, T>

View File

@ -11,6 +11,7 @@ kotlin {
dependencies {
api project(":postssystem.features.content.text.common")
api project(":postssystem.features.common.server")
api project(":postssystem.features.content.server")
}
}
}

View File

@ -0,0 +1,53 @@
package dev.inmo.postssystem.features.content.text.server
import com.benasher44.uuid.uuid4
import dev.inmo.micro_utils.repos.exposed.AbstractExposedCRUDRepo
import dev.inmo.micro_utils.repos.exposed.initTable
import dev.inmo.postssystem.features.content.common.ContentId
import dev.inmo.postssystem.features.content.common.RegisteredContent
import dev.inmo.postssystem.features.content.server.ServerContentStorage
import dev.inmo.postssystem.features.content.text.common.TextContent
import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.statements.InsertStatement
import org.jetbrains.exposed.sql.statements.UpdateStatement
class TextServerContentStorage(
override val database: Database
) : ServerContentStorage<TextContent>,
AbstractExposedCRUDRepo<RegisteredContent, ContentId, TextContent>(tableName = "TextContent") {
val idColumn = text("id")
private val textColumn = text("text")
override val selectByIds: SqlExpressionBuilder.(List<ContentId>) -> Op<Boolean> = {
idColumn.inList(it.map { it.string })
}
override val selectById: SqlExpressionBuilder.(ContentId) -> Op<Boolean> = {
idColumn.eq(it.string)
}
override val ResultRow.asObject: RegisteredContent
get() = RegisteredContent(
ContentId(get(idColumn)),
TextContent(get(textColumn))
)
init {
initTable()
}
override fun insert(value: TextContent, it: InsertStatement<Number>) {
it[idColumn] = uuid4().toString()
it[textColumn] = value.text
}
override fun update(id: ContentId, value: TextContent, it: UpdateStatement) {
it[textColumn] = value.text
}
override fun InsertStatement<Number>.asObject(value: TextContent): RegisteredContent {
return RegisteredContent(
ContentId(get(idColumn)),
TextContent(get(textColumn))
)
}
}

View File

@ -1,7 +1,7 @@
package dev.inmo.postssystem.features.files.client
import dev.inmo.postssystem.features.files.common.*
import dev.inmo.postssystem.features.files.common.storage.FilesStorage
import dev.inmo.postssystem.features.files.common.storage.ReadFilesStorage
import dev.inmo.micro_utils.ktor.client.UnifiedRequester
import dev.inmo.micro_utils.ktor.common.buildStandardUrl
import dev.inmo.micro_utils.repos.ReadCRUDRepo
@ -13,11 +13,11 @@ import io.ktor.client.statement.readBytes
import kotlinx.serialization.BinaryFormat
import kotlinx.serialization.builtins.nullable
class ClientFilesStorage(
class ClientReadFilesStorage(
baseUrl: String,
private val client: HttpClient,
private val serialFormat: BinaryFormat
) : FilesStorage, ReadCRUDRepo<MetaFileInfoStorageWrapper, FileId> by KtorReadStandardCrudRepo(
) : ReadFilesStorage, ReadCRUDRepo<MetaFileInfoStorageWrapper, FileId> by KtorReadStandardCrudRepo(
buildStandardUrl(baseUrl, filesRootPathPart),
UnifiedRequester(client, serialFormat),
MetaFileInfoStorageWrapper.serializer(),

View File

@ -1,9 +1,8 @@
package dev.inmo.postssystem.features.files.common.storage
import dev.inmo.postssystem.features.files.common.*
import dev.inmo.micro_utils.repos.ReadCRUDRepo
interface FilesStorage : ReadFilesStorage, WriteFilesStorage
interface FilesStorage : ReadCRUDRepo<MetaFileInfoStorageWrapper, FileId> {
suspend fun getBytes(id: FileId): ByteArray
suspend fun getFullFileInfo(id: FileId): FullFileInfoStorageWrapper?
}
class DefaultFilesStorage(
filesStorage: ReadFilesStorage,
writeFilesStorage: WriteFilesStorage
) : FilesStorage, ReadFilesStorage by filesStorage, WriteFilesStorage by writeFilesStorage

View File

@ -1,8 +0,0 @@
package dev.inmo.postssystem.features.files.common.storage
interface FullFilesStorage : FilesStorage, WriteFilesStorage
class DefaultFullFilesStorage(
filesStorage: FilesStorage,
writeFilesStorage: WriteFilesStorage
) : FullFilesStorage, FilesStorage by filesStorage, WriteFilesStorage by writeFilesStorage

View File

@ -0,0 +1,9 @@
package dev.inmo.postssystem.features.files.common.storage
import dev.inmo.postssystem.features.files.common.*
import dev.inmo.micro_utils.repos.ReadCRUDRepo
interface ReadFilesStorage : ReadCRUDRepo<MetaFileInfoStorageWrapper, FileId> {
suspend fun getBytes(id: FileId): ByteArray
suspend fun getFullFileInfo(id: FileId): FullFileInfoStorageWrapper?
}

View File

@ -1,14 +1,14 @@
package dev.inmo.postssystem.features.files.common
import dev.inmo.postssystem.features.files.common.storage.FilesStorage
import dev.inmo.postssystem.features.files.common.storage.ReadFilesStorage
import dev.inmo.micro_utils.pagination.*
import dev.inmo.micro_utils.repos.ReadKeyValueRepo
import java.io.File
class DiskFilesStorage(
class DiskReadFilesStorage(
private val filesFolder: File,
private val metasKeyValueRepo: ReadKeyValueRepo<FileId, MetaFileInfo>
) : FilesStorage {
) : ReadFilesStorage {
private val FileId.file
get() = File(filesFolder, string)

View File

@ -13,11 +13,11 @@ import io.ktor.routing.*
import kotlinx.serialization.builtins.nullable
class FilesRoutingConfigurator(
private val filesStorage: FilesStorage,
private val filesStorage: ReadFilesStorage,
private val writeFilesStorage: WriteFilesStorage?,
private val unifierRouter: UnifiedRouter
) : ApplicationRoutingConfigurator.Element {
constructor(fullFilesStorage: FullFilesStorage, unifierRouter: UnifiedRouter) : this(fullFilesStorage, fullFilesStorage, unifierRouter)
constructor(filesStorage: FilesStorage, unifierRouter: UnifiedRouter) : this(filesStorage, filesStorage, unifierRouter)
override fun Route.invoke() {
authenticate {

View File

@ -18,4 +18,9 @@ data class Config(
) {
val filesFolderFile: File
get() = File(filesFolder)
val commonFilesFolder: File
get() = File(filesFolderFile, "common").also { it.mkdirs() }
val binaryFilesFolder: File
get() = File(filesFolderFile, "binary_content").also { it.mkdirs() }
}

View File

@ -26,9 +26,12 @@ import dev.inmo.micro_utils.repos.exposed.keyvalue.ExposedKeyValueRepo
import dev.inmo.micro_utils.repos.exposed.onetomany.ExposedOneToManyKeyValueRepo
import dev.inmo.postssystem.features.common.common.*
import dev.inmo.postssystem.features.content.binary.common.BinaryContentSerializerModuleConfigurator
import dev.inmo.postssystem.features.content.common.ContentSerializersModuleConfigurator
import dev.inmo.postssystem.features.content.common.OtherContentSerializerModuleConfigurator
import dev.inmo.postssystem.features.content.binary.server.BinaryServerContentStorage
import dev.inmo.postssystem.features.content.common.*
import dev.inmo.postssystem.features.content.server.ServerContentStorage
import dev.inmo.postssystem.features.content.server.ServerContentStorageAggregator
import dev.inmo.postssystem.features.content.text.common.TextContentSerializerModuleConfigurator
import dev.inmo.postssystem.features.content.text.server.TextServerContentStorage
import dev.inmo.postssystem.features.publication.server.PublicationManager
import dev.inmo.postssystem.features.publication.server.PublicationTarget
import dev.inmo.postssystem.targets.telegram.publication.server.PublicationTargetTelegram
@ -70,8 +73,12 @@ fun getDIModule(
val config = configJson.decodeFromString(Config.serializer(), File(args.first()).readText())
val originalFilesMetasKeyValueRepoQualifier = StringQualifier("OriginalFilesMetaKV")
val filesMetasKeyValueRepoQualifier = StringQualifier("FilesMetaKV")
val filesFolderQualifier = StringQualifier("filesFolder")
val binaryOriginalFilesMetasKeyValueRepoQualifier = StringQualifier("BinaryOriginalFilesMetaKV")
val commonFilesMetasKeyValueRepoQualifier = StringQualifier("CommonFilesMetaKV")
val binaryFilesMetasKeyValueRepoQualifier = StringQualifier("BinaryFilesMetaKV")
val filesFolderQualifier = StringQualifier("rootFilesFolder")
val commonFilesFolderQualifier = StringQualifier("commonFilesFolder")
val binaryFilesFolderQualifier = StringQualifier("binaryFilesFolder")
val usersRolesKeyValueFactoryQualifier = StringQualifier("usersRolesKeyValueFactory")
return module {
@ -100,20 +107,25 @@ fun getDIModule(
singleWithBinds { get<Config>().databaseConfig }
singleWithBinds { get<Config>().authConfig }
singleWithBinds(filesFolderQualifier) { get<Config>().filesFolderFile }
singleWithBinds(commonFilesFolderQualifier) { get<Config>().commonFilesFolder }
singleWithBinds(binaryFilesFolderQualifier) { get<Config>().binaryFilesFolder }
singleWithBinds { get<DatabaseConfig>().database }
singleWithBinds(originalFilesMetasKeyValueRepoQualifier) {
ExposedKeyValueRepo(get(), { text("fileid") }, { text("metaInfo") }, "FileIdsToMetas")
}
singleWithBinds(filesMetasKeyValueRepoQualifier) {
singleWithBinds(binaryOriginalFilesMetasKeyValueRepoQualifier) {
ExposedKeyValueRepo(get(), { text("fileid") }, { text("metaInfo") }, "BinaryContentFileIdsToMetas")
}
singleWithBinds(commonFilesMetasKeyValueRepoQualifier) {
MetasKeyValueRepo(
get(),
get(originalFilesMetasKeyValueRepoQualifier)
)
}
single<FilesStorage> { DiskFilesStorage(get(filesFolderQualifier), get(filesMetasKeyValueRepoQualifier)) }
single<WriteFilesStorage> { WriteDistFilesStorage(get(filesFolderQualifier), get(filesMetasKeyValueRepoQualifier)) }
single<FullFilesStorage> { DefaultFullFilesStorage(get(), get()) }
single<ReadFilesStorage> { DiskReadFilesStorage(get(commonFilesFolderQualifier), get(commonFilesMetasKeyValueRepoQualifier)) }
single<WriteFilesStorage> { WriteDistFilesStorage(get(commonFilesFolderQualifier), get(commonFilesMetasKeyValueRepoQualifier)) }
single<FilesStorage> { DefaultFilesStorage(get(), get()) }
singleWithBinds { ExposedUsersStorage(get()) }
singleWithBinds { exposedUsersAuthenticator(get(), get()) }
@ -142,6 +154,25 @@ fun getDIModule(
factory<CoroutineScope> { baseScope.LinkedSupervisorScope() }
// Content storages
val binaryStorageFilesQualifier = StringQualifier("binaryContentFiles")
singleWithBinds(binaryFilesMetasKeyValueRepoQualifier) {
MetasKeyValueRepo(
get(),
get(binaryOriginalFilesMetasKeyValueRepoQualifier)
)
}
single<FilesStorage>(binaryStorageFilesQualifier) {
DefaultFilesStorage(
DiskReadFilesStorage(get(binaryFilesFolderQualifier), get(binaryFilesMetasKeyValueRepoQualifier)),
WriteDistFilesStorage(get(binaryFilesFolderQualifier), get(binaryFilesMetasKeyValueRepoQualifier))
)
}
singleWithRandomQualifier { BinaryServerContentStorage(get(binaryStorageFilesQualifier)) }
singleWithRandomQualifier { TextServerContentStorage(get()) }
single<ServerContentStorage<Content>> { ServerContentStorageAggregator(getAll(), get()) }
// Routing configurators
singleWithBinds { FilesRoutingConfigurator(get(), null, get()) }
singleWithBinds { StatusRoutingConfigurator }