PostsSystem/core
1
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

temp progress on binary correct including

Browse Source
This commit is contained in:
InsanusMokrassar 2022-03-09 13:49:00 +06:00
parent 186bfd7ac0
commit 99b953635e
15 changed files with 342 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
features/common/common/build.gradle
View File

@ -12,6 +12,7 @@ kotlin {
dependencies {
api libs.microutils.common
api libs.microutils.serialization.typedserializer
api libs.microutils.mimetypes
api libs.klock
api "io.insert-koin:koin-core:$koin_version"
api "com.benasher44:uuid:$uuid_version"

6
features/common/common/src/commonMain/kotlin/dev/inmo/postssystem/features/common/common/FileMimeType.kt Normal file
View File

@ -0,0 +1,6 @@
package dev.inmo.postssystem.features.common.common
import dev.inmo.micro_utils.common.MPPFile
import dev.inmo.micro_utils.mime_types.MimeType
expect val MPPFile.mimeType: MimeType

8
features/common/common/src/jsMain/kotlin/dev/inmo/postssystem/features/common/common/ActualFileMimeType.kt Normal file
View File

@ -0,0 +1,8 @@
package dev.inmo.postssystem.features.common.common
import dev.inmo.micro_utils.common.MPPFile
import dev.inmo.micro_utils.mime_types.*
actual val MPPFile.mimeType: MimeType
get() = findBuiltinMimeType(type) ?: KnownMimeTypes.Any

10
features/common/common/src/jvmMain/kotlin/dev/inmo/postssystem/features/common/common/ActualFileMimeType.kt Normal file
View File

@ -0,0 +1,10 @@
package dev.inmo.postssystem.features.common.common
import dev.inmo.micro_utils.common.MPPFile
import dev.inmo.micro_utils.common.filename
import dev.inmo.micro_utils.mime_types.*
import java.net.URLConnection
actual val MPPFile.mimeType: MimeType
get() = URLConnection.getFileNameMap().getContentTypeFor(filename.name) ?.let(::findBuiltinMimeType) ?: KnownMimeTypes.Any

120
services/posts/client/src/commonMain/kotlin/dev/inmo/postssystem/services/posts/client/ClientWritePostsService.kt
View File

@ -7,6 +7,8 @@ import dev.inmo.micro_utils.ktor.common.encodeHex
import dev.inmo.micro_utils.repos.ktor.common.crud.createRouting
import dev.inmo.micro_utils.repos.ktor.common.crud.updateRouting
import dev.inmo.micro_utils.repos.ktor.common.one_to_many.removeRoute
import dev.inmo.postssystem.features.common.common.FileBasedInputProvider
import dev.inmo.postssystem.features.common.common.SimpleInputProvider
import dev.inmo.postssystem.features.content.common.*
import dev.inmo.postssystem.features.posts.common.PostId
import dev.inmo.postssystem.features.posts.common.RegisteredPost
@ -20,9 +22,15 @@ import kotlinx.serialization.builtins.*
class ClientWritePostsService(
private val baseUrl: String,
private val unifiedRequester: UnifiedRequester
unifiedRequester: UnifiedRequester
) : WritePostsService {
private val root = buildStandardUrl(baseUrl, postsRootPath)
private val unifiedRequester = UnifiedRequester(
unifiedRequester.client,
unifiedRequester.serialFormat.createWithSerializerModuleExtension {
contextual()
}
)
private val contentEitherSerializer = EitherSerializer(ContentId.serializer(), ContentSerializer)
private val contentsEitherSerializer = ListSerializer(contentEitherSerializer)
@ -36,83 +44,59 @@ class ClientWritePostsService(
root,
removeRoute
)
private val tempUploadFullPath = buildStandardUrl(
baseUrl,
postsCreateTempPathPart
)
private suspend fun prepareContent(content: Content): Content? {
return (content as? BinaryContent) ?.let {
when (val provider = it.inputProvider) {
is FileBasedInputProvider -> {
val fileId = unifiedRequester.tempUpload(
tempUploadFullPath,
provider.file
)
it.copy(inputProvider = TempFileIdentifierInputProvider(fileId))
}
is TempFileIdentifierInputProvider -> it
else -> return@prepareContent null
}
} ?: content
}
override suspend fun create(newPost: FullNewPost): RegisteredPost? {
return if (newPost.content.any { it is BinaryContent }) {
val answer = unifiedRequester.client.post<ByteArray>(createFullPath) {
formData {
newPost.content.forEachIndexed { i, content ->
when (content) {
is BinaryContent -> append(
i.toString(),
InputProvider(block = content.inputProvider::invoke),
headers {
append(HttpHeaders.ContentType, content.mimeType.raw)
append(HttpHeaders.ContentDisposition, "filename=\"${content.filename.name}\"")
}.build()
)
else -> append(
i.toString(),
unifiedRequester.serialFormat.encodeHex(ContentSerializer, content)
)
}
}
}
}
unifiedRequester.serialFormat.decodeFromByteArray(RegisteredPost.serializer().nullable, answer)
} else {
unifiedRequester.unipost(
createFullPath,
contentsSerializer to newPost.content,
RegisteredPost.serializer().nullable
)
val mappedContent = newPost.content.mapNotNull {
prepareContent(it)
}
val mappedPost = newPost.copy(
content = mappedContent
)
return unifiedRequester.unipost(
createFullPath,
contentsSerializer to mappedPost.content,
RegisteredPost.serializer().nullable
)
}
override suspend fun update(
postId: PostId,
content: List<Either<ContentId, Content>>
): RegisteredPost? {
return if (content.any { it.optionalT2.data is BinaryContent }) {
val answer = unifiedRequester.client.post<ByteArray>(createFullPath) {
formData {
content.forEachIndexed { i, eitherContent ->
eitherContent.onFirst {
append(
i.toString(),
unifiedRequester.serialFormat.encodeHex(contentEitherSerializer, it.either())
)
}.onSecond {
when (it) {
is BinaryContent -> append(
i.toString(),
InputProvider(block = it.inputProvider::invoke),
headers {
append(HttpHeaders.ContentType, it.mimeType.raw)
append(HttpHeaders.ContentDisposition, "filename=\"${it.filename.name}\"")
}.build()
)
else -> append(
i.toString(),
unifiedRequester.serialFormat.encodeHex(contentEitherSerializer, it.either())
)
}
}
}
}
}
unifiedRequester.serialFormat.decodeFromByteArray(RegisteredPost.serializer().nullable, answer)
} else {
unifiedRequester.unipost(
buildStandardUrl(
root,
updateRouting,
postsPostIdParameter to unifiedRequester.encodeUrlQueryValue(PostId.serializer(), postId)
),
contentsEitherSerializer to content,
RegisteredPost.serializer().nullable
)
val mappedContent = content.mapNotNull {
it.mapOnSecond { content ->
prepareContent(content) ?.either() ?: return@mapNotNull null
} ?: it
}
return unifiedRequester.unipost(
buildStandardUrl(
root,
updateRouting,
postsPostIdParameter to unifiedRequester.encodeUrlQueryValue(PostId.serializer(), postId)
),
contentsEitherSerializer to mappedContent,
RegisteredPost.serializer().nullable
)
}
override suspend fun remove(postId: PostId) = unifiedRequester.unipost(

15
services/posts/client/src/commonMain/kotlin/dev/inmo/postssystem/services/posts/client/TempFileIdentifierInputProvider.kt Normal file
View File

@ -0,0 +1,15 @@
package dev.inmo.postssystem.services.posts.client
import dev.inmo.postssystem.features.common.common.SimpleInputProvider
import dev.inmo.postssystem.features.files.common.FileId
import io.ktor.utils.io.core.Input
import kotlinx.serialization.Serializable
@Serializable
internal data class TempFileIdentifierInputProvider(
private val tempFile: FileId
) : SimpleInputProvider {
override fun invoke(): Input {
TODO("Not yet implemented")
}
}

12
services/posts/client/src/commonMain/kotlin/dev/inmo/postssystem/services/posts/client/TempUpload.kt Normal file
View File

@ -0,0 +1,12 @@
package dev.inmo.postssystem.services.posts.client
import dev.inmo.micro_utils.common.MPPFile
import dev.inmo.micro_utils.ktor.client.UnifiedRequester
import dev.inmo.postssystem.features.files.common.FileId
internal expect suspend fun UnifiedRequester.tempUpload(
fullTempUploadDraftPath: String,
file: MPPFile,
onUpload: (Long, Long) -> Unit = { _, _ -> }
): FileId

8
services/posts/client/src/commonMain/kotlin/dev/inmo/postssystem/services/posts/client/ui/create/PostCreateUIModel.kt Normal file
View File

@ -0,0 +1,8 @@
package dev.inmo.postssystem.services.posts.client.ui.create
import dev.inmo.postssystem.features.common.common.UIModel
import dev.inmo.postssystem.features.content.common.Content
interface PostCreateUIModel : UIModel<PostCreateUIState> {
suspend fun create(content: List<Content>)
}

14
services/posts/client/src/commonMain/kotlin/dev/inmo/postssystem/services/posts/client/ui/create/PostCreateUIState.kt Normal file
View File

@ -0,0 +1,14 @@
package dev.inmo.postssystem.services.posts.client.ui.create
import kotlinx.serialization.Serializable
@Serializable
sealed class PostCreateUIState {
@Serializable
object Init : PostCreateUIState()
@Serializable
object Uploading : PostCreateUIState()
@Serializable
object Completed : PostCreateUIState()
}

50
services/posts/client/src/jsMain/kotlin/dev/inmo/postssystem/services/posts/client/ActualTempUpload.kt Normal file
View File

@ -0,0 +1,50 @@
package dev.inmo.postssystem.services.posts.client
import dev.inmo.micro_utils.common.MPPFile
import dev.inmo.micro_utils.ktor.client.UnifiedRequester
import dev.inmo.postssystem.features.files.common.FileId
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.GlobalScope.coroutineContext
import kotlinx.coroutines.job
import org.w3c.xhr.*
internal actual suspend fun UnifiedRequester.tempUpload(
fullTempUploadDraftPath: String,
file: MPPFile,
onUpload: (Long, Long) -> Unit
): FileId {
val formData = FormData()
val answer = CompletableDeferred<FileId>()
formData.append(
"data",
file
)
val request = XMLHttpRequest()
request.responseType = XMLHttpRequestResponseType.TEXT
request.upload.onprogress = {
onUpload(it.loaded.toLong(), it.total.toLong())
}
request.onload = {
if (request.status == 200.toShort()) {
answer.complete(FileId(request.responseText))
} else {
answer.completeExceptionally(Exception("Something went wrong"))
}
}
request.onerror = {
answer.completeExceptionally(Exception("Something went wrong"))
}
request.open("POST", fullTempUploadDraftPath, true)
request.send(formData)
coroutineContext.job.invokeOnCompletion {
runCatching {
request.abort()
}
}
return answer.await()
}

38
services/posts/client/src/jvmMain/kotlin/dev/inmo/postssystem/services/posts/client/ActualTempUpload.kt Normal file
View File

@ -0,0 +1,38 @@
package dev.inmo.postssystem.services.posts.client
import dev.inmo.micro_utils.common.MPPFile
import dev.inmo.micro_utils.common.filename
import dev.inmo.micro_utils.ktor.client.UnifiedRequester
import dev.inmo.micro_utils.ktor.client.inputProvider
import dev.inmo.postssystem.features.common.common.mimeType
import dev.inmo.postssystem.features.files.common.FileId
import io.ktor.client.features.onUpload
import io.ktor.client.request.forms.formData
import io.ktor.client.request.forms.submitFormWithBinaryData
import io.ktor.http.Headers
import io.ktor.http.HttpHeaders
internal actual suspend fun UnifiedRequester.tempUpload(
fullTempUploadDraftPath: String,
file: MPPFile,
onUpload: (Long, Long) -> Unit
): FileId {
val inputProvider = file.inputProvider()
val fileId = client.submitFormWithBinaryData<String>(
fullTempUploadDraftPath,
formData = formData {
append(
"data",
inputProvider,
Headers.build {
append(HttpHeaders.ContentType, file.mimeType.raw)
append(HttpHeaders.ContentDisposition, "filename=\"${file.filename.string}\"")
}
)
}
) {
onUpload(onUpload)
}
return FileId(fileId)
}

1
services/posts/common/build.gradle
View File

@ -12,6 +12,7 @@ kotlin {
dependencies {
api project(":postssystem.features.common.common")
api project(":postssystem.features.posts.common")
api project(":postssystem.features.files.common")
api libs.microutils.repos.common
api libs.microutils.repos.ktor.client
}

15
services/posts/common/src/commonMain/kotlin/dev/inmo/postssystem/services/posts/common/Constants.kt
View File

@ -1,5 +1,20 @@
package dev.inmo.postssystem.services.posts.common
import kotlinx.serialization.SerialFormat
import kotlinx.serialization.cbor.Cbor
import kotlinx.serialization.modules.SerializersModule
import kotlinx.serialization.modules.SerializersModuleBuilder
const val postsRootPath = "posts"
const val postsCreateTempPathPart = "temp"
const val postsPostIdParameter = "postId"
fun SerialFormat.createWithSerializerModuleExtension(
configurator: SerializersModuleBuilder.() -> Unit
) = Cbor {
serializersModule = SerializersModule {
include(this@createWithSerializerModuleExtension.serializersModule)
configurator()
}
}

209
services/posts/server/src/jvmMain/kotlin/dev/inmo/postssystem/services/posts/server/ServerPostsServiceRoutingConfigurator.kt
View File

@ -1,6 +1,8 @@
package dev.inmo.postssystem.services.posts.server
import com.benasher44.uuid.uuid4
import dev.inmo.micro_utils.common.*
import dev.inmo.micro_utils.coroutines.launchSafelyWithoutExceptions
import dev.inmo.micro_utils.ktor.common.decodeHex
import dev.inmo.micro_utils.ktor.server.*
import dev.inmo.micro_utils.ktor.server.configurators.ApplicationRoutingConfigurator
@ -11,154 +13,102 @@ import dev.inmo.micro_utils.repos.ktor.common.one_to_many.removeRoute
import dev.inmo.micro_utils.repos.ktor.server.crud.configureReadStandardCrudRepoRoutes
import dev.inmo.postssystem.features.common.common.FileBasedInputProvider
import dev.inmo.postssystem.features.content.common.*
import dev.inmo.postssystem.features.files.common.FileId
import dev.inmo.postssystem.features.posts.common.*
import dev.inmo.postssystem.services.posts.common.*
import io.ktor.application.ApplicationCall
import io.ktor.application.call
import io.ktor.auth.authenticate
import io.ktor.http.HttpStatusCode
import io.ktor.http.content.PartData
import io.ktor.http.content.streamProvider
import io.ktor.request.isMultipart
import io.ktor.request.receiveMultipart
import io.ktor.response.respond
import io.ktor.routing.*
import io.ktor.util.asStream
import io.ktor.util.pipeline.PipelineContext
import io.ktor.utils.io.core.use
import io.ktor.utils.io.streams.asInput
import kotlinx.coroutines.*
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.serialization.builtins.*
import java.io.File
import java.nio.file.Files
import java.nio.file.attribute.FileTime
import java.util.concurrent.TimeUnit
class ServerPostsServiceRoutingConfigurator(
private val readPostsService: ReadPostsService,
private val writePostsService: WritePostsService? = readPostsService as? WritePostsService,
private val scope: CoroutineScope,
private val unifiedRouter: UnifiedRouter
) : ApplicationRoutingConfigurator.Element {
private val contentEitherSerializer = EitherSerializer(ContentId.serializer(), ContentSerializer)
private val contentsEitherSerializer = ListSerializer(contentEitherSerializer)
private val contentsSerializer = ListSerializer(ContentSerializer)
private val temporalFilesMap = mutableMapOf<FileId, MPPFile>()
private val temporalFilesMutex = Mutex()
private val removingJob = scope.launchSafelyWithoutExceptions {
while (isActive) {
val filesWithCreationInfo = temporalFilesMap.mapNotNull { (fileId, file) ->
fileId to ((Files.getAttribute(file.toPath(), "creationTime") as? FileTime) ?.toMillis() ?: return@mapNotNull null)
}
if (filesWithCreationInfo.isEmpty()) {
delay(TimeUnit.HOURS.toMillis(1L))
continue
}
var min = filesWithCreationInfo.first()
for (fileWithCreationInfo in filesWithCreationInfo) {
if (fileWithCreationInfo.second < min.second) {
min = fileWithCreationInfo
}
}
delay(System.currentTimeMillis() - (min.second + TimeUnit.HOURS.toMillis(1)))
temporalFilesMutex.withLock {
temporalFilesMap.remove(min.first)
} ?.delete()
}
}
private suspend fun mapBinary(content: BinaryContent): BinaryContent? {
val provider = content.inputProvider
if (provider is TempFileIdentifierInputProvider) {
val newProvider = temporalFilesMutex.withLock {
temporalFilesMap.remove(provider.tempFile)
} ?.let(::FileBasedInputProvider)
if (newProvider != null) {
return content.copy(
inputProvider = newProvider
)
}
}
return null
}
private suspend fun PipelineContext<Unit, ApplicationCall>.receiveContents(): List<Content> {
return unifiedRouter.run {
if (call.request.isMultipart()) {
val multipart = call.receiveMultipart()
val list = mutableListOf<Pair<String, Content>>()
var part = multipart.readPart()
while (part != null) {
val name = part.name
val capturedPart = part
when {
name == null -> {}
capturedPart is PartData.FormItem -> {
list.add(
name to unifiedRouter.serialFormat.decodeHex(
ContentSerializer,
capturedPart.value
)
)
}
capturedPart is PartData.FileItem -> {
val filename = capturedPart.originalFileName ?.let(::FileName) ?: error("File name is unknown for default part")
val mimeType = capturedPart.contentType ?.let {
findBuiltinMimeType("${it.contentType}/${it.contentSubtype}")
} ?: error("File type is unknown for default part")
val resultInput = MPPFile.createTempFile(
filename.nameWithoutExtension.let {
var resultName = it
while (resultName.length < 3) {
resultName += "_"
}
resultName
},
".${filename.extension}"
).apply {
outputStream().use { fileStream ->
capturedPart.provider().asStream().copyTo(fileStream)
}
}
list.add(
name to BinaryContent(
filename,
mimeType,
FileBasedInputProvider(resultInput)
)
)
}
else -> {}
}
part = multipart.readPart()
}
list.sortedBy { it.first }.map { it.second }
} else {
uniload(contentsSerializer)
uniload(contentsSerializer).mapNotNull {
mapBinary(it as? BinaryContent ?: return@mapNotNull it)
}
}
}
private suspend fun PipelineContext<Unit, ApplicationCall>.receiveContentsEithers(): List<Either<ContentId, Content>> {
return unifiedRouter.run {
if (call.request.isMultipart()) {
val multipart = call.receiveMultipart()
val list = mutableListOf<Pair<String, Either<ContentId, Content>>>()
var part = multipart.readPart()
while (part != null) {
val name = part.name
val capturedPart = part
when {
name == null -> {}
capturedPart is PartData.FormItem -> {
list.add(
name to unifiedRouter.serialFormat.decodeHex(
contentEitherSerializer,
capturedPart.value
)
)
}
capturedPart is PartData.FileItem -> {
val filename = capturedPart.originalFileName ?.let(::FileName) ?: error("File name is unknown for default part")
val mimeType = capturedPart.contentType ?.let {
findBuiltinMimeType("${it.contentType}/${it.contentSubtype}")
} ?: error("File type is unknown for default part")
val resultInput = MPPFile.createTempFile(
filename.nameWithoutExtension.let {
var resultName = it
while (resultName.length < 3) {
resultName += "_"
}
resultName
},
".${filename.extension}"
).apply {
outputStream().use { fileStream ->
capturedPart.provider().asStream().copyTo(fileStream)
}
}
list.add(
name to BinaryContent(
filename,
mimeType,
FileBasedInputProvider(resultInput)
).either()
)
}
else -> {}
}
part = multipart.readPart()
}
list.sortedBy { it.first }.map { it.second }
} else {
uniload(contentsEitherSerializer)
uniload(contentsEitherSerializer).mapNotNull {
it.mapOnSecond {
mapBinary(it as? BinaryContent ?: return@mapOnSecond null) ?.either()
} ?: it
}
}
}
override fun Route.invoke() {
authenticate {
route(postsRootPath) {
@ -205,6 +155,41 @@ class ServerPostsServiceRoutingConfigurator(
)
}
post(postsCreateTempPathPart) {
val multipart = call.receiveMultipart()
var fileInfo: Pair<FileId, MPPFile>? = null
var part = multipart.readPart()
while (part != null) {
if (part is PartData.FileItem) {
break
}
part = multipart.readPart()
}
part ?.let {
if (it is PartData.FileItem) {
val fileId = FileId(uuid4().toString())
val fileName = it.originalFileName ?.let { FileName(it) } ?: return@let
fileInfo = fileId to File.createTempFile(fileId.string, ".${fileName.extension}").apply {
outputStream().use { outputStream ->
it.streamProvider().use {
it.copyTo(outputStream)
}
}
deleteOnExit()
}
}
}
fileInfo ?.also { (fileId, file) ->
temporalFilesMutex.withLock {
temporalFilesMap[fileId] = file
}
call.respond(fileId.string)
} ?: call.respond(HttpStatusCode.BadRequest)
}
}
}

15
services/posts/server/src/jvmMain/kotlin/dev/inmo/postssystem/services/posts/server/TempFileIdentifierInputProvider.kt Normal file
View File

@ -0,0 +1,15 @@
package dev.inmo.postssystem.services.posts.server
import dev.inmo.postssystem.features.common.common.SimpleInputProvider
import dev.inmo.postssystem.features.files.common.FileId
import io.ktor.utils.io.core.Input
import kotlinx.serialization.Serializable
@Serializable
internal data class TempFileIdentifierInputProvider(
val tempFile: FileId
) : SimpleInputProvider {
override fun invoke(): Input {
TODO("Not yet implemented")
}
}