From 03117ac565b3a8437443b1e6de7142014e17c25a Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Tue, 15 Mar 2022 15:31:58 +0600 Subject: [PATCH 1/3] start 0.9.14 --- CHANGELOG.md | 2 ++ gradle.properties | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0afde485962..fbfb7482175 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 0.9.14 + ## 0.9.13 * `Versions`: diff --git a/gradle.properties b/gradle.properties index 5136fde7f8c..1b5a7190bbd 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,5 +14,5 @@ crypto_js_version=4.1.1 # Project data group=dev.inmo -version=0.9.13 -android_code_version=103 +version=0.9.14 +android_code_version=104 From 21195e1bcb2c7184d2f16e9f6c586e46b69a39f9 Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Tue, 15 Mar 2022 15:33:45 +0600 Subject: [PATCH 2/3] temporal files uploading functionality --- CHANGELOG.md | 3 + .../micro_utils/ktor/client/TemporalUpload.kt | 19 +++ .../ktor/client/ActualTemporalUpload.kt | 54 +++++++ .../ktor/client/ActualTemporalUpload.kt | 39 ++++++ ktor/common/build.gradle | 1 + .../micro_utils/ktor/common/TemporalFiles.kt | 8 ++ .../ktor/server/ServerRoutingShortcuts.kt | 4 + .../TemporalFilesRoutingConfigurator.kt | 132 ++++++++++++++++++ 8 files changed, 260 insertions(+) create mode 100644 ktor/client/src/commonMain/kotlin/dev/inmo/micro_utils/ktor/client/TemporalUpload.kt create mode 100644 ktor/client/src/jsMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt create mode 100644 ktor/client/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt create mode 100644 ktor/common/src/commonMain/kotlin/dev/inmo/micro_utils/ktor/common/TemporalFiles.kt create mode 100644 ktor/server/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/server/TemporalFilesRoutingConfigurator.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index fbfb7482175..f8da494186c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## 0.9.14 +* `Ktor`: + * Add temporal files uploading functionality (for clients to upload and for server to receive) + ## 0.9.13 * `Versions`: diff --git a/ktor/client/src/commonMain/kotlin/dev/inmo/micro_utils/ktor/client/TemporalUpload.kt b/ktor/client/src/commonMain/kotlin/dev/inmo/micro_utils/ktor/client/TemporalUpload.kt new file mode 100644 index 00000000000..a00f85f904d --- /dev/null +++ b/ktor/client/src/commonMain/kotlin/dev/inmo/micro_utils/ktor/client/TemporalUpload.kt @@ -0,0 +1,19 @@ +package dev.inmo.micro_utils.ktor.client + +import dev.inmo.micro_utils.common.MPPFile +import dev.inmo.micro_utils.ktor.common.* +import io.ktor.client.HttpClient + +expect suspend fun HttpClient.tempUpload( + fullTempUploadDraftPath: String, + file: MPPFile, + onUpload: (uploaded: Long, count: Long) -> Unit = { _, _ -> } +): TemporalFileId + +suspend fun UnifiedRequester.tempUpload( + fullTempUploadDraftPath: String, + file: MPPFile, + onUpload: (uploaded: Long, count: Long) -> Unit = { _, _ -> } +): TemporalFileId = client.tempUpload( + fullTempUploadDraftPath, file, onUpload +) diff --git a/ktor/client/src/jsMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt b/ktor/client/src/jsMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt new file mode 100644 index 00000000000..98070bd2430 --- /dev/null +++ b/ktor/client/src/jsMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt @@ -0,0 +1,54 @@ +package dev.inmo.micro_utils.ktor.client + +import dev.inmo.micro_utils.common.MPPFile +import dev.inmo.micro_utils.ktor.common.TemporalFileId +import io.ktor.client.HttpClient +import kotlinx.coroutines.* +import org.w3c.xhr.* + +suspend fun tempUpload( + fullTempUploadDraftPath: String, + file: MPPFile, + onUpload: (Long, Long) -> Unit +): TemporalFileId { + val formData = FormData() + val answer = CompletableDeferred() + + formData.append( + "data", + file + ) + + val request = XMLHttpRequest() + request.responseType = XMLHttpRequestResponseType.TEXT + request.upload.onprogress = { + onUpload(it.loaded.toLong(), it.total.toLong()) + } + request.onload = { + if (request.status == 200.toShort()) { + answer.complete(TemporalFileId(request.responseText)) + } else { + answer.completeExceptionally(Exception("Something went wrong")) + } + } + request.onerror = { + answer.completeExceptionally(Exception("Something went wrong")) + } + request.open("POST", fullTempUploadDraftPath, true) + request.send(formData) + + currentCoroutineContext().job.invokeOnCompletion { + runCatching { + request.abort() + } + } + + return answer.await() +} + + +actual suspend fun HttpClient.tempUpload( + fullTempUploadDraftPath: String, + file: MPPFile, + onUpload: (uploaded: Long, count: Long) -> Unit +): TemporalFileId = dev.inmo.micro_utils.ktor.client.tempUpload(fullTempUploadDraftPath, file, onUpload) diff --git a/ktor/client/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt b/ktor/client/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt new file mode 100644 index 00000000000..f7fbdf9c727 --- /dev/null +++ b/ktor/client/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt @@ -0,0 +1,39 @@ +package dev.inmo.micro_utils.ktor.client + +import dev.inmo.micro_utils.common.MPPFile +import dev.inmo.micro_utils.common.filename +import dev.inmo.micro_utils.ktor.common.TemporalFileId +import io.ktor.client.HttpClient +import io.ktor.client.features.onUpload +import io.ktor.client.request.forms.formData +import io.ktor.client.request.forms.submitFormWithBinaryData +import io.ktor.http.Headers +import io.ktor.http.HttpHeaders +import java.net.URLConnection + +internal val MPPFile.mimeType: String + get() = URLConnection.getFileNameMap().getContentTypeFor(filename.name) ?: "*/*" + +actual suspend fun HttpClient.tempUpload( + fullTempUploadDraftPath: String, + file: MPPFile, + onUpload: (Long, Long) -> Unit +): TemporalFileId { + val inputProvider = file.inputProvider() + val fileId = submitFormWithBinaryData( + fullTempUploadDraftPath, + formData = formData { + append( + "data", + inputProvider, + Headers.build { + append(HttpHeaders.ContentType, file.mimeType) + append(HttpHeaders.ContentDisposition, "filename=\"${file.filename.string}\"") + } + ) + } + ) { + onUpload(onUpload) + } + return TemporalFileId(fileId) +} diff --git a/ktor/common/build.gradle b/ktor/common/build.gradle index ad2dd123f61..7446daad0c7 100644 --- a/ktor/common/build.gradle +++ b/ktor/common/build.gradle @@ -13,6 +13,7 @@ kotlin { api internalProject("micro_utils.common") api libs.kt.serialization.cbor api libs.klock + api libs.uuid } } } diff --git a/ktor/common/src/commonMain/kotlin/dev/inmo/micro_utils/ktor/common/TemporalFiles.kt b/ktor/common/src/commonMain/kotlin/dev/inmo/micro_utils/ktor/common/TemporalFiles.kt new file mode 100644 index 00000000000..cabeffa5bdb --- /dev/null +++ b/ktor/common/src/commonMain/kotlin/dev/inmo/micro_utils/ktor/common/TemporalFiles.kt @@ -0,0 +1,8 @@ +package dev.inmo.micro_utils.ktor.common + +import kotlin.jvm.JvmInline + +const val DefaultTemporalFilesSubPath = "temp_upload" + +@JvmInline +value class TemporalFileId(val string: String) diff --git a/ktor/server/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/server/ServerRoutingShortcuts.kt b/ktor/server/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/server/ServerRoutingShortcuts.kt index f2c06bc5e52..5fd9357630e 100644 --- a/ktor/server/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/server/ServerRoutingShortcuts.kt +++ b/ktor/server/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/server/ServerRoutingShortcuts.kt @@ -92,6 +92,10 @@ class UnifiedRouter( call.respond(HttpStatusCode.BadRequest, "Request query parameters must contains $field") } } + + companion object { + val default = defaultUnifiedRouter + } } val defaultUnifiedRouter = UnifiedRouter() diff --git a/ktor/server/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/server/TemporalFilesRoutingConfigurator.kt b/ktor/server/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/server/TemporalFilesRoutingConfigurator.kt new file mode 100644 index 00000000000..9d33c127cbf --- /dev/null +++ b/ktor/server/src/jvmMain/kotlin/dev/inmo/micro_utils/ktor/server/TemporalFilesRoutingConfigurator.kt @@ -0,0 +1,132 @@ +package dev.inmo.micro_utils.ktor.server + +import com.benasher44.uuid.uuid4 +import dev.inmo.micro_utils.common.FileName +import dev.inmo.micro_utils.common.MPPFile +import dev.inmo.micro_utils.coroutines.launchSafelyWithoutExceptions +import dev.inmo.micro_utils.ktor.common.DefaultTemporalFilesSubPath +import dev.inmo.micro_utils.ktor.common.TemporalFileId +import dev.inmo.micro_utils.ktor.server.configurators.ApplicationRoutingConfigurator +import io.ktor.application.call +import io.ktor.http.HttpStatusCode +import io.ktor.http.content.PartData +import io.ktor.http.content.streamProvider +import io.ktor.request.receiveMultipart +import io.ktor.response.respond +import io.ktor.routing.Route +import io.ktor.routing.post +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.* +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.io.File +import java.nio.file.Files +import java.nio.file.attribute.FileTime + +class TemporalFilesRoutingConfigurator( + private val subpath: String = DefaultTemporalFilesSubPath, + private val unifiedRouter: UnifiedRouter = UnifiedRouter.default, + private val temporalFilesUtilizer: TemporalFilesUtilizer = TemporalFilesUtilizer +) : ApplicationRoutingConfigurator.Element { + interface TemporalFilesUtilizer { + fun start(filesMap: MutableMap, filesMutex: Mutex, onNewFileFlow: Flow): Job + + companion object : TemporalFilesUtilizer { + class ByTimerUtilizer( + private val removeMillis: Long, + private val scope: CoroutineScope + ) : TemporalFilesUtilizer { + override fun start( + filesMap: MutableMap, + filesMutex: Mutex, + onNewFileFlow: Flow + ): Job = scope.launchSafelyWithoutExceptions { + while (isActive) { + val filesWithCreationInfo = filesMap.mapNotNull { (fileId, file) -> + fileId to ((Files.getAttribute(file.toPath(), "creationTime") as? FileTime) ?.toMillis() ?: return@mapNotNull null) + } + if (filesWithCreationInfo.isEmpty()) { + delay(removeMillis) + continue + } + var min = filesWithCreationInfo.first() + for (fileWithCreationInfo in filesWithCreationInfo) { + if (fileWithCreationInfo.second < min.second) { + min = fileWithCreationInfo + } + } + delay(System.currentTimeMillis() - (min.second + removeMillis)) + filesMutex.withLock { + filesMap.remove(min.first) + } ?.delete() + } + + } + } + + override fun start( + filesMap: MutableMap, + filesMutex: Mutex, + onNewFileFlow: Flow + ): Job = Job() + } + } + + private val temporalFilesMap = mutableMapOf() + private val temporalFilesMutex = Mutex() + private val filesFlow = MutableSharedFlow() + val utilizerJob = temporalFilesUtilizer.start(temporalFilesMap, temporalFilesMutex, filesFlow.asSharedFlow()) + + override fun Route.invoke() { + post(subpath) { + unifiedRouter.apply { + val multipart = call.receiveMultipart() + + var fileInfo: Pair? = null + var part = multipart.readPart() + + while (part != null) { + if (part is PartData.FileItem) { + break + } + part = multipart.readPart() + } + + part ?.let { + if (it is PartData.FileItem) { + val fileId = TemporalFileId(uuid4().toString()) + val fileName = it.originalFileName ?.let { FileName(it) } ?: return@let + fileInfo = fileId to File.createTempFile(fileId.string, ".${fileName.extension}").apply { + outputStream().use { outputStream -> + it.streamProvider().use { + it.copyTo(outputStream) + } + } + deleteOnExit() + } + } + } + + fileInfo ?.also { (fileId, file) -> + temporalFilesMutex.withLock { + temporalFilesMap[fileId] = file + } + call.respond(fileId.string) + launchSafelyWithoutExceptions { filesFlow.emit(fileId) } + } ?: call.respond(HttpStatusCode.BadRequest) + } + } + } + + suspend fun removeTemporalFile(temporalFileId: TemporalFileId) { + temporalFilesMutex.withLock { + temporalFilesMap.remove(temporalFileId) + } + } + + fun getTemporalFile(temporalFileId: TemporalFileId) = temporalFilesMap[temporalFileId] + + suspend fun getAndRemoveTemporalFile(temporalFileId: TemporalFileId) = temporalFilesMutex.withLock { + temporalFilesMap.remove(temporalFileId) + } +} From 9c667f4b7861f0ccb9450db1ec9f028d63723490 Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Tue, 15 Mar 2022 15:37:41 +0600 Subject: [PATCH 3/3] several fixes in actual temporal upload of js client --- .../micro_utils/ktor/client/ActualTemporalUpload.kt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ktor/client/src/jsMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt b/ktor/client/src/jsMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt index 98070bd2430..0824c19c516 100644 --- a/ktor/client/src/jsMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt +++ b/ktor/client/src/jsMain/kotlin/dev/inmo/micro_utils/ktor/client/ActualTemporalUpload.kt @@ -28,22 +28,26 @@ suspend fun tempUpload( if (request.status == 200.toShort()) { answer.complete(TemporalFileId(request.responseText)) } else { - answer.completeExceptionally(Exception("Something went wrong")) + answer.completeExceptionally(Exception("Something went wrong: $it")) } } request.onerror = { - answer.completeExceptionally(Exception("Something went wrong")) + answer.completeExceptionally(Exception("Something went wrong: $it")) } request.open("POST", fullTempUploadDraftPath, true) request.send(formData) - currentCoroutineContext().job.invokeOnCompletion { + val handle = currentCoroutineContext().job.invokeOnCompletion { runCatching { request.abort() } } - return answer.await() + return runCatching { + answer.await() + }.also { + handle.dispose() + }.getOrThrow() }