From 555e1da3551888f8dd82e22c836464dcbe401dc8 Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Mon, 19 Aug 2019 14:13:43 +0600 Subject: [PATCH] started 0.4.0 --- CHANGELOG.md | 6 +- build.gradle | 2 +- gradle.properties | 8 +- .../insanusmokrassar/SauceNaoAPI/Constants.kt | 4 + .../insanusmokrassar/SauceNaoAPI/Launcher.kt | 18 ++- .../SauceNaoAPI/SauceNaoAPI.kt | 131 +++++++++++++++--- .../additional/header/AccountInfo.kt | 8 +- .../exceptions/TooManyRequestsException.kt | 13 ++ .../SauceNaoAPI/models/Header.kt | 8 +- 9 files changed, 160 insertions(+), 38 deletions(-) create mode 100644 src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt create mode 100644 src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/exceptions/TooManyRequestsException.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f5013d..dd7768e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ -# 0.3.0 +# SauceNaoAPI Changelog + +## 0.4.0 + +## 0.3.0 * Now `results` field of `SauceNaoAnswer` is optional and is empty list by default * Adapted structure almost completed and now can be used with raw results diff --git a/build.gradle b/build.gradle index b12728e..1e90e37 100644 --- a/build.gradle +++ b/build.gradle @@ -1,4 +1,4 @@ -project.version = "0.3.0" +project.version = "0.4.0" project.group = "com.github.insanusmokrassar" buildscript { diff --git a/gradle.properties b/gradle.properties index c3cb8f2..b4426c2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,9 +1,9 @@ kotlin.code.style=official -kotlin_version=1.3.31 -kotlin_coroutines_version=1.2.1 -kotlin_serialisation_runtime_version=0.11.0 +kotlin_version=1.3.41 +kotlin_coroutines_version=1.2.2 +kotlin_serialisation_runtime_version=0.11.1 joda_time_version=2.10.1 -ktor_version=1.1.4 +ktor_version=1.2.3 project_public_name=SauceNao API project_public_description=SauceNao API library diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt new file mode 100644 index 0000000..9698774 --- /dev/null +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt @@ -0,0 +1,4 @@ +package com.github.insanusmokrassar.SauceNaoAPI + +const val LONG_TIME_LIMIT_MILLIS: Int = 1 * 24 * 60 * 60 * 1000 +const val SHORT_TIME_LIMIT_MILLIS: Int = 30 * 1000 diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt index 81e0133..10f3c99 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt @@ -1,16 +1,22 @@ package com.github.insanusmokrassar.SauceNaoAPI -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.* fun main(vararg args: String) { val key = args.first() val api = SauceNaoAPI(key) - runBlocking { - api.request( - args[1] - ).also { - println(it) + val launch = GlobalScope.launch { + api.use { + it.request( + args[1] + ).also { + println(it) + } } } + + runBlocking { + launch.join() + } } diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt index 86c44ee..0493761 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt @@ -1,13 +1,19 @@ package com.github.insanusmokrassar.SauceNaoAPI +import com.github.insanusmokrassar.SauceNaoAPI.exceptions.sauceNaoAPIException import com.github.insanusmokrassar.SauceNaoAPI.models.SauceNaoAnswer import io.ktor.client.HttpClient -import io.ktor.client.call.call import io.ktor.client.engine.okhttp.OkHttp -import io.ktor.client.request.parameter -import io.ktor.client.request.url +import io.ktor.client.features.ClientRequestException +import io.ktor.client.request.* import io.ktor.client.response.readText +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import kotlinx.io.core.Closeable import kotlinx.serialization.json.Json +import org.joda.time.DateTime +import java.util.logging.Logger +import kotlin.coroutines.* private const val API_TOKEN_FIELD = "api_key" private const val OUTPUT_TYPE_FIELD = "output_type" @@ -25,7 +31,40 @@ data class SauceNaoAPI( private val outputType: OutputType = JsonOutputType, private val client: HttpClient = HttpClient(OkHttp), private val searchUrl: String = SEARCH_URL -) { +) : Closeable { + private val logger = Logger.getLogger("SauceNaoAPI") + + private val requestsChannel = Channel, HttpRequestBuilder>>(Channel.UNLIMITED) + private val requestsSendTimes = mutableListOf() + + init { + CoroutineScope(Dispatchers.Default).launch { + for ((callback, requestBuilder) in requestsChannel) { + try { + val answer = makeRequest(requestBuilder) + callback.resumeWith(Result.success(answer)) + + val sleepUntil = if (answer.header.longRemaining == 0) { + getMostOldestInLongPeriod() ?.plusMillis(LONG_TIME_LIMIT_MILLIS) + } else { + if (answer.header.shortRemaining == 0) { + getMostOldestInShortPeriod() ?.plusMillis(SHORT_TIME_LIMIT_MILLIS) + } else { + null + } + } + + sleepUntil ?.also { _ -> + logger.warning("LONG LIMIT REACHED, SLEEP UNTIL $sleepUntil") + delay(sleepUntil.millis - DateTime.now().millis) + } + } catch (e: Exception) { + callback.resumeWith(Result.failure(e)) + } + } + } + } + suspend fun request( url: String, resultsCount: Int? = null, @@ -72,6 +111,57 @@ data class SauceNaoAPI( minSimilarity = minSimilarity ) + private suspend fun makeRequest( + builder: HttpRequestBuilder + ): SauceNaoAnswer { + return try { + val call = client.execute(builder) + val answerText = call.response.readText() + logger.info(answerText) + addRequestTimesAndClear() + Json.nonstrict.parse( + SauceNaoAnswer.serializer(), + answerText + ) + } catch (e: ClientRequestException) { + throw e.sauceNaoAPIException + } + } + + private fun addRequestTimesAndClear() { + val newDateTime = DateTime.now() + + clearRequestTimes(newDateTime) + + requestsSendTimes.add(newDateTime) + } + + private fun clearRequestTimes(relatedTo: DateTime = DateTime.now()) { + val limitValue = relatedTo.minusMillis(LONG_TIME_LIMIT_MILLIS) + + requestsSendTimes.removeAll { + it < limitValue + } + } + + private fun getMostOldestInLongPeriod(): DateTime? { + clearRequestTimes() + + return requestsSendTimes.min() + } + + private fun getMostOldestInShortPeriod(): DateTime? { + val now = DateTime.now() + + val limitTime = now.minusMillis(SHORT_TIME_LIMIT_MILLIS) + + clearRequestTimes(now) + + return requestsSendTimes.asSequence().filter { + limitTime < it + }.min() + } + private suspend fun makeRequest( url: String, db: Int? = null, @@ -80,21 +170,26 @@ data class SauceNaoAPI( resultsCount: Int? = null, minSimilarity: Float? = null ): SauceNaoAnswer? { - return client.call { - url(searchUrl) - parameter(URL_FIELD, url) - parameter(API_TOKEN_FIELD, apiToken) - parameter(OUTPUT_TYPE_FIELD, outputType.typeCode) - db ?.also { parameter(DB_FIELD, it) } - dbmask ?.also { parameter(DBMASK_FIELD, it) } - dbmaski ?.also { parameter(DBMASKI_FIELD, it) } - resultsCount ?.also { parameter(RESULTS_COUNT_FIELD, it) } - minSimilarity ?.also { parameter(MINIMAL_SIMILARITY_FIELD, it) } - }.response.readText().let { - Json.nonstrict.parse( - SauceNaoAnswer.serializer(), - it + return suspendCoroutine { + requestsChannel.offer( + it to HttpRequestBuilder().apply { + url(searchUrl) + parameter(URL_FIELD, url) + parameter(API_TOKEN_FIELD, apiToken) + parameter(OUTPUT_TYPE_FIELD, outputType.typeCode) + db ?.also { parameter(DB_FIELD, it) } + dbmask ?.also { parameter(DBMASK_FIELD, it) } + dbmaski ?.also { parameter(DBMASKI_FIELD, it) } + resultsCount ?.also { parameter(RESULTS_COUNT_FIELD, it) } + minSimilarity ?.also { parameter(MINIMAL_SIMILARITY_FIELD, it) } + } ) } } + + override fun close() { + requestsChannel.close() + requestsSendTimes.clear() + client.close() + } } \ No newline at end of file diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/additional/header/AccountInfo.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/additional/header/AccountInfo.kt index 710c286..fce54ac 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/additional/header/AccountInfo.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/additional/header/AccountInfo.kt @@ -5,14 +5,14 @@ import com.github.insanusmokrassar.SauceNaoAPI.models.Header val Header.shortLimitStatus: LimitStatus get() = LimitStatus( - shortRemaining ?: Int.MAX_VALUE, - shortLimit ?: Int.MAX_VALUE + shortRemaining, + shortLimit ) val Header.longLimitStatus: LimitStatus get() = LimitStatus( - longRemaining ?: Int.MAX_VALUE, - longLimit ?: Int.MAX_VALUE + longRemaining, + longLimit ) val Header.limits diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/exceptions/TooManyRequestsException.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/exceptions/TooManyRequestsException.kt new file mode 100644 index 0000000..e177104 --- /dev/null +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/exceptions/TooManyRequestsException.kt @@ -0,0 +1,13 @@ +package com.github.insanusmokrassar.SauceNaoAPI.exceptions + +import io.ktor.client.features.ClientRequestException +import io.ktor.http.HttpStatusCode.Companion.TooManyRequests +import kotlinx.io.IOException + +val ClientRequestException.sauceNaoAPIException: Exception + get() = when (response.status) { + TooManyRequests -> TooManyRequestsException() + else -> this + } + +class TooManyRequestsException : IOException() diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/models/Header.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/models/Header.kt index 8406825..15e54fd 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/models/Header.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/models/Header.kt @@ -25,13 +25,13 @@ data class Header( @SerialName("query_image") val queryImage: String? = null, // something like "uuid.jpg" @SerialName("short_remaining") - val shortRemaining: Int? = null, + val shortRemaining: Int = Int.MAX_VALUE, @SerialName("long_remaining") - val longRemaining: Int? = null, + val longRemaining: Int = Int.MAX_VALUE, @SerialName("short_limit") - val shortLimit: Int? = null, + val shortLimit: Int = Int.MAX_VALUE, @SerialName("long_limit") - val longLimit: Int? = null, + val longLimit: Int = Int.MAX_VALUE, @SerialName("account_type") val accountType: Int? = null, @SerialName("user_id")