From 555e1da3551888f8dd82e22c836464dcbe401dc8 Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Mon, 19 Aug 2019 14:13:43 +0600 Subject: [PATCH 1/5] started 0.4.0 --- CHANGELOG.md | 6 +- build.gradle | 2 +- gradle.properties | 8 +- .../insanusmokrassar/SauceNaoAPI/Constants.kt | 4 + .../insanusmokrassar/SauceNaoAPI/Launcher.kt | 18 ++- .../SauceNaoAPI/SauceNaoAPI.kt | 131 +++++++++++++++--- .../additional/header/AccountInfo.kt | 8 +- .../exceptions/TooManyRequestsException.kt | 13 ++ .../SauceNaoAPI/models/Header.kt | 8 +- 9 files changed, 160 insertions(+), 38 deletions(-) create mode 100644 src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt create mode 100644 src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/exceptions/TooManyRequestsException.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f5013d..dd7768e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ -# 0.3.0 +# SauceNaoAPI Changelog + +## 0.4.0 + +## 0.3.0 * Now `results` field of `SauceNaoAnswer` is optional and is empty list by default * Adapted structure almost completed and now can be used with raw results diff --git a/build.gradle b/build.gradle index b12728e..1e90e37 100644 --- a/build.gradle +++ b/build.gradle @@ -1,4 +1,4 @@ -project.version = "0.3.0" +project.version = "0.4.0" project.group = "com.github.insanusmokrassar" buildscript { diff --git a/gradle.properties b/gradle.properties index c3cb8f2..b4426c2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,9 +1,9 @@ kotlin.code.style=official -kotlin_version=1.3.31 -kotlin_coroutines_version=1.2.1 -kotlin_serialisation_runtime_version=0.11.0 +kotlin_version=1.3.41 +kotlin_coroutines_version=1.2.2 +kotlin_serialisation_runtime_version=0.11.1 joda_time_version=2.10.1 -ktor_version=1.1.4 +ktor_version=1.2.3 project_public_name=SauceNao API project_public_description=SauceNao API library diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt new file mode 100644 index 0000000..9698774 --- /dev/null +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt @@ -0,0 +1,4 @@ +package com.github.insanusmokrassar.SauceNaoAPI + +const val LONG_TIME_LIMIT_MILLIS: Int = 1 * 24 * 60 * 60 * 1000 +const val SHORT_TIME_LIMIT_MILLIS: Int = 30 * 1000 diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt index 81e0133..10f3c99 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt @@ -1,16 +1,22 @@ package com.github.insanusmokrassar.SauceNaoAPI -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.* fun main(vararg args: String) { val key = args.first() val api = SauceNaoAPI(key) - runBlocking { - api.request( - args[1] - ).also { - println(it) + val launch = GlobalScope.launch { + api.use { + it.request( + args[1] + ).also { + println(it) + } } } + + runBlocking { + launch.join() + } } diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt index 86c44ee..0493761 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt @@ -1,13 +1,19 @@ package com.github.insanusmokrassar.SauceNaoAPI +import com.github.insanusmokrassar.SauceNaoAPI.exceptions.sauceNaoAPIException import com.github.insanusmokrassar.SauceNaoAPI.models.SauceNaoAnswer import io.ktor.client.HttpClient -import io.ktor.client.call.call import io.ktor.client.engine.okhttp.OkHttp -import io.ktor.client.request.parameter -import io.ktor.client.request.url +import io.ktor.client.features.ClientRequestException +import io.ktor.client.request.* import io.ktor.client.response.readText +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import kotlinx.io.core.Closeable import kotlinx.serialization.json.Json +import org.joda.time.DateTime +import java.util.logging.Logger +import kotlin.coroutines.* private const val API_TOKEN_FIELD = "api_key" private const val OUTPUT_TYPE_FIELD = "output_type" @@ -25,7 +31,40 @@ data class SauceNaoAPI( private val outputType: OutputType = JsonOutputType, private val client: HttpClient = HttpClient(OkHttp), private val searchUrl: String = SEARCH_URL -) { +) : Closeable { + private val logger = Logger.getLogger("SauceNaoAPI") + + private val requestsChannel = Channel, HttpRequestBuilder>>(Channel.UNLIMITED) + private val requestsSendTimes = mutableListOf() + + init { + CoroutineScope(Dispatchers.Default).launch { + for ((callback, requestBuilder) in requestsChannel) { + try { + val answer = makeRequest(requestBuilder) + callback.resumeWith(Result.success(answer)) + + val sleepUntil = if (answer.header.longRemaining == 0) { + getMostOldestInLongPeriod() ?.plusMillis(LONG_TIME_LIMIT_MILLIS) + } else { + if (answer.header.shortRemaining == 0) { + getMostOldestInShortPeriod() ?.plusMillis(SHORT_TIME_LIMIT_MILLIS) + } else { + null + } + } + + sleepUntil ?.also { _ -> + logger.warning("LONG LIMIT REACHED, SLEEP UNTIL $sleepUntil") + delay(sleepUntil.millis - DateTime.now().millis) + } + } catch (e: Exception) { + callback.resumeWith(Result.failure(e)) + } + } + } + } + suspend fun request( url: String, resultsCount: Int? = null, @@ -72,6 +111,57 @@ data class SauceNaoAPI( minSimilarity = minSimilarity ) + private suspend fun makeRequest( + builder: HttpRequestBuilder + ): SauceNaoAnswer { + return try { + val call = client.execute(builder) + val answerText = call.response.readText() + logger.info(answerText) + addRequestTimesAndClear() + Json.nonstrict.parse( + SauceNaoAnswer.serializer(), + answerText + ) + } catch (e: ClientRequestException) { + throw e.sauceNaoAPIException + } + } + + private fun addRequestTimesAndClear() { + val newDateTime = DateTime.now() + + clearRequestTimes(newDateTime) + + requestsSendTimes.add(newDateTime) + } + + private fun clearRequestTimes(relatedTo: DateTime = DateTime.now()) { + val limitValue = relatedTo.minusMillis(LONG_TIME_LIMIT_MILLIS) + + requestsSendTimes.removeAll { + it < limitValue + } + } + + private fun getMostOldestInLongPeriod(): DateTime? { + clearRequestTimes() + + return requestsSendTimes.min() + } + + private fun getMostOldestInShortPeriod(): DateTime? { + val now = DateTime.now() + + val limitTime = now.minusMillis(SHORT_TIME_LIMIT_MILLIS) + + clearRequestTimes(now) + + return requestsSendTimes.asSequence().filter { + limitTime < it + }.min() + } + private suspend fun makeRequest( url: String, db: Int? = null, @@ -80,21 +170,26 @@ data class SauceNaoAPI( resultsCount: Int? = null, minSimilarity: Float? = null ): SauceNaoAnswer? { - return client.call { - url(searchUrl) - parameter(URL_FIELD, url) - parameter(API_TOKEN_FIELD, apiToken) - parameter(OUTPUT_TYPE_FIELD, outputType.typeCode) - db ?.also { parameter(DB_FIELD, it) } - dbmask ?.also { parameter(DBMASK_FIELD, it) } - dbmaski ?.also { parameter(DBMASKI_FIELD, it) } - resultsCount ?.also { parameter(RESULTS_COUNT_FIELD, it) } - minSimilarity ?.also { parameter(MINIMAL_SIMILARITY_FIELD, it) } - }.response.readText().let { - Json.nonstrict.parse( - SauceNaoAnswer.serializer(), - it + return suspendCoroutine { + requestsChannel.offer( + it to HttpRequestBuilder().apply { + url(searchUrl) + parameter(URL_FIELD, url) + parameter(API_TOKEN_FIELD, apiToken) + parameter(OUTPUT_TYPE_FIELD, outputType.typeCode) + db ?.also { parameter(DB_FIELD, it) } + dbmask ?.also { parameter(DBMASK_FIELD, it) } + dbmaski ?.also { parameter(DBMASKI_FIELD, it) } + resultsCount ?.also { parameter(RESULTS_COUNT_FIELD, it) } + minSimilarity ?.also { parameter(MINIMAL_SIMILARITY_FIELD, it) } + } ) } } + + override fun close() { + requestsChannel.close() + requestsSendTimes.clear() + client.close() + } } \ No newline at end of file diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/additional/header/AccountInfo.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/additional/header/AccountInfo.kt index 710c286..fce54ac 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/additional/header/AccountInfo.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/additional/header/AccountInfo.kt @@ -5,14 +5,14 @@ import com.github.insanusmokrassar.SauceNaoAPI.models.Header val Header.shortLimitStatus: LimitStatus get() = LimitStatus( - shortRemaining ?: Int.MAX_VALUE, - shortLimit ?: Int.MAX_VALUE + shortRemaining, + shortLimit ) val Header.longLimitStatus: LimitStatus get() = LimitStatus( - longRemaining ?: Int.MAX_VALUE, - longLimit ?: Int.MAX_VALUE + longRemaining, + longLimit ) val Header.limits diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/exceptions/TooManyRequestsException.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/exceptions/TooManyRequestsException.kt new file mode 100644 index 0000000..e177104 --- /dev/null +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/exceptions/TooManyRequestsException.kt @@ -0,0 +1,13 @@ +package com.github.insanusmokrassar.SauceNaoAPI.exceptions + +import io.ktor.client.features.ClientRequestException +import io.ktor.http.HttpStatusCode.Companion.TooManyRequests +import kotlinx.io.IOException + +val ClientRequestException.sauceNaoAPIException: Exception + get() = when (response.status) { + TooManyRequests -> TooManyRequestsException() + else -> this + } + +class TooManyRequestsException : IOException() diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/models/Header.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/models/Header.kt index 8406825..15e54fd 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/models/Header.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/models/Header.kt @@ -25,13 +25,13 @@ data class Header( @SerialName("query_image") val queryImage: String? = null, // something like "uuid.jpg" @SerialName("short_remaining") - val shortRemaining: Int? = null, + val shortRemaining: Int = Int.MAX_VALUE, @SerialName("long_remaining") - val longRemaining: Int? = null, + val longRemaining: Int = Int.MAX_VALUE, @SerialName("short_limit") - val shortLimit: Int? = null, + val shortLimit: Int = Int.MAX_VALUE, @SerialName("long_limit") - val longLimit: Int? = null, + val longLimit: Int = Int.MAX_VALUE, @SerialName("account_type") val accountType: Int? = null, @SerialName("user_id") From d5cfb7d36e41c0b77053db471f78857a5c345f86 Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Thu, 10 Oct 2019 23:23:05 +0600 Subject: [PATCH 2/5] update consts calling --- .../insanusmokrassar/SauceNaoAPI/Constants.kt | 4 ---- .../insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt | 14 ++++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) delete mode 100644 src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt deleted file mode 100644 index 9698774..0000000 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Constants.kt +++ /dev/null @@ -1,4 +0,0 @@ -package com.github.insanusmokrassar.SauceNaoAPI - -const val LONG_TIME_LIMIT_MILLIS: Int = 1 * 24 * 60 * 60 * 1000 -const val SHORT_TIME_LIMIT_MILLIS: Int = 30 * 1000 diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt index 0493761..95ea7d7 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt @@ -1,5 +1,7 @@ package com.github.insanusmokrassar.SauceNaoAPI +import com.github.insanusmokrassar.SauceNaoAPI.additional.LONG_TIME_RECALCULATING_MILLIS +import com.github.insanusmokrassar.SauceNaoAPI.additional.SHORT_TIME_RECALCULATING_MILLIS import com.github.insanusmokrassar.SauceNaoAPI.exceptions.sauceNaoAPIException import com.github.insanusmokrassar.SauceNaoAPI.models.SauceNaoAnswer import io.ktor.client.HttpClient @@ -44,11 +46,11 @@ data class SauceNaoAPI( val answer = makeRequest(requestBuilder) callback.resumeWith(Result.success(answer)) - val sleepUntil = if (answer.header.longRemaining == 0) { - getMostOldestInLongPeriod() ?.plusMillis(LONG_TIME_LIMIT_MILLIS) + val sleepUntil = if (answer.header.longRemaining < 1) { + getMostOldestInLongPeriod() ?.plusMillis(LONG_TIME_RECALCULATING_MILLIS) } else { - if (answer.header.shortRemaining == 0) { - getMostOldestInShortPeriod() ?.plusMillis(SHORT_TIME_LIMIT_MILLIS) + if (answer.header.shortRemaining < 1) { + getMostOldestInShortPeriod() ?.plusMillis(SHORT_TIME_RECALCULATING_MILLIS) } else { null } @@ -137,7 +139,7 @@ data class SauceNaoAPI( } private fun clearRequestTimes(relatedTo: DateTime = DateTime.now()) { - val limitValue = relatedTo.minusMillis(LONG_TIME_LIMIT_MILLIS) + val limitValue = relatedTo.minusMillis(LONG_TIME_RECALCULATING_MILLIS) requestsSendTimes.removeAll { it < limitValue @@ -153,7 +155,7 @@ data class SauceNaoAPI( private fun getMostOldestInShortPeriod(): DateTime? { val now = DateTime.now() - val limitTime = now.minusMillis(SHORT_TIME_LIMIT_MILLIS) + val limitTime = now.minusMillis(SHORT_TIME_RECALCULATING_MILLIS) clearRequestTimes(now) From e61e09449530ad4c5a2ec67be1e74a42595137d0 Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Thu, 10 Oct 2019 23:25:35 +0600 Subject: [PATCH 3/5] update libraries --- gradle.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gradle.properties b/gradle.properties index b4426c2..77fc6d3 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,9 +1,9 @@ kotlin.code.style=official -kotlin_version=1.3.41 -kotlin_coroutines_version=1.2.2 -kotlin_serialisation_runtime_version=0.11.1 -joda_time_version=2.10.1 -ktor_version=1.2.3 +kotlin_version=1.3.50 +kotlin_coroutines_version=1.3.2 +kotlin_serialisation_runtime_version=0.13.0 +joda_time_version=2.10.4 +ktor_version=1.2.5 project_public_name=SauceNao API project_public_description=SauceNao API library From ab67180d1986ce7c2213f482f525391a696a1bdc Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Fri, 11 Oct 2019 00:10:19 +0600 Subject: [PATCH 4/5] now scope in SauceNaoAPI can be passed via constructor --- .../insanusmokrassar/SauceNaoAPI/Launcher.kt | 20 +++++++------------ .../SauceNaoAPI/SauceNaoAPI.kt | 7 ++++--- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt index 10f3c99..0feab82 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/Launcher.kt @@ -3,20 +3,14 @@ package com.github.insanusmokrassar.SauceNaoAPI import kotlinx.coroutines.* fun main(vararg args: String) { - val key = args.first() - val api = SauceNaoAPI(key) - - val launch = GlobalScope.launch { - api.use { - it.request( - args[1] - ).also { - println(it) - } - } - } + val (key, requestUrl) = args runBlocking { - launch.join() + val api = SauceNaoAPI(key, scope = GlobalScope) + api.use { + println( + it.request(requestUrl) + ) + } } } diff --git a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt index 95ea7d7..eba233a 100644 --- a/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt +++ b/src/main/kotlin/com/github/insanusmokrassar/SauceNaoAPI/SauceNaoAPI.kt @@ -32,7 +32,8 @@ data class SauceNaoAPI( private val apiToken: String, private val outputType: OutputType = JsonOutputType, private val client: HttpClient = HttpClient(OkHttp), - private val searchUrl: String = SEARCH_URL + private val searchUrl: String = SEARCH_URL, + private val scope: CoroutineScope = CoroutineScope(Dispatchers.Default) ) : Closeable { private val logger = Logger.getLogger("SauceNaoAPI") @@ -40,7 +41,7 @@ data class SauceNaoAPI( private val requestsSendTimes = mutableListOf() init { - CoroutineScope(Dispatchers.Default).launch { + scope.launch { for ((callback, requestBuilder) in requestsChannel) { try { val answer = makeRequest(requestBuilder) @@ -191,7 +192,7 @@ data class SauceNaoAPI( override fun close() { requestsChannel.close() - requestsSendTimes.clear() client.close() + requestsSendTimes.clear() } } \ No newline at end of file From 1e5c9b09c6e0f09fd99261931b25bbf92e363d9c Mon Sep 17 00:00:00 2001 From: InsanusMokrassar Date: Fri, 11 Oct 2019 00:23:05 +0600 Subject: [PATCH 5/5] fill changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd7768e..c937ce0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## 0.4.0 +* Update libraries versions + * Kotlin `1.3.31` -> `1.3.50` + * Coroutines `1.2.1` -> `1.3.2` + * Serialization `0.11.0` -> `0.13.0` + * Joda Time `2.10.1` -> `2.10.4` + * Ktor `1.1.4` -> `1.2.5` +* Now `SauceNaoAPI` is `Closeable` +* Now `SauceNaoAPI` working with synchronous queue +* `SauceNaoAPI` now will wait for some time when one of limits will be achieved + ## 0.3.0 * Now `results` field of `SauceNaoAnswer` is optional and is empty list by default