Merge pull request #4 from InsanusMokrassar/0.4.1

Managers
This commit is contained in:
InsanusMokrassar 2019-10-12 13:01:18 +06:00 committed by GitHub
commit f9a3176fec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 289 additions and 69 deletions

View File

@ -12,6 +12,15 @@
* Now `SauceNaoAPI` working with synchronous queue
* `SauceNaoAPI` now will wait for some time when one of limits will be achieved
### 0.4.1 Managers experiments and row format in answer
* Add `TimeManager` - it will manage work with requests times
* Add `RequestQuotaMagager` - it will manage quota for requests and call suspend
if they will be over
* `SauceNaoAPI` now working (almost) asynchronously
* Now `SauceNaoAnswer` have field `row` which contains `JsonObject` with
all original answer fields
## 0.3.0
* Now `results` field of `SauceNaoAnswer` is optional and is empty list by default

View File

@ -1 +1,25 @@
# SauceNaoAPI
It is wrapper for [SauceNAO](https://saucenao.com/) API. For now, library is
in preview state. It can be fully used, but some of info can be unavailable from
wrapper classes, but now you can access them via `SauceNaoAnswer#row` field.
## Requester
For the requests we are using `SauceNaoAPI` object. Unfortunately, for now it
supports only url strings as source of request. For example:
```kotlin
val key = // here must be your Sauce NAO API key
val requestUrl = // here must be your link to some image
val api = SauceNaoAPI(key)
api.use {
println(
it.request(requestUrl)
)
}
```
Most of others requests use the same etymology and meaning as in the
`SauceNAO` API docs.

View File

@ -1,4 +1,4 @@
project.version = "0.4.0"
project.version = "0.4.1"
project.group = "com.github.insanusmokrassar"
buildscript {
@ -15,7 +15,6 @@ buildscript {
}
}
apply plugin: 'java-library'
apply plugin: 'kotlin'
apply plugin: 'kotlinx-serialization'

View File

@ -2,10 +2,9 @@ package com.github.insanusmokrassar.SauceNaoAPI
import kotlinx.coroutines.*
fun main(vararg args: String) {
suspend fun main(vararg args: String) {
val (key, requestUrl) = args
runBlocking {
val api = SauceNaoAPI(key, scope = GlobalScope)
api.use {
println(
@ -13,4 +12,3 @@ fun main(vararg args: String) {
)
}
}
}

View File

@ -4,6 +4,8 @@ import com.github.insanusmokrassar.SauceNaoAPI.additional.LONG_TIME_RECALCULATIN
import com.github.insanusmokrassar.SauceNaoAPI.additional.SHORT_TIME_RECALCULATING_MILLIS
import com.github.insanusmokrassar.SauceNaoAPI.exceptions.sauceNaoAPIException
import com.github.insanusmokrassar.SauceNaoAPI.models.SauceNaoAnswer
import com.github.insanusmokrassar.SauceNaoAPI.utils.*
import com.github.insanusmokrassar.SauceNaoAPI.utils.calculateSleepTime
import io.ktor.client.HttpClient
import io.ktor.client.engine.okhttp.OkHttp
import io.ktor.client.features.ClientRequestException
@ -38,31 +40,24 @@ data class SauceNaoAPI(
private val logger = Logger.getLogger("SauceNaoAPI")
private val requestsChannel = Channel<Pair<Continuation<SauceNaoAnswer>, HttpRequestBuilder>>(Channel.UNLIMITED)
private val requestsSendTimes = mutableListOf<DateTime>()
private val timeManager = TimeManager(scope)
private val quotaManager = RequestQuotaManager(scope)
init {
scope.launch {
private val requestsJob = scope.launch {
for ((callback, requestBuilder) in requestsChannel) {
quotaManager.getQuota()
launch {
try {
val answer = makeRequest(requestBuilder)
callback.resumeWith(Result.success(answer))
val sleepUntil = if (answer.header.longRemaining < 1) {
getMostOldestInLongPeriod() ?.plusMillis(LONG_TIME_RECALCULATING_MILLIS)
} else {
if (answer.header.shortRemaining < 1) {
getMostOldestInShortPeriod() ?.plusMillis(SHORT_TIME_RECALCULATING_MILLIS)
} else {
null
}
}
sleepUntil ?.also { _ ->
logger.warning("LONG LIMIT REACHED, SLEEP UNTIL $sleepUntil")
delay(sleepUntil.millis - DateTime.now().millis)
}
quotaManager.updateQuota(answer.header, timeManager)
} catch (e: Exception) {
try {
callback.resumeWith(Result.failure(e))
} catch (e: IllegalStateException) { // may happen when already resumed and api was closed
// do nothing
}
}
}
}
@ -121,7 +116,7 @@ data class SauceNaoAPI(
val call = client.execute(builder)
val answerText = call.response.readText()
logger.info(answerText)
addRequestTimesAndClear()
timeManager.addTimeAndClear()
Json.nonstrict.parse(
SauceNaoAnswer.serializer(),
answerText
@ -131,40 +126,6 @@ data class SauceNaoAPI(
}
}
private fun addRequestTimesAndClear() {
val newDateTime = DateTime.now()
clearRequestTimes(newDateTime)
requestsSendTimes.add(newDateTime)
}
private fun clearRequestTimes(relatedTo: DateTime = DateTime.now()) {
val limitValue = relatedTo.minusMillis(LONG_TIME_RECALCULATING_MILLIS)
requestsSendTimes.removeAll {
it < limitValue
}
}
private fun getMostOldestInLongPeriod(): DateTime? {
clearRequestTimes()
return requestsSendTimes.min()
}
private fun getMostOldestInShortPeriod(): DateTime? {
val now = DateTime.now()
val limitTime = now.minusMillis(SHORT_TIME_RECALCULATING_MILLIS)
clearRequestTimes(now)
return requestsSendTimes.asSequence().filter {
limitTime < it
}.min()
}
private suspend fun makeRequest(
url: String,
db: Int? = null,
@ -193,6 +154,8 @@ data class SauceNaoAPI(
override fun close() {
requestsChannel.close()
client.close()
requestsSendTimes.clear()
requestsJob.cancel()
timeManager.close()
quotaManager.close()
}
}

View File

@ -1,9 +1,40 @@
package com.github.insanusmokrassar.SauceNaoAPI.models
import kotlinx.serialization.Serializable
import kotlinx.serialization.*
import kotlinx.serialization.internal.ArrayListSerializer
import kotlinx.serialization.json.*
@Serializable
@Serializable(SauceNaoAnswerSerializer::class)
data class SauceNaoAnswer(
val header: Header,
val results: List<Result> = emptyList()
val results: List<Result> = emptyList(),
val raw: JsonObject
)
@Serializer(SauceNaoAnswer::class)
object SauceNaoAnswerSerializer : KSerializer<SauceNaoAnswer> {
private val resultsSerializer = ArrayListSerializer(Result.serializer())
private const val headerField = "header"
private const val resultsField = "results"
private val serializer = Json.nonstrict
override fun deserialize(decoder: Decoder): SauceNaoAnswer {
val raw = JsonObjectSerializer.deserialize(decoder)
val header = serializer.fromJson(Header.serializer(), raw.getObject(headerField))
val results = serializer.fromJson(resultsSerializer, raw.getArray(resultsField))
return SauceNaoAnswer(header, results, raw)
}
override fun serialize(encoder: Encoder, obj: SauceNaoAnswer) {
val resultObject = JsonObject(
obj.raw.content.let {
it + mapOf(
headerField to serializer.toJson(Header.serializer(), obj.header),
resultsField to serializer.toJson(resultsSerializer, obj.results)
)
}
)
JsonObject.serializer().serialize(encoder, resultObject)
}
}

View File

@ -0,0 +1,74 @@
package com.github.insanusmokrassar.SauceNaoAPI.utils
import com.github.insanusmokrassar.SauceNaoAPI.additional.LONG_TIME_RECALCULATING_MILLIS
import com.github.insanusmokrassar.SauceNaoAPI.additional.SHORT_TIME_RECALCULATING_MILLIS
import com.github.insanusmokrassar.SauceNaoAPI.models.Header
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.Channel
import kotlinx.io.core.Closeable
import org.joda.time.DateTime
import kotlin.coroutines.suspendCoroutine
import kotlin.math.min
class RequestQuotaManager (
private val scope: CoroutineScope
) : Closeable {
private var longQuota = 1
private var shortQuota = 1
private var longMaxQuota = 1
private var shortMaxQuota = 1
private val quotaActions = Channel<suspend () -> Unit>(Channel.UNLIMITED)
private val quotaJob = scope.launch {
for (callback in quotaActions) {
callback()
}
}
suspend fun updateQuota(header: Header, timeManager: TimeManager) {
quotaActions.send(
suspend {
longMaxQuota = header.longLimit
shortMaxQuota = header.shortLimit
longQuota = min(header.longLimit, header.longRemaining)
shortQuota = min(header.shortLimit, header.shortRemaining)
when {
shortQuota < 1 -> timeManager.getMostOldestInShortPeriod() ?.millis ?.plus(SHORT_TIME_RECALCULATING_MILLIS) ?: let {
shortQuota = 1
null
}
longQuota < 1 -> timeManager.getMostOldestInLongPeriod() ?.millis ?.plus(LONG_TIME_RECALCULATING_MILLIS) ?: let {
longQuota = 1
null
}
else -> null
} ?.let {
delay(it - DateTime.now().millis)
}
Unit
}
)
}
suspend fun getQuota() {
return suspendCoroutine {
lateinit var callback: suspend () -> Unit
callback = suspend {
if (longQuota > 0 && shortQuota > 0) {
it.resumeWith(Result.success(Unit))
} else {
quotaActions.send(callback)
}
}
quotaActions.offer(callback)
}
}
override fun close() {
quotaJob.cancel()
quotaActions.close()
}
}

View File

@ -0,0 +1,18 @@
package com.github.insanusmokrassar.SauceNaoAPI.utils
import com.github.insanusmokrassar.SauceNaoAPI.additional.LONG_TIME_RECALCULATING_MILLIS
import com.github.insanusmokrassar.SauceNaoAPI.additional.SHORT_TIME_RECALCULATING_MILLIS
import com.github.insanusmokrassar.SauceNaoAPI.models.Header
import org.joda.time.DateTime
internal suspend fun calculateSleepTime(
header: Header,
mostOldestInShortPeriodGetter: suspend () -> DateTime?,
mostOldestInLongPeriodGetter: suspend () -> DateTime?
): DateTime? {
return when {
header.longRemaining < 1 -> mostOldestInLongPeriodGetter() ?.plusMillis(LONG_TIME_RECALCULATING_MILLIS)
header.shortRemaining < 1 -> mostOldestInShortPeriodGetter() ?.plusMillis(SHORT_TIME_RECALCULATING_MILLIS)
else -> null
}
}

View File

@ -0,0 +1,104 @@
package com.github.insanusmokrassar.SauceNaoAPI.utils
import com.github.insanusmokrassar.SauceNaoAPI.additional.LONG_TIME_RECALCULATING_MILLIS
import com.github.insanusmokrassar.SauceNaoAPI.additional.SHORT_TIME_RECALCULATING_MILLIS
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.launch
import kotlinx.io.core.Closeable
import org.joda.time.DateTime
import kotlin.coroutines.Continuation
import kotlin.coroutines.suspendCoroutine
private fun MutableList<DateTime>.clearTooOldTimes(relatedTo: DateTime = DateTime.now()) {
val limitValue = relatedTo.minusMillis(LONG_TIME_RECALCULATING_MILLIS)
removeAll {
it < limitValue
}
}
private interface TimeManagerAction {
suspend fun makeChangeWith(times: MutableList<DateTime>)
suspend operator fun invoke(times: MutableList<DateTime>) = makeChangeWith(times)
}
private data class TimeManagerClean(private val relatedTo: DateTime = DateTime.now()) : TimeManagerAction {
override suspend fun makeChangeWith(times: MutableList<DateTime>) {
times.clearTooOldTimes(relatedTo)
}
}
private data class TimeManagerTimeAdder(
private val time: DateTime = DateTime.now()
) : TimeManagerAction {
override suspend fun makeChangeWith(times: MutableList<DateTime>) {
times.add(time)
times.clearTooOldTimes()
}
}
private data class TimeManagerMostOldestInLongGetter(
private val continuation: Continuation<DateTime?>
) : TimeManagerAction {
override suspend fun makeChangeWith(times: MutableList<DateTime>) {
times.clearTooOldTimes()
continuation.resumeWith(Result.success(times.min()))
}
}
private data class TimeManagerMostOldestInShortGetter(
private val continuation: Continuation<DateTime?>
) : TimeManagerAction {
override suspend fun makeChangeWith(times: MutableList<DateTime>) {
times.clearTooOldTimes()
val now = DateTime.now()
val limitTime = now.minusMillis(SHORT_TIME_RECALCULATING_MILLIS)
continuation.resumeWith(
Result.success(
times.asSequence().filter {
limitTime < it
}.min()
)
)
}
}
class TimeManager(
scope: CoroutineScope
) : Closeable {
private val actionsChannel = Channel<TimeManagerAction>(Channel.UNLIMITED)
private val timeUpdateJob = scope.launch {
val times = mutableListOf<DateTime>()
for (action in actionsChannel) {
action(times)
}
}
suspend fun addTimeAndClear() {
actionsChannel.send(TimeManagerTimeAdder())
}
suspend fun getMostOldestInLongPeriod(): DateTime? {
return suspendCoroutine {
actionsChannel.offer(
TimeManagerMostOldestInLongGetter(it)
)
}
}
suspend fun getMostOldestInShortPeriod(): DateTime? {
return suspendCoroutine {
actionsChannel.offer(TimeManagerMostOldestInShortGetter(it))
}
}
override fun close() {
actionsChannel.close()
timeUpdateJob.cancel()
}
}