InsanusMokrassar
/
SauceNaoAPI
mirror of https://github.com/InsanusMokrassar/SauceNaoAPI.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

improvements and actualization

This commit is contained in:
InsanusMokrassar 2022-05-14 13:43:28 +06:00
parent 8543917da7
commit 0b79f27b38
12 changed files with 198 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGELOG.md
View File

@ -1,5 +1,13 @@
# SauceNaoAPI Changelog
## 0.11.0
* Versions updates:
* `Kotlin`: `1.6.10` -> `1.6.21`
* `Serialization`: `1.3.2` -> `1.3.3`
* `Klock`: `2.6.3` -> `2.7.0`
* `Ktor`: `1.6.8` -> `2.0.1`
## 0.10.1
* Versions updates:

2
gradle/libs.versions.toml
View File

@ -1,7 +1,7 @@
[versions]
kt = "1.6.21"
kt-serialization = "1.3.2"
kt-serialization = "1.3.3"
kt-coroutines = "1.6.1"
klock = "2.7.0"

43
kotlin-js-store/yarn.lock
View File

@ -286,11 +286,6 @@ argparse@^2.0.1:
resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38"
integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==
async-limiter@~1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd"
integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==
balanced-match@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
@ -1289,10 +1284,12 @@ neo-async@^2.6.2:
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
node-fetch@2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
node-fetch@2.6.7:
version "2.6.7"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
dependencies:
whatwg-url "^5.0.0"
node-releases@^2.0.2:
version "2.0.2"
@ -1734,6 +1731,11 @@ toidentifier@1.0.1:
resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35"
integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==
tr46@~0.0.3:
version "0.0.3"
resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
type-is@~1.6.18:
version "1.6.18"
resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131"
@ -1797,6 +1799,11 @@ watchpack@^2.2.0:
glob-to-regexp "^0.4.1"
graceful-fs "^4.1.2"
webidl-conversions@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=
webpack-cli@4.9.0:
version "4.9.0"
resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-4.9.0.tgz#dc43e6e0f80dd52e89cbf73d5294bcd7ad6eb343"
@ -1866,6 +1873,14 @@ webpack@5.57.1:
watchpack "^2.2.0"
webpack-sources "^3.2.0"
whatwg-url@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0=
dependencies:
tr46 "~0.0.3"
webidl-conversions "^3.0.0"
which@2.0.2, which@^2.0.1:
version "2.0.2"
resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"
@ -1904,12 +1919,10 @@ wrappy@1:
resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
ws@6.2.1:
version "6.2.1"
resolved "https://registry.yarnpkg.com/ws/-/ws-6.2.1.tgz#442fdf0a47ed64f59b6a5d8ff130f4748ed524fb"
integrity sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==
dependencies:
async-limiter "~1.0.0"
ws@8.5.0:
version "8.5.0"
resolved "https://registry.yarnpkg.com/ws/-/ws-8.5.0.tgz#bfb4be96600757fe5382de12c670dab984a1ed4f"
integrity sha512-BWX0SWVgLPzYwF8lTzEy1egjhS4S4OEAHfsO8o65WOVsrnSRGaSiUaa9e0ggGlkMTtBlmOpEXiie9RUcBO86qg==
ws@~7.4.2:
version "7.4.6"

109
src/commonMain/kotlin/dev/inmo/saucenaoapi/SauceNaoAPI.kt
View File

@ -5,18 +5,16 @@ import dev.inmo.saucenaoapi.exceptions.sauceNaoAPIException
import dev.inmo.saucenaoapi.models.*
import dev.inmo.saucenaoapi.utils.*
import io.ktor.client.HttpClient
import io.ktor.client.features.ClientRequestException
import io.ktor.client.plugins.ClientRequestException
import io.ktor.client.request.*
import io.ktor.client.request.forms.MultiPartFormDataContent
import io.ktor.client.request.forms.formData
import io.ktor.client.statement.HttpResponse
import io.ktor.client.statement.readText
import io.ktor.client.statement.*
import io.ktor.http.*
import io.ktor.utils.io.core.Input
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.Channel
import kotlinx.serialization.json.Json
import kotlin.coroutines.*
private const val API_TOKEN_FIELD = "api_key"
private const val OUTPUT_TYPE_FIELD = "output_type"
@ -40,26 +38,30 @@ val defaultSauceNaoParser = Json {
data class SauceNaoAPI(
private val apiToken: String? = null,
private val outputType: dev.inmo.saucenaoapi.OutputType = dev.inmo.saucenaoapi.JsonOutputType,
private val client: HttpClient = HttpClient(),
private val searchUrl: String = SEARCH_URL,
private val scope: CoroutineScope = CoroutineScope(Dispatchers.Default),
private val parser: Json = defaultSauceNaoParser
) : SauceCloseable {
private val requestsChannel = Channel<Pair<Continuation<SauceNaoAnswer>, HttpRequestBuilder>>(Channel.UNLIMITED)
private val timeManager = TimeManager(scope)
private val quotaManager = RequestQuotaManager(scope)
private val requestsChannel = Channel<Pair<CompletableDeferred<SauceNaoAnswer>, HttpRequestBuilder>>(Channel.UNLIMITED)
private val subscope = CoroutineScope(scope.coroutineContext + SupervisorJob(scope.coroutineContext.job)).also {
it.coroutineContext.job.invokeOnCompletion {
requestsChannel.close(it)
}
}
private val timeManager = TimeManager(subscope)
private val quotaManager = RequestQuotaManager(subscope)
val limitsState: LimitsState
get() = quotaManager.limitsState
private val requestsJob = scope.launch {
private val requestsJob = subscope.launch {
for ((callback, requestBuilder) in requestsChannel) {
quotaManager.getQuota()
launch {
try {
val answer = makeRequest(requestBuilder)
callback.resume(answer)
callback.complete(answer)
quotaManager.updateQuota(answer.header, timeManager)
} catch (e: TooManyRequestsException) {
@ -67,7 +69,7 @@ data class SauceNaoAPI(
requestsChannel.send(callback to requestBuilder)
} catch (e: Exception) {
try {
callback.resumeWithException(e)
callback.completeExceptionally(e)
} catch (e: IllegalStateException) { // may happen when already resumed and api was closed
// do nothing
}
@ -80,7 +82,7 @@ data class SauceNaoAPI(
url: String,
resultsCount: Int? = null,
minSimilarity: Float? = null
): SauceNaoAnswer? = makeRequest(
): SauceNaoAnswer = makeRequest(
url.asSauceRequestSubject,
resultsCount = resultsCount,
minSimilarity = minSimilarity
@ -91,18 +93,29 @@ data class SauceNaoAPI(
mimeType: ContentType,
resultsCount: Int? = null,
minSimilarity: Float? = null
): SauceNaoAnswer? = makeRequest(
): SauceNaoAnswer = makeRequest(
mediaInput.asSauceRequestSubject(mimeType),
resultsCount = resultsCount,
minSimilarity = minSimilarity
)
suspend fun request(
file: MPPFile,
resultsCount: Int? = null,
minSimilarity: Float? = null
): SauceNaoAnswer = request(
file.input,
file.contentType,
resultsCount = resultsCount,
minSimilarity = minSimilarity
)
suspend fun requestByDb(
url: String,
db: Int,
resultsCount: Int? = null,
minSimilarity: Float? = null
): SauceNaoAnswer? = makeRequest(
): SauceNaoAnswer = makeRequest(
url.asSauceRequestSubject,
db = db,
resultsCount = resultsCount,
@ -114,7 +127,7 @@ data class SauceNaoAPI(
dbmask: Int,
resultsCount: Int? = null,
minSimilarity: Float? = null
): SauceNaoAnswer? = makeRequest(
): SauceNaoAnswer = makeRequest(
url.asSauceRequestSubject,
dbmask = dbmask,
resultsCount = resultsCount,
@ -126,7 +139,7 @@ data class SauceNaoAPI(
dbmaski: Int,
resultsCount: Int? = null,
minSimilarity: Float? = null
): SauceNaoAnswer? = makeRequest(
): SauceNaoAnswer = makeRequest(
url.asSauceRequestSubject,
dbmaski = dbmaski,
resultsCount = resultsCount,
@ -137,8 +150,8 @@ data class SauceNaoAPI(
builder: HttpRequestBuilder
): SauceNaoAnswer {
return try {
val call = client.request<HttpResponse>(builder)
val answerText = call.readText()
val call = client.request(builder)
val answerText = call.bodyAsText()
timeManager.addTimeAndClear()
parser.decodeFromString(
SauceNaoAnswerSerializer,
@ -156,29 +169,32 @@ data class SauceNaoAPI(
dbmaski: Int? = null,
resultsCount: Int? = null,
minSimilarity: Float? = null
): SauceNaoAnswer? {
return suspendCoroutine<SauceNaoAnswer> {
requestsChannel.trySend(
it to HttpRequestBuilder().apply {
url(searchUrl)
): SauceNaoAnswer {
val deferred = CompletableDeferred<SauceNaoAnswer>()
apiToken ?.also { parameter(API_TOKEN_FIELD, it) }
parameter(OUTPUT_TYPE_FIELD, outputType.typeCode)
db ?.also { parameter(DB_FIELD, it) }
dbmask ?.also { parameter(DBMASK_FIELD, it) }
dbmaski ?.also { parameter(DBMASKI_FIELD, it) }
resultsCount ?.also { parameter(RESULTS_COUNT_FIELD, it) }
minSimilarity ?.also { parameter(MINIMAL_SIMILARITY_FIELD, it) }
requestsChannel.trySend(
deferred to HttpRequestBuilder().apply {
url(searchUrl)
when (request) {
is UrlSauceRequestSubject -> {
parameter(URL_FIELD, request.url)
}
is InputRequestSubject -> {
val mimeType = request.mimeType
apiToken ?.also { parameter(API_TOKEN_FIELD, it) }
parameter(OUTPUT_TYPE_FIELD, JsonOutputType.typeCode)
db ?.also { parameter(DB_FIELD, it) }
dbmask ?.also { parameter(DBMASK_FIELD, it) }
dbmaski ?.also { parameter(DBMASKI_FIELD, it) }
resultsCount ?.also { parameter(RESULTS_COUNT_FIELD, it) }
minSimilarity ?.also { parameter(MINIMAL_SIMILARITY_FIELD, it) }
method = HttpMethod.Post
body = MultiPartFormDataContent(formData {
when (request) {
is UrlSauceRequestSubject -> {
parameter(URL_FIELD, request.url)
}
is InputRequestSubject -> {
val mimeType = request.mimeType
method = HttpMethod.Post
setBody(
MultiPartFormDataContent(
formData {
appendInput(
FILE_FIELD,
Headers.build {
@ -197,19 +213,18 @@ data class SauceNaoAPI(
},
block = request::input
)
})
}
}
)
)
}
}
)
}
}
)
return deferred.await()
}
override fun close() {
requestsChannel.close()
client.close()
requestsJob.cancel()
timeManager.close()
quotaManager.close()
subscope.cancel()
}
}

6
src/commonMain/kotlin/dev/inmo/saucenaoapi/exceptions/TooManyRequestsException.kt
View File

@ -3,15 +3,15 @@ package dev.inmo.saucenaoapi.exceptions
import dev.inmo.saucenaoapi.additional.LONG_TIME_RECALCULATING_MILLIS
import dev.inmo.saucenaoapi.additional.SHORT_TIME_RECALCULATING_MILLIS
import com.soywiz.klock.TimeSpan
import io.ktor.client.features.ClientRequestException
import io.ktor.client.statement.readText
import io.ktor.client.plugins.ClientRequestException
import io.ktor.client.statement.bodyAsText
import io.ktor.http.HttpStatusCode.Companion.TooManyRequests
import io.ktor.utils.io.errors.IOException
internal suspend fun ClientRequestException.sauceNaoAPIException(): Exception {
return when (response.status) {
TooManyRequests -> {
val answerContent = response.readText()
val answerContent = response.bodyAsText()
when {
answerContent.contains("daily limit") -> TooManyRequestsLongException(answerContent)
else -> TooManyRequestsShortException(answerContent)

9
src/commonMain/kotlin/dev/inmo/saucenaoapi/utils/MPPFile.kt Normal file
View File

@ -0,0 +1,9 @@
package dev.inmo.saucenaoapi.utils
import io.ktor.http.ContentType
import io.ktor.utils.io.core.Input
expect class MPPFile
expect val MPPFile.input: Input
expect val MPPFile.contentType: ContentType

30
src/commonMain/kotlin/dev/inmo/saucenaoapi/utils/RequestQuotaManager.kt
View File

@ -9,13 +9,12 @@ import dev.inmo.saucenaoapi.models.LimitsState
import com.soywiz.klock.DateTime
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.Channel
import kotlin.coroutines.suspendCoroutine
import kotlin.math.max
import kotlin.math.min
internal class RequestQuotaManager (
scope: CoroutineScope
) : SauceCloseable {
) {
private var longQuota = 1
private var shortQuota = 1
private var longMaxQuota = 1
@ -35,6 +34,10 @@ internal class RequestQuotaManager (
for (callback in quotaActions) {
callback()
}
}.also {
it.invokeOnCompletion {
quotaActions.close(it)
}
}
private suspend fun updateQuota(
@ -83,21 +86,16 @@ internal class RequestQuotaManager (
)
suspend fun getQuota() {
return suspendCoroutine {
lateinit var callback: suspend () -> Unit
callback = suspend {
if (longQuota > 0 && shortQuota > 0) {
it.resumeWith(Result.success(Unit))
} else {
quotaActions.send(callback)
}
val job = Job()
lateinit var callback: suspend () -> Unit
callback = suspend {
if (longQuota > 0 && shortQuota > 0) {
job.complete()
} else {
quotaActions.send(callback)
}
quotaActions.trySend(callback)
}
}
override fun close() {
quotaJob.cancel()
quotaActions.close()
quotaActions.trySend(callback)
return job.join()
}
}

3
src/commonMain/kotlin/dev/inmo/saucenaoapi/utils/SauceCloseable.kt
View File

@ -6,12 +6,13 @@ interface SauceCloseable {
fun close()
}
fun <T> SauceCloseable.use(block: (SauceCloseable) -> T): T = try {
inline fun <T> SauceCloseable.use(block: (SauceCloseable) -> T): T = try {
block(this)
} finally {
close()
}
@Deprecated("Useless")
suspend fun <T> SauceCloseable.useSafe(block: suspend (SauceCloseable) -> T): T = try {
supervisorScope {
block(this@useSafe)

48
src/commonMain/kotlin/dev/inmo/saucenaoapi/utils/TimeManager.kt
View File

@ -3,11 +3,8 @@ package dev.inmo.saucenaoapi.utils
import dev.inmo.saucenaoapi.additional.LONG_TIME_RECALCULATING_MILLIS
import dev.inmo.saucenaoapi.additional.SHORT_TIME_RECALCULATING_MILLIS
import com.soywiz.klock.DateTime
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.launch
import kotlin.coroutines.Continuation
import kotlin.coroutines.suspendCoroutine
private fun MutableList<DateTime>.clearTooOldTimes(relatedTo: DateTime = DateTime.now()) {
val limitValue = relatedTo - LONG_TIME_RECALCULATING_MILLIS
@ -38,16 +35,16 @@ private data class TimeManagerTimeAdder(
}
private data class TimeManagerMostOldestInLongGetter(
private val continuation: Continuation<DateTime?>
private val deferred: CompletableDeferred<DateTime?>
) : TimeManagerAction {
override suspend fun makeChangeWith(times: MutableList<DateTime>) {
times.clearTooOldTimes()
continuation.resumeWith(Result.success(times.minOrNull()))
deferred.complete(times.minOrNull())
}
}
private data class TimeManagerMostOldestInShortGetter(
private val continuation: Continuation<DateTime?>
private val deferred: CompletableDeferred<DateTime?>
) : TimeManagerAction {
override suspend fun makeChangeWith(times: MutableList<DateTime>) {
times.clearTooOldTimes()
@ -56,19 +53,17 @@ private data class TimeManagerMostOldestInShortGetter(
val limitTime = now - SHORT_TIME_RECALCULATING_MILLIS
continuation.resumeWith(
Result.success(
times.asSequence().filter {
limitTime < it
}.minOrNull()
)
deferred.complete(
times.asSequence().filter {
limitTime < it
}.minOrNull()
)
}
}
internal class TimeManager(
scope: CoroutineScope
) : SauceCloseable {
) {
private val actionsChannel = Channel<TimeManagerAction>(Channel.UNLIMITED)
private val timeUpdateJob = scope.launch {
@ -76,6 +71,10 @@ internal class TimeManager(
for (action in actionsChannel) {
action(times)
}
}.also {
it.invokeOnCompletion {
actionsChannel.close(it)
}
}
suspend fun addTimeAndClear() {
@ -83,21 +82,20 @@ internal class TimeManager(
}
suspend fun getMostOldestInLongPeriod(): DateTime? {
return suspendCoroutine {
actionsChannel.trySend(
TimeManagerMostOldestInLongGetter(it)
)
val deferred = CompletableDeferred<DateTime?>()
return if (actionsChannel.trySend(TimeManagerMostOldestInLongGetter(deferred)).isSuccess) {
deferred.await()
} else {
null
}
}
suspend fun getMostOldestInShortPeriod(): DateTime? {
return suspendCoroutine {
actionsChannel.trySend(TimeManagerMostOldestInShortGetter(it))
val deferred = CompletableDeferred<DateTime?>()
return if (actionsChannel.trySend(TimeManagerMostOldestInShortGetter(deferred)).isSuccess) {
deferred.await()
} else {
null
}
}
override fun close() {
actionsChannel.close()
timeUpdateJob.cancel()
}
}

18
src/jsMain/kotlin/dev/inmo/saucenaoapi/utils/ActualMPPFile.kt Normal file
View File

@ -0,0 +1,18 @@
package dev.inmo.saucenaoapi.utils
import io.ktor.http.ContentType
import io.ktor.utils.io.core.ByteReadPacket
import io.ktor.utils.io.core.Input
import org.khronos.webgl.Int8Array
import org.w3c.files.File
import org.w3c.files.FileReaderSync
actual typealias MPPFile = File
actual val MPPFile.input: Input
get() {
val reader = FileReaderSync()
return ByteReadPacket(Int8Array(reader.readAsArrayBuffer(this)) as ByteArray)
}
actual val MPPFile.contentType: ContentType
get() = ContentType.parse(type)

14
src/jvmMain/kotlin/dev/inmo/saucenaoapi/utils/ActualMPPFile.kt Normal file
View File

@ -0,0 +1,14 @@
package dev.inmo.saucenaoapi.utils
import io.ktor.http.ContentType
import io.ktor.utils.io.core.Input
import io.ktor.utils.io.streams.asInput
import java.io.File
import java.nio.file.Files
actual typealias MPPFile = File
actual val MPPFile.input: Input
get() = inputStream().asInput()
actual val MPPFile.contentType: ContentType
get() = ContentType.parse(Files.probeContentType(toPath()))

35
src/jvmTest/kotlin/Launcher.kt
View File

@ -1,30 +1,27 @@
import dev.inmo.saucenaoapi.SauceNaoAPI
import dev.inmo.saucenaoapi.utils.useSafe
import io.ktor.http.ContentType
import io.ktor.utils.io.streams.asInput
import io.ktor.client.HttpClient
import kotlinx.coroutines.*
import java.io.File
import java.nio.file.Files
suspend fun main(vararg args: String) {
val (key, requestSubject) = args
val scope = CoroutineScope(Dispatchers.Default)
val api = SauceNaoAPI(key, scope = scope)
api.useSafe { _ ->
println(
when {
requestSubject.startsWith("/") -> File(requestSubject).let {
api.request(
it.inputStream().asInput(),
ContentType.parse(Files.probeContentType(it.toPath()))
)
}
else -> api.request(requestSubject)
}
)
val client = HttpClient()
val scope = CoroutineScope(Dispatchers.IO).also {
it.coroutineContext.job.invokeOnCompletion {
client.close()
}
}
val api = SauceNaoAPI(key, client, scope = scope)
println(
when {
requestSubject.startsWith("/") -> File(requestSubject).let {
api.request(it)
}
else -> api.request(requestSubject)
}
)
scope.cancel()
}