nitter/src/tokens.nim

167 lines
4.5 KiB
Nim
Raw Normal View History

2021-12-27 01:37:38 +00:00
# SPDX-License-Identifier: AGPL-3.0-only
2022-01-05 21:48:45 +00:00
import asyncdispatch, httpclient, times, sequtils, json, random
import strutils, tables
import zippy
2022-01-17 03:13:27 +00:00
import types, consts, http_pool
2020-06-01 00:16:24 +00:00
const
maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires
maxAge = 2.hours + 55.minutes # tokens expire after 3 hours
failDelay = initDuration(minutes=30)
2020-07-09 07:18:14 +00:00
var
2022-01-02 10:21:03 +00:00
clientPool: HttpPool
tokenPool: seq[Token]
2020-07-09 07:18:14 +00:00
lastFailed: Time
2022-06-05 19:47:25 +00:00
enableLogging = false
template log(str) =
if enableLogging: echo "[tokens] ", str
2022-01-05 23:42:18 +00:00
proc getPoolJson*(): JsonNode =
var
list = newJObject()
totalReqs = 0
totalPending = 0
reqsPerApi: Table[string, int]
for token in tokenPool:
2022-01-05 23:42:18 +00:00
totalPending.inc(token.pending)
list[token.tok] = %*{
"apis": newJObject(),
"pending": token.pending,
"init": $token.init,
"lastUse": $token.lastUse
}
for api in token.apis.keys:
list[token.tok]["apis"][$api] = %token.apis[api]
2022-01-05 23:42:18 +00:00
let
maxReqs =
case api
of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId, Api.userScreenName: 500
2022-01-05 23:42:18 +00:00
of Api.timeline: 187
else: 180
reqs = maxReqs - token.apis[api].remaining
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
totalReqs.inc(reqs)
return %*{
"amount": tokenPool.len,
"requests": totalReqs,
"pending": totalPending,
"apis": reqsPerApi,
"tokens": list
}
proc rateLimitError*(): ref RateLimitError =
2022-01-05 21:48:45 +00:00
newException(RateLimitError, "rate limited")
2020-06-01 00:16:24 +00:00
proc fetchToken(): Future[Token] {.async.} =
if getTime() - lastFailed < failDelay:
raise rateLimitError()
2020-07-09 07:18:14 +00:00
let headers = newHttpHeaders({
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"accept-encoding": "gzip",
"accept-language": "en-US,en;q=0.5",
"connection": "keep-alive",
"authorization": auth
})
2020-06-01 00:16:24 +00:00
2020-06-02 18:37:55 +00:00
try:
2022-01-05 21:48:45 +00:00
let
resp = clientPool.use(headers): await c.postContent(activate)
tokNode = parseJson(uncompress(resp))["guest_token"]
tok = tokNode.getStr($(tokNode.getInt))
time = getTime()
2020-06-01 00:16:24 +00:00
2022-01-05 21:48:45 +00:00
return Token(tok: tok, init: time, lastUse: time)
2020-06-24 13:02:34 +00:00
except Exception as e:
2022-06-05 19:47:25 +00:00
echo "[tokens] fetching token failed: ", e.msg
if "Try again" notin e.msg:
echo "[tokens] fetching tokens paused, resuming in 30 minutes"
lastFailed = getTime()
2020-06-01 00:16:24 +00:00
2022-01-05 21:48:45 +00:00
proc expired(token: Token): bool =
let time = getTime()
2022-01-05 21:48:45 +00:00
token.init < time - maxAge or token.lastUse < time - maxLastUse
2020-06-01 00:16:24 +00:00
2022-01-05 21:48:45 +00:00
proc isLimited(token: Token; api: Api): bool =
if token.isNil or token.expired:
return true
if api in token.apis:
let limit = token.apis[api]
return (limit.remaining <= 10 and limit.reset > epochTime().int)
2022-01-05 21:48:45 +00:00
else:
return false
2020-06-01 00:16:24 +00:00
proc isReady(token: Token; api: Api): bool =
not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api))
proc release*(token: Token; used=false; invalid=false) =
if token.isNil: return
if invalid or token.expired:
2022-06-05 19:47:25 +00:00
if invalid: log "discarding invalid token"
elif token.expired: log "discarding expired token"
let idx = tokenPool.find(token)
if idx > -1: tokenPool.delete(idx)
elif used:
dec token.pending
token.lastUse = getTime()
2020-06-01 00:16:24 +00:00
2022-01-05 21:48:45 +00:00
proc getToken*(api: Api): Future[Token] {.async.} =
2020-06-01 00:16:24 +00:00
for i in 0 ..< tokenPool.len:
if result.isReady(api): break
release(result)
result = tokenPool.sample()
2020-06-01 00:16:24 +00:00
if not result.isReady(api):
release(result)
2020-06-01 00:16:24 +00:00
result = await fetchToken()
2022-06-05 19:47:25 +00:00
log "added new token to pool"
tokenPool.add result
if not result.isNil:
inc result.pending
else:
raise rateLimitError()
2022-01-05 21:48:45 +00:00
proc setRateLimit*(token: Token; api: Api; remaining, reset: int) =
# avoid undefined behavior in race conditions
if api in token.apis:
let limit = token.apis[api]
if limit.reset >= reset and limit.remaining < remaining:
return
token.apis[api] = RateLimit(remaining: remaining, reset: reset)
2022-01-05 21:48:45 +00:00
2020-06-01 00:16:24 +00:00
proc poolTokens*(amount: int) {.async.} =
var futs: seq[Future[Token]]
for i in 0 ..< amount:
futs.add fetchToken()
for token in futs:
var newToken: Token
try: newToken = await token
except: discard
2022-01-05 21:48:45 +00:00
if not newToken.isNil:
2022-06-05 19:47:25 +00:00
log "added new token to pool"
tokenPool.add newToken
2020-06-01 00:16:24 +00:00
proc initTokenPool*(cfg: Config) {.async.} =
clientPool = HttpPool()
2022-06-05 19:47:25 +00:00
enableLogging = cfg.enableDebug
2020-06-01 00:16:24 +00:00
while true:
2022-01-05 21:48:45 +00:00
if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens:
await poolTokens(min(4, cfg.minTokens - tokenPool.len))
await sleepAsync(2000)