Improve rate limit handling, minor refactor
This commit is contained in:
parent
986b91ac73
commit
898b19b92f
|
@ -61,13 +61,6 @@ proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
|
|||
"DNT": "1"
|
||||
})
|
||||
|
||||
template updateAccount() =
|
||||
if resp.headers.hasKey(rlRemaining):
|
||||
let
|
||||
remaining = parseInt(resp.headers[rlRemaining])
|
||||
reset = parseInt(resp.headers[rlReset])
|
||||
account.setRateLimit(api, remaining, reset)
|
||||
|
||||
template fetchImpl(result, fetchBody) {.dirty.} =
|
||||
once:
|
||||
pool = HttpPool()
|
||||
|
@ -89,28 +82,46 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
badClient = true
|
||||
raise newException(BadClientError, "Bad client")
|
||||
|
||||
if resp.headers.hasKey(rlRemaining):
|
||||
let
|
||||
remaining = parseInt(resp.headers[rlRemaining])
|
||||
reset = parseInt(resp.headers[rlReset])
|
||||
account.setRateLimit(api, remaining, reset)
|
||||
|
||||
if result.len > 0:
|
||||
if resp.headers.getOrDefault("content-encoding") == "gzip":
|
||||
result = uncompress(result, dfGzip)
|
||||
else:
|
||||
echo "non-gzip body, url: ", url, ", body: ", result
|
||||
|
||||
if result.startsWith("{\"errors"):
|
||||
let errors = result.fromJson(Errors)
|
||||
if errors in {expiredToken, badToken}:
|
||||
echo "fetch error: ", errors
|
||||
invalidate(account)
|
||||
raise rateLimitError()
|
||||
elif errors in {rateLimited}:
|
||||
# rate limit hit, resets after 24 hours
|
||||
setLimited(account, api)
|
||||
raise rateLimitError()
|
||||
elif result.startsWith("429 Too Many Requests"):
|
||||
account.apis[api].remaining = 0
|
||||
# rate limit hit, resets after the 15 minute window
|
||||
raise rateLimitError()
|
||||
|
||||
fetchBody
|
||||
|
||||
release(account, used=true)
|
||||
|
||||
if resp.status == $Http400:
|
||||
raise newException(InternalError, $url)
|
||||
except InternalError as e:
|
||||
raise e
|
||||
except BadClientError as e:
|
||||
release(account, used=true)
|
||||
raise e
|
||||
except OSError as e:
|
||||
raise e
|
||||
except Exception as e:
|
||||
echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", account.id, ", url: ", url
|
||||
if "length" notin e.msg and "descriptor" notin e.msg:
|
||||
release(account, invalid=true)
|
||||
raise rateLimitError()
|
||||
finally:
|
||||
release(account)
|
||||
|
||||
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
|
||||
var body: string
|
||||
|
@ -121,36 +132,14 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
|
|||
echo resp.status, ": ", body, " --- url: ", url
|
||||
result = newJNull()
|
||||
|
||||
updateAccount()
|
||||
|
||||
let error = result.getError
|
||||
if error in {invalidToken, badToken}:
|
||||
echo "fetch error: ", result.getError
|
||||
release(account, invalid=true)
|
||||
if error in {expiredToken, badToken}:
|
||||
echo "fetchBody error: ", error
|
||||
invalidate(account)
|
||||
raise rateLimitError()
|
||||
|
||||
if body.startsWith("{\"errors"):
|
||||
let errors = body.fromJson(Errors)
|
||||
if errors in {invalidToken, badToken}:
|
||||
echo "fetch error: ", errors
|
||||
release(account, invalid=true)
|
||||
raise rateLimitError()
|
||||
elif errors in {rateLimited}:
|
||||
account.apis[api].limited = true
|
||||
account.apis[api].limitedAt = epochTime().int
|
||||
echo "[accounts] rate limited, api: ", api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id
|
||||
|
||||
proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
|
||||
fetchImpl result:
|
||||
if not (result.startsWith('{') or result.startsWith('[')):
|
||||
echo resp.status, ": ", result, " --- url: ", url
|
||||
result.setLen(0)
|
||||
|
||||
updateAccount()
|
||||
|
||||
if result.startsWith("{\"errors"):
|
||||
let errors = result.fromJson(Errors)
|
||||
if errors in {invalidToken, badToken}:
|
||||
echo "fetch error: ", errors
|
||||
release(account, invalid=true)
|
||||
raise rateLimitError()
|
||||
|
|
|
@ -11,7 +11,7 @@ var
|
|||
accountPool: seq[GuestAccount]
|
||||
enableLogging = false
|
||||
|
||||
template log(str) =
|
||||
template log(str: varargs[string, `$`]) =
|
||||
if enableLogging: echo "[accounts] ", str
|
||||
|
||||
proc getPoolJson*(): JsonNode =
|
||||
|
@ -91,7 +91,7 @@ proc isLimited(account: GuestAccount; api: Api): bool =
|
|||
|
||||
if limit.limited and (epochTime().int - limit.limitedAt) > dayInSeconds:
|
||||
account.apis[api].limited = false
|
||||
log "resetting limit, api: " & $api & ", id: " & $account.id
|
||||
log "resetting limit, api: ", api, ", id: ", account.id
|
||||
|
||||
return limit.limited or (limit.remaining <= 10 and limit.reset > epochTime().int)
|
||||
else:
|
||||
|
@ -100,15 +100,18 @@ proc isLimited(account: GuestAccount; api: Api): bool =
|
|||
proc isReady(account: GuestAccount; api: Api): bool =
|
||||
not (account.isNil or account.pending > maxConcurrentReqs or account.isLimited(api))
|
||||
|
||||
proc release*(account: GuestAccount; used=false; invalid=false) =
|
||||
proc invalidate*(account: var GuestAccount) =
|
||||
if account.isNil: return
|
||||
if invalid:
|
||||
log "discarding invalid account: " & account.id
|
||||
log "invalidating expired account: ", account.id
|
||||
|
||||
let idx = accountPool.find(account)
|
||||
if idx > -1: accountPool.delete(idx)
|
||||
elif used:
|
||||
dec account.pending
|
||||
# TODO: This isn't sufficient, but it works for now
|
||||
let idx = accountPool.find(account)
|
||||
if idx > -1: accountPool.delete(idx)
|
||||
account = nil
|
||||
|
||||
proc release*(account: GuestAccount; invalid=false) =
|
||||
if account.isNil: return
|
||||
dec account.pending
|
||||
|
||||
proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} =
|
||||
for i in 0 ..< accountPool.len:
|
||||
|
@ -119,9 +122,14 @@ proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} =
|
|||
if not result.isNil and result.isReady(api):
|
||||
inc result.pending
|
||||
else:
|
||||
log "no accounts available for API: " & $api
|
||||
log "no accounts available for API: ", api
|
||||
raise rateLimitError()
|
||||
|
||||
proc setLimited*(account: GuestAccount; api: Api) =
|
||||
account.apis[api].limited = true
|
||||
account.apis[api].limitedAt = epochTime().int
|
||||
log "rate limited, api: ", api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id
|
||||
|
||||
proc setRateLimit*(account: GuestAccount; api: Api; remaining, reset: int) =
|
||||
# avoid undefined behavior in race conditions
|
||||
if api in account.apis:
|
||||
|
|
|
@ -56,7 +56,7 @@ type
|
|||
userNotFound = 50
|
||||
suspended = 63
|
||||
rateLimited = 88
|
||||
invalidToken = 89
|
||||
expiredToken = 89
|
||||
listIdOrSlug = 112
|
||||
tweetNotFound = 144
|
||||
tweetNotAuthorized = 179
|
||||
|
|
Loading…
Reference in New Issue