From 412055864940a79d3203aa41568c6ed67c5bb5f8 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 31 Oct 2023 12:04:32 +0000 Subject: [PATCH] Replace /.tokens with /.health and /.accounts --- nitter.example.conf | 2 +- src/apiutils.nim | 4 +- src/{tokens.nim => auth.nim} | 113 +++++++++++++++-------- src/experimental/parser/guestaccount.nim | 3 +- src/nitter.nim | 2 +- src/routes/debug.nim | 9 +- src/types.nim | 2 +- 7 files changed, 86 insertions(+), 49 deletions(-) rename src/{tokens.nim => auth.nim} (69%) diff --git a/nitter.example.conf b/nitter.example.conf index 0d4deb7..f0b4214 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -23,7 +23,7 @@ redisMaxConnections = 30 hmacKey = "secretkey" # random key for cryptographic signing of video urls base64Media = false # use base64 encoding for proxied media urls enableRSS = true # set this to false to disable RSS feeds -enableDebug = false # enable request logs and debug endpoints (/.tokens) +enableDebug = false # enable request logs and debug endpoints (/.accounts) proxy = "" # http/https url, SOCKS proxies are not supported proxyAuth = "" tokenCount = 10 diff --git a/src/apiutils.nim b/src/apiutils.nim index 9ac101e..1ff05eb 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only import httpclient, asyncdispatch, options, strutils, uri, times, math, tables import jsony, packedjson, zippy, oauth1 -import types, tokens, consts, parserutils, http_pool +import types, auth, consts, parserutils, http_pool import experimental/types/common const @@ -120,7 +120,7 @@ template fetchImpl(result, fetchBody) {.dirty.} = except OSError as e: raise e except Exception as e: - let id = if account.isNil: "null" else: account.id + let id = if account.isNil: "null" else: $account.id echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url raise rateLimitError() finally: diff --git a/src/tokens.nim b/src/auth.nim similarity index 69% rename from src/tokens.nim rename to src/auth.nim index ca74ddc..560fb84 100644 --- a/src/tokens.nim +++ b/src/auth.nim @@ -1,5 +1,5 @@ #SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, times, json, random, strutils, tables, sets, os +import asyncdispatch, times, json, random, strutils, tables, intsets, os import types import experimental/parser/guestaccount @@ -7,6 +7,21 @@ import experimental/parser/guestaccount const maxConcurrentReqs = 2 dayInSeconds = 24 * 60 * 60 + apiMaxReqs: Table[Api, int] = { + Api.search: 50, + Api.tweetDetail: 150, + Api.photoRail: 180, + Api.userTweets: 500, + Api.userTweetsAndReplies: 500, + Api.userMedia: 500, + Api.userRestId: 500, + Api.userScreenName: 500, + Api.tweetResult: 500, + Api.list: 500, + Api.listTweets: 500, + Api.listMembers: 500, + Api.listBySlug: 500 + }.toTable var accountPool: seq[GuestAccount] @@ -15,20 +30,64 @@ var template log(str: varargs[string, `$`]) = if enableLogging: echo "[accounts] ", str.join("") -proc getPoolJson*(): JsonNode = - var - list = newJObject() - totalReqs = 0 - totalPending = 0 - limited: HashSet[string] - reqsPerApi: Table[string, int] - +proc getAccountPoolHealth*(): JsonNode = let now = epochTime().int - for account in accountPool: - totalPending.inc(account.pending) + var + totalReqs = 0 + limited: IntSet + reqsPerApi: Table[string, int] + oldest = now + newest = 0 + average = 0 - var includeAccount = false + for account in accountPool: + # Twitter snowflake conversion + let created = ((account.id shr 22) + 1288834974657) div 1000 + + if created > newest: + newest = created + if created < oldest: + oldest = created + average.inc created + + for api in account.apis.keys: + let + apiStatus = account.apis[api] + reqs = apiMaxReqs[api] - apiStatus.remaining + + reqsPerApi.mgetOrPut($api, 0).inc reqs + totalReqs.inc reqs + + if apiStatus.limited: + limited.incl account.id + + if accountPool.len > 0: + average = average div accountPool.len + else: + oldest = 0 + average = 0 + + return %*{ + "accounts": %*{ + "total": accountPool.len, + "active": accountPool.len - limited.card, + "limited": limited.card, + "oldest": $fromUnix(oldest), + "newest": $fromUnix(newest), + "average": $fromUnix(average) + }, + "requests": %*{ + "total": totalReqs, + "apis": reqsPerApi + } + } + +proc getAccountPoolDebug*(): JsonNode = + let now = epochTime().int + var list = newJObject() + + for account in accountPool: let accountJson = %*{ "apis": newJObject(), "pending": account.pending, @@ -47,37 +106,11 @@ proc getPoolJson*(): JsonNode = if apiStatus.limited: obj["limited"] = %true - limited.incl account.id accountJson{"apis", $api} = obj - includeAccount = true + list[$account.id] = accountJson - let - maxReqs = - case api - of Api.search: 50 - of Api.tweetDetail: 150 - of Api.photoRail: 180 - of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia, - Api.userRestId, Api.userScreenName, - Api.tweetResult, - Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500 - reqs = maxReqs - apiStatus.remaining - - reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs - totalReqs.inc(reqs) - - if includeAccount: - list[account.id] = accountJson - - return %*{ - "amount": accountPool.len, - "limited": limited.card, - "requests": totalReqs, - "pending": totalPending, - "apis": reqsPerApi, - "accounts": list - } + return %list proc rateLimitError*(): ref RateLimitError = newException(RateLimitError, "rate limited") diff --git a/src/experimental/parser/guestaccount.nim b/src/experimental/parser/guestaccount.nim index 4d8ff47..f7e6d34 100644 --- a/src/experimental/parser/guestaccount.nim +++ b/src/experimental/parser/guestaccount.nim @@ -1,3 +1,4 @@ +import std/strutils import jsony import ../types/guestaccount from ../../types import GuestAccount @@ -5,7 +6,7 @@ from ../../types import GuestAccount proc toGuestAccount(account: RawAccount): GuestAccount = let id = account.oauthToken[0 ..< account.oauthToken.find('-')] result = GuestAccount( - id: id, + id: parseBiggestInt(id), oauthToken: account.oauthToken, oauthSecret: account.oauthTokenSecret ) diff --git a/src/nitter.nim b/src/nitter.nim index 1b4862b..dfc1dfd 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -6,7 +6,7 @@ from os import getEnv import jester -import types, config, prefs, formatters, redis_cache, http_pool, tokens +import types, config, prefs, formatters, redis_cache, http_pool, auth import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, diff --git a/src/routes/debug.nim b/src/routes/debug.nim index 192786e..895a285 100644 --- a/src/routes/debug.nim +++ b/src/routes/debug.nim @@ -1,10 +1,13 @@ # SPDX-License-Identifier: AGPL-3.0-only import jester import router_utils -import ".."/[tokens, types] +import ".."/[auth, types] proc createDebugRouter*(cfg: Config) = router debug: - get "/.tokens": + get "/.health": + respJson getAccountPoolHealth() + + get "/.accounts": cond cfg.enableDebug - respJson getPoolJson() + respJson getAccountPoolDebug() diff --git a/src/types.nim b/src/types.nim index 3f5f8ac..3b0d55c 100644 --- a/src/types.nim +++ b/src/types.nim @@ -36,7 +36,7 @@ type limitedAt*: int GuestAccount* = ref object - id*: string + id*: BiggestInt oauthToken*: string oauthSecret*: string pending*: int