nitter/src/apiutils.nim

import httpclient, asyncdispatch, options, times, strutils, uri
import packedjson, zippy
import types, tokens, consts, parserutils, http_pool

const rl = "x-rate-limit-"

var pool {.threadvar.}: HttpPool

proc genParams*(pars: openarray[(string, string)] = @[]; cursor="";
                count="20"; ext=true): seq[(string, string)] =
  result = timelineParams
  for p in pars:
    result &= p
  if ext:
    result &= ("ext", "mediaStats")
  if cursor.len > 0:
    result &= ("cursor", cursor)
  if count.len > 0:
    result &= ("count", count)

proc genHeaders*(token: Token = nil): HttpHeaders =
  result = newHttpHeaders({
    "connection": "keep-alive",
    "authorization": auth,
    "content-type": "application/json",
    "x-guest-token": if token == nil: "" else: token.tok,
    "x-twitter-active-user": "yes",
    "authority": "api.twitter.com",
    "accept-encoding": "gzip",
    "accept-language": "en-US,en;q=0.9",
    "accept": "*/*",
    "DNT": "1"
  })

proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} =
  once:
    pool = HttpPool()

  var token = await getToken()
  if token.tok.len == 0:
    raise rateLimitError()

  let headers = genHeaders(token)
  try:
    var resp: AsyncResponse
    let body = pool.use(headers):
      resp = await c.get($url)
      uncompress(await resp.body)

    if body.startsWith('{') or body.startsWith('['):
      result = parseJson(body)
    else:
      echo resp.status, ": ", body
      result = newJNull()

    if not oldApi and resp.headers.hasKey(rl & "reset"):
      token.remaining = parseInt(resp.headers[rl & "remaining"])
      token.reset = fromUnix(parseInt(resp.headers[rl & "reset"]))

    if result.getError notin {invalidToken, forbidden, badToken}:
      token.lastUse = getTime()
    else:
      echo "fetch error: ", result.getError
      release(token, true)
      raise rateLimitError()
  except Exception as e:
    echo "error: ", e.msg, ", token: ", token[], ", url: ", url
    if "length" notin e.msg and "descriptor" notin e.msg:
      release(token, true)
    raise rateLimitError()