blankie
/
nitter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use gzip for API calls to lower bandwidth and RAM

This commit is contained in:
Zed 2021-12-26 06:49:27 +01:00
parent 5e49e94bf1
commit 3da6c50cb8
4 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nitter.nimble
View File

@ -22,6 +22,7 @@ requires "supersnappy#2.1.1"
requires "redpool#f880f49" requires "redpool#f880f49"
requires "https://github.com/zedeus/redis#d0a0e6f" requires "https://github.com/zedeus/redis#d0a0e6f"
requires "https://github.com/disruptek/frosty#0.3.1" requires "https://github.com/disruptek/frosty#0.3.1"
requires "zippy#0.7.3"
# Tasks # Tasks

5
src/apiutils.nim
View File

@ -1,5 +1,5 @@
import httpclient, asyncdispatch, options, times, strutils, uri import httpclient, asyncdispatch, options, times, strutils, uri
import packedjson import packedjson, zippy
import types, tokens, consts, parserutils, http_pool import types, tokens, consts, parserutils, http_pool
const rl = "x-rate-limit-" const rl = "x-rate-limit-"
@ -26,6 +26,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
"x-guest-token": if token == nil: "" else: token.tok, "x-guest-token": if token == nil: "" else: token.tok,
"x-twitter-active-user": "yes", "x-twitter-active-user": "yes",
"authority": "api.twitter.com", "authority": "api.twitter.com",
"accept-encoding": "gzip",
"accept-language": "en-US,en;q=0.9", "accept-language": "en-US,en;q=0.9",
"accept": "*/*", "accept": "*/*",
"DNT": "1" "DNT": "1"
@ -44,7 +45,7 @@ proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} =
var resp: AsyncResponse var resp: AsyncResponse
let body = pool.use(headers): let body = pool.use(headers):
resp = await c.get($url) resp = await c.get($url)
await resp.body uncompress(await resp.body)
if body.startsWith('{') or body.startsWith('['): if body.startsWith('{') or body.startsWith('['):
result = parseJson(body) result = parseJson(body)

3
src/http_pool.nim
View File

@ -7,7 +7,8 @@ type
var maxConns {.threadvar.}: int var maxConns {.threadvar.}: int
let keepAlive* = newHttpHeaders({ let keepAlive* = newHttpHeaders({
"Connection": "Keep-Alive" "connection": "Keep-Alive",
"accept-encoding": "gzip"
}) })
proc setMaxHttpConns*(n: int) = proc setMaxHttpConns*(n: int) =

4
src/tokens.nim
View File

@ -1,5 +1,6 @@
import asyncdispatch, httpclient, times, sequtils, json, math, random import asyncdispatch, httpclient, times, sequtils, json, math, random
import strutils, strformat import strutils, strformat
import zippy
import types, agents, consts, http_pool import types, agents, consts, http_pool
const const
@ -28,6 +29,7 @@ proc fetchToken(): Future[Token] {.async.} =
let headers = newHttpHeaders({ let headers = newHttpHeaders({
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"accept-encoding": "gzip",
"accept-language": "en-US,en;q=0.5", "accept-language": "en-US,en;q=0.5",
"connection": "keep-alive", "connection": "keep-alive",
"user-agent": getAgent(), "user-agent": getAgent(),
@ -41,7 +43,7 @@ proc fetchToken(): Future[Token] {.async.} =
try: try:
resp = clientPool.use(headers): await c.postContent(activate) resp = clientPool.use(headers): await c.postContent(activate)
tokNode = parseJson(resp)["guest_token"] tokNode = parseJson(uncompress(resp))["guest_token"]
tok = tokNode.getStr($(tokNode.getInt)) tok = tokNode.getStr($(tokNode.getInt))
let time = getTime() let time = getTime()