From 3da6c50cb8cc8cfa2c361465fa5099fa86d80d68 Mon Sep 17 00:00:00 2001
From: Zed <zedeus@pm.me>
Date: Sun, 26 Dec 2021 06:49:27 +0100
Subject: [PATCH] Use gzip for API calls to lower bandwidth and RAM

---
 nitter.nimble     | 1 +
 src/apiutils.nim  | 5 +++--
 src/http_pool.nim | 3 ++-
 src/tokens.nim    | 4 +++-
 4 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/nitter.nimble b/nitter.nimble
index d8ffa98..48f9b74 100644
--- a/nitter.nimble
+++ b/nitter.nimble
@@ -22,6 +22,7 @@ requires "supersnappy#2.1.1"
 requires "redpool#f880f49"
 requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "https://github.com/disruptek/frosty#0.3.1"
+requires "zippy#0.7.3"
 
 
 # Tasks
diff --git a/src/apiutils.nim b/src/apiutils.nim
index 6b2b83f..cee5ab5 100644
--- a/src/apiutils.nim
+++ b/src/apiutils.nim
@@ -1,5 +1,5 @@
 import httpclient, asyncdispatch, options, times, strutils, uri
-import packedjson
+import packedjson, zippy
 import types, tokens, consts, parserutils, http_pool
 
 const rl = "x-rate-limit-"
@@ -26,6 +26,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
     "x-guest-token": if token == nil: "" else: token.tok,
     "x-twitter-active-user": "yes",
     "authority": "api.twitter.com",
+    "accept-encoding": "gzip",
     "accept-language": "en-US,en;q=0.9",
     "accept": "*/*",
     "DNT": "1"
@@ -44,7 +45,7 @@ proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} =
     var resp: AsyncResponse
     let body = pool.use(headers):
       resp = await c.get($url)
-      await resp.body
+      uncompress(await resp.body)
 
     if body.startsWith('{') or body.startsWith('['):
       result = parseJson(body)
diff --git a/src/http_pool.nim b/src/http_pool.nim
index 2daf030..9fb8457 100644
--- a/src/http_pool.nim
+++ b/src/http_pool.nim
@@ -7,7 +7,8 @@ type
 var maxConns {.threadvar.}: int
 
 let keepAlive* = newHttpHeaders({
-  "Connection": "Keep-Alive"
+  "connection": "Keep-Alive",
+  "accept-encoding": "gzip"
 })
 
 proc setMaxHttpConns*(n: int) =
diff --git a/src/tokens.nim b/src/tokens.nim
index 1cf35cf..9ce7bcf 100644
--- a/src/tokens.nim
+++ b/src/tokens.nim
@@ -1,5 +1,6 @@
 import asyncdispatch, httpclient, times, sequtils, json, math, random
 import strutils, strformat
+import zippy
 import types, agents, consts, http_pool
 
 const
@@ -28,6 +29,7 @@ proc fetchToken(): Future[Token] {.async.} =
 
   let headers = newHttpHeaders({
     "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    "accept-encoding": "gzip",
     "accept-language": "en-US,en;q=0.5",
     "connection": "keep-alive",
     "user-agent": getAgent(),
@@ -41,7 +43,7 @@ proc fetchToken(): Future[Token] {.async.} =
 
   try:
     resp = clientPool.use(headers): await c.postContent(activate)
-    tokNode = parseJson(resp)["guest_token"]
+    tokNode = parseJson(uncompress(resp))["guest_token"]
     tok = tokNode.getStr($(tokNode.getInt))
 
     let time = getTime()