Switch to packedjson to try lowering memory usage

nitter.nimble

@@ -11,10 +11,8 @@ bin           = @["nitter"]
 # Dependencies
 
 requires "nim >= 1.2.0"
-requires "norm#head"
 requires "jester#head"
 requires "regex#2d96bab"
-requires "q >= 0.0.7"
 requires "nimcrypto >= 0.4.11"
 requires "karax >= 1.1.2"
 requires "sass"
@@ -22,6 +20,7 @@ requires "markdown#head"
 requires "https://github.com/zedeus/redis#head"
 requires "redpool#head"
 requires "msgpack4nim >= 0.3.1"
+requires "packedjson"
 
 
 # Tasks

src/api.nim

@@ -1,4 +1,5 @@
-import asyncdispatch, httpclient, uri, strutils, json
+import asyncdispatch, httpclient, uri, strutils
+import packedjson
 import types, query, formatters, consts, apiutils, parser
 
 proc getGraphProfile*(username: string): Future[Profile] {.async.} =

src/apiutils.nim

@@ -1,5 +1,6 @@
-import httpclient, asyncdispatch, options, times, strutils, json, uri
+import httpclient, asyncdispatch, options, times, strutils, uri
-import types, agents, tokens, consts
+import packedjson
+import types, agents, tokens, consts, parserutils
 
 proc genParams*(pars: openarray[(string, string)] = @[];
                 cursor=""): seq[(string, string)] =
@@ -41,19 +42,20 @@ proc fetch*(url: Uri; retried=false; oldApi=false): Future[JsonNode] {.async.} =
 
     if resp.status != $Http200:
       if "Bad guest token" in body:
-        return
+        keepToken = false
+        return newJNull()
       elif not body.startsWith('{'):
         echo resp.status, " ", body
-        return
+        return newJNull()
 
     result = parseJson(body)
 
-    if result{"errors"} != nil and result.getError == forbidden:
+    if result{"errors"}.notNull and result.getError == forbidden:
       keepToken = false
       echo "bad token"
   except:
     echo "error: ", url
-    result = nil
+    result = newJNull()
   finally:
     if keepToken:
       token.release()

src/parser.nim

@@ -1,8 +1,9 @@
-import json, strutils, options, tables, times, math
+import strutils, options, tables, times, math
+import packedjson
 import types, parserutils
 
 proc parseProfile(js: JsonNode; id=""): Profile =
-  if js == nil: return
+  if js.isNull: return
   result = Profile(
     id: if id.len > 0: id else: js{"id_str"}.getStr,
     username: js{"screen_name"}.getStr,
@@ -24,7 +25,7 @@ proc parseProfile(js: JsonNode; id=""): Profile =
   result.expandProfileEntities(js)
 
 proc parseUserShow*(js: JsonNode; username: string): Profile =
-  if js == nil: return
+  if js.isNull: return
   with error, js{"errors"}:
     result = Profile(username: username)
     if error.getError == suspended:
@@ -34,7 +35,7 @@ proc parseUserShow*(js: JsonNode; username: string): Profile =
   result = parseProfile(js)
 
 proc parseGraphProfile*(js: JsonNode; username: string): Profile =
-  if js == nil: return
+  if js.isNull: return
   with error, js{"errors"}:
     result = Profile(username: username)
     if error.getError == suspended:
@@ -46,12 +47,12 @@ proc parseGraphProfile*(js: JsonNode; username: string): Profile =
   parseProfile(user, id)
 
 proc parseGraphList*(js: JsonNode): List =
-  if js == nil: return
+  if js.isNull: return
 
   var list = js{"data", "user_by_screen_name", "list"}
-  if list == nil:
+  if list.isNull:
     list = js{"data", "list"}
-  if list == nil:
+  if list.isNull:
     return
 
   result = List(
@@ -70,7 +71,7 @@ proc parseListMembers*(js: JsonNode; cursor: string): Result[Profile] =
     query: Query(kind: userList)
   )
 
-  if js == nil: return
+  if js.isNull: return
 
   result.top = js{"previous_cursor_str"}.getStr
   result.bottom = js{"next_cursor_str"}.getStr
@@ -116,7 +117,7 @@ proc parseVideo(js: JsonNode): Video =
 
   for v in js{"video_info", "variants"}:
     result.variants.add VideoVariant(
-      videoType: v{"content_type"}.to(VideoType),
+      videoType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
       bitrate: v{"bitrate"}.getInt,
       url: v{"url"}.getStr
     )
@@ -192,14 +193,14 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
       break
 
 proc parseTweet(js: JsonNode): Tweet =
-  if js == nil: return
+  if js.isNull: return
   result = Tweet(
     id: js{"id_str"}.getId,
     threadId: js{"conversation_id_str"}.getId,
     replyId: js{"in_reply_to_status_id_str"}.getId,
     text: js{"full_text"}.getStr,
     time: js{"created_at"}.getTime,
-    hasThread: js{"self_thread"} != nil,
+    hasThread: js{"self_thread"}.notNull,
     available: true,
     profile: Profile(id: js{"user_id_str"}.getStr),
     stats: TweetStats(
@@ -354,7 +355,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
   if instructions.len == 0: return
 
   for i in instructions:
-    if result.beginning and i{"pinEntry"} != nil:
+    if result.beginning and i{"pinEntry"}.notNull:
       with pin, parsePin(i, global):
         result.content.add pin
     else:

src/parserutils.nim

@@ -1,5 +1,5 @@
-import json, strutils, times, macros, htmlgen, uri, unicode, options
+import strutils, times, macros, htmlgen, uri, unicode, options
-import regex
+import regex, packedjson
 import types, utils, formatters
 
 const
@@ -11,9 +11,12 @@ const
 
 let localTimezone = local()
 
+template isNull*(js: JsonNode): bool = js.kind == JNull
+template notNull*(js: JsonNode): bool = js.kind != JNull
+
 template `?`*(js: JsonNode): untyped =
   let j = js
-  if j == nil: return
+  if j.isNull: return
   else: j
 
 template `with`*(ident, value, body): untyped =
@@ -24,8 +27,7 @@ template `with`*(ident, value, body): untyped =
 template `with`*(ident; value: JsonNode; body): untyped =
   block:
     let ident {.inject.} = value
-    if ident != nil and ident.kind != JNull:
+    if value.notNull: body
-      body
 
 template getCursor*(js: JsonNode): string =
   js{"content", "operation", "cursor", "value"}.getStr
@@ -50,7 +52,6 @@ proc getId*(id: string): string {.inline.} =
   id[start + 1 ..< id.len]
 
 proc getId*(js: JsonNode): int64 {.inline.} =
-  if js == nil: return
   case js.kind
   of JString: return parseBiggestInt(js.getStr("0"))
   of JInt: return js.getBiggestInt()
@@ -117,10 +118,9 @@ template getSlice(text: string; slice: seq[int]): string =
   text.runeSubStr(slice[0], slice[1] - slice[0])
 
 proc getSlice(text: string; js: JsonNode): string =
-  if js == nil or js.kind != JArray or js.len < 2 or
+  if js.kind != JArray or js.len < 2 or js[0].kind != JInt: return text
-    js[0].kind != JInt: return text
 
-  let slice = js.to(seq[int])
+  let slice = @[js{0}.getInt, js{1}.getInt]
   text.getSlice(slice)
 
 proc expandUrl(text: var string; js: JsonNode; tLen: int; hideTwitter=false) =
@@ -130,9 +130,9 @@ proc expandUrl(text: var string; js: JsonNode; tLen: int; hideTwitter=false) =
 
   let
     url = js{"expanded_url"}.getStr
-    slice = js{"indices"}.to(seq[int])
+    slice = js{"indices"}[1].getInt
 
-  if hideTwitter and slice[1] >= tLen and url.isTwitterUrl:
+  if hideTwitter and slice >= tLen and url.isTwitterUrl:
     text = text.replace(u, "")
     text.removeSuffix(' ')
     text.removeSuffix('\n')
@@ -178,7 +178,8 @@ proc expandProfileEntities*(profile: var Profile; js: JsonNode) =
 proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
   let
     orig = tweet.text
-    slice = js{"display_text_range"}.to(seq[int])
+    textRange = js{"display_text_range"}
+    slice = @[textRange{0}.getInt, textRange{1}.getInt]
     hasQuote = js{"is_quote_status"}.getBool
     hasCard = tweet.card.isSome