Switch to packedjson to try lowering memory usage

This commit is contained in:
Zed 2020-06-02 16:22:44 +02:00
parent 2fa76db099
commit fb591e43b8
5 changed files with 37 additions and 33 deletions

View File

@ -11,10 +11,8 @@ bin = @["nitter"]
# Dependencies
requires "nim >= 1.2.0"
requires "norm#head"
requires "jester#head"
requires "regex#2d96bab"
requires "q >= 0.0.7"
requires "nimcrypto >= 0.4.11"
requires "karax >= 1.1.2"
requires "sass"
@ -22,6 +20,7 @@ requires "markdown#head"
requires "https://github.com/zedeus/redis#head"
requires "redpool#head"
requires "msgpack4nim >= 0.3.1"
requires "packedjson"
# Tasks

View File

@ -1,4 +1,5 @@
import asyncdispatch, httpclient, uri, strutils, json
import asyncdispatch, httpclient, uri, strutils
import packedjson
import types, query, formatters, consts, apiutils, parser
proc getGraphProfile*(username: string): Future[Profile] {.async.} =

View File

@ -1,5 +1,6 @@
import httpclient, asyncdispatch, options, times, strutils, json, uri
import types, agents, tokens, consts
import httpclient, asyncdispatch, options, times, strutils, uri
import packedjson
import types, agents, tokens, consts, parserutils
proc genParams*(pars: openarray[(string, string)] = @[];
cursor=""): seq[(string, string)] =
@ -41,19 +42,20 @@ proc fetch*(url: Uri; retried=false; oldApi=false): Future[JsonNode] {.async.} =
if resp.status != $Http200:
if "Bad guest token" in body:
return
keepToken = false
return newJNull()
elif not body.startsWith('{'):
echo resp.status, " ", body
return
return newJNull()
result = parseJson(body)
if result{"errors"} != nil and result.getError == forbidden:
if result{"errors"}.notNull and result.getError == forbidden:
keepToken = false
echo "bad token"
except:
echo "error: ", url
result = nil
result = newJNull()
finally:
if keepToken:
token.release()

View File

@ -1,8 +1,9 @@
import json, strutils, options, tables, times, math
import strutils, options, tables, times, math
import packedjson
import types, parserutils
proc parseProfile(js: JsonNode; id=""): Profile =
if js == nil: return
if js.isNull: return
result = Profile(
id: if id.len > 0: id else: js{"id_str"}.getStr,
username: js{"screen_name"}.getStr,
@ -24,7 +25,7 @@ proc parseProfile(js: JsonNode; id=""): Profile =
result.expandProfileEntities(js)
proc parseUserShow*(js: JsonNode; username: string): Profile =
if js == nil: return
if js.isNull: return
with error, js{"errors"}:
result = Profile(username: username)
if error.getError == suspended:
@ -34,7 +35,7 @@ proc parseUserShow*(js: JsonNode; username: string): Profile =
result = parseProfile(js)
proc parseGraphProfile*(js: JsonNode; username: string): Profile =
if js == nil: return
if js.isNull: return
with error, js{"errors"}:
result = Profile(username: username)
if error.getError == suspended:
@ -46,12 +47,12 @@ proc parseGraphProfile*(js: JsonNode; username: string): Profile =
parseProfile(user, id)
proc parseGraphList*(js: JsonNode): List =
if js == nil: return
if js.isNull: return
var list = js{"data", "user_by_screen_name", "list"}
if list == nil:
if list.isNull:
list = js{"data", "list"}
if list == nil:
if list.isNull:
return
result = List(
@ -70,7 +71,7 @@ proc parseListMembers*(js: JsonNode; cursor: string): Result[Profile] =
query: Query(kind: userList)
)
if js == nil: return
if js.isNull: return
result.top = js{"previous_cursor_str"}.getStr
result.bottom = js{"next_cursor_str"}.getStr
@ -116,7 +117,7 @@ proc parseVideo(js: JsonNode): Video =
for v in js{"video_info", "variants"}:
result.variants.add VideoVariant(
videoType: v{"content_type"}.to(VideoType),
videoType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
bitrate: v{"bitrate"}.getInt,
url: v{"url"}.getStr
)
@ -192,14 +193,14 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
break
proc parseTweet(js: JsonNode): Tweet =
if js == nil: return
if js.isNull: return
result = Tweet(
id: js{"id_str"}.getId,
threadId: js{"conversation_id_str"}.getId,
replyId: js{"in_reply_to_status_id_str"}.getId,
text: js{"full_text"}.getStr,
time: js{"created_at"}.getTime,
hasThread: js{"self_thread"} != nil,
hasThread: js{"self_thread"}.notNull,
available: true,
profile: Profile(id: js{"user_id_str"}.getStr),
stats: TweetStats(
@ -354,7 +355,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
if instructions.len == 0: return
for i in instructions:
if result.beginning and i{"pinEntry"} != nil:
if result.beginning and i{"pinEntry"}.notNull:
with pin, parsePin(i, global):
result.content.add pin
else:

View File

@ -1,5 +1,5 @@
import json, strutils, times, macros, htmlgen, uri, unicode, options
import regex
import strutils, times, macros, htmlgen, uri, unicode, options
import regex, packedjson
import types, utils, formatters
const
@ -11,9 +11,12 @@ const
let localTimezone = local()
template isNull*(js: JsonNode): bool = js.kind == JNull
template notNull*(js: JsonNode): bool = js.kind != JNull
template `?`*(js: JsonNode): untyped =
let j = js
if j == nil: return
if j.isNull: return
else: j
template `with`*(ident, value, body): untyped =
@ -24,8 +27,7 @@ template `with`*(ident, value, body): untyped =
template `with`*(ident; value: JsonNode; body): untyped =
block:
let ident {.inject.} = value
if ident != nil and ident.kind != JNull:
body
if value.notNull: body
template getCursor*(js: JsonNode): string =
js{"content", "operation", "cursor", "value"}.getStr
@ -50,7 +52,6 @@ proc getId*(id: string): string {.inline.} =
id[start + 1 ..< id.len]
proc getId*(js: JsonNode): int64 {.inline.} =
if js == nil: return
case js.kind
of JString: return parseBiggestInt(js.getStr("0"))
of JInt: return js.getBiggestInt()
@ -117,10 +118,9 @@ template getSlice(text: string; slice: seq[int]): string =
text.runeSubStr(slice[0], slice[1] - slice[0])
proc getSlice(text: string; js: JsonNode): string =
if js == nil or js.kind != JArray or js.len < 2 or
js[0].kind != JInt: return text
if js.kind != JArray or js.len < 2 or js[0].kind != JInt: return text
let slice = js.to(seq[int])
let slice = @[js{0}.getInt, js{1}.getInt]
text.getSlice(slice)
proc expandUrl(text: var string; js: JsonNode; tLen: int; hideTwitter=false) =
@ -130,9 +130,9 @@ proc expandUrl(text: var string; js: JsonNode; tLen: int; hideTwitter=false) =
let
url = js{"expanded_url"}.getStr
slice = js{"indices"}.to(seq[int])
slice = js{"indices"}[1].getInt
if hideTwitter and slice[1] >= tLen and url.isTwitterUrl:
if hideTwitter and slice >= tLen and url.isTwitterUrl:
text = text.replace(u, "")
text.removeSuffix(' ')
text.removeSuffix('\n')
@ -178,7 +178,8 @@ proc expandProfileEntities*(profile: var Profile; js: JsonNode) =
proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
let
orig = tweet.text
slice = js{"display_text_range"}.to(seq[int])
textRange = js{"display_text_range"}
slice = @[textRange{0}.getInt, textRange{1}.getInt]
hasQuote = js{"is_quote_status"}.getBool
hasCard = tweet.card.isSome