nitter/src/parserutils.nim

277 lines
8.1 KiB
Nim
Raw Normal View History

2021-12-27 01:37:38 +00:00
# SPDX-License-Identifier: AGPL-3.0-only
2020-11-14 23:01:13 +00:00
import strutils, times, macros, htmlgen, unicode, options, algorithm
import regex, packedjson
2020-06-01 00:16:24 +00:00
import types, utils, formatters
const
unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
unReplace = "$1<a href=\"/$2\">@$2</a>"
2020-11-14 23:01:13 +00:00
htRegex = re"(^|[^\w-_./?])([#$])([\w_]+)"
2020-06-06 08:17:19 +00:00
htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
2020-06-01 00:16:24 +00:00
2020-11-14 23:01:13 +00:00
type
ReplaceSliceKind = enum
rkRemove, rkUrl, rkHashtag, rkMention
ReplaceSlice = object
slice: Slice[int]
kind: ReplaceSliceKind
url, display: string
template isNull*(js: JsonNode): bool = js.kind == JNull
template notNull*(js: JsonNode): bool = js.kind != JNull
2020-06-01 00:16:24 +00:00
template `?`*(js: JsonNode): untyped =
let j = js
if j.isNull: return
2020-11-10 13:04:01 +00:00
j
2020-06-01 00:16:24 +00:00
template `with`*(ident, value, body): untyped =
block:
let ident {.inject.} = value
2020-06-01 11:40:26 +00:00
if ident != nil: body
2020-06-01 00:16:24 +00:00
template `with`*(ident; value: JsonNode; body): untyped =
block:
let ident {.inject.} = value
if value.notNull: body
2020-06-01 00:16:24 +00:00
2020-06-01 11:47:43 +00:00
template getCursor*(js: JsonNode): string =
2020-06-01 00:16:24 +00:00
js{"content", "operation", "cursor", "value"}.getStr
2020-06-01 19:53:21 +00:00
template getError*(js: JsonNode): Error =
if js.kind != JArray or js.len == 0: null
else: Error(js[0]{"code"}.getInt)
2021-12-20 02:11:12 +00:00
template parseTime(time: string; f: static string; flen: int): DateTime =
2020-06-01 00:16:24 +00:00
if time.len != flen: return
2021-12-20 02:11:12 +00:00
parse(time, f, utc())
2020-06-01 00:16:24 +00:00
2021-12-20 02:11:12 +00:00
proc getDateTime*(js: JsonNode): DateTime =
2020-06-01 00:16:24 +00:00
parseTime(js.getStr, "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", 20)
2021-12-20 02:11:12 +00:00
proc getTime*(js: JsonNode): DateTime =
2020-06-01 00:16:24 +00:00
parseTime(js.getStr, "ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
2020-06-01 11:47:43 +00:00
proc getId*(id: string): string {.inline.} =
2020-06-01 00:16:24 +00:00
let start = id.rfind("-")
if start < 0: return id
id[start + 1 ..< id.len]
2020-06-01 11:47:43 +00:00
proc getId*(js: JsonNode): int64 {.inline.} =
2020-06-01 00:16:24 +00:00
case js.kind
of JString: return parseBiggestInt(js.getStr("0"))
of JInt: return js.getBiggestInt()
else: return 0
2020-11-07 21:48:49 +00:00
proc getEntryId*(js: JsonNode): string {.inline.} =
let entry = js{"entryId"}.getStr
if entry.len == 0: return
if "tweet" in entry or "sq-I-t" in entry:
2020-11-07 21:48:49 +00:00
return entry.getId
elif "tombstone" in entry:
return js{"content", "item", "content", "tombstone", "tweet", "id"}.getStr
else:
echo "unknown entry: ", entry
return
2020-06-01 11:47:43 +00:00
template getStrVal*(js: JsonNode; default=""): string =
2020-06-01 00:16:24 +00:00
js{"string_value"}.getStr(default)
proc getImageStr*(js: JsonNode): string =
result = js.getStr
result.removePrefix(https)
result.removePrefix(twimg)
template getImageVal*(js: JsonNode): string =
js{"image_value", "url"}.getImageStr
2020-06-01 00:16:24 +00:00
proc getCardUrl*(js: JsonNode; kind: CardKind): string =
result = js{"website_url"}.getStrVal
if kind == promoVideoConvo:
result = js{"thank_you_url"}.getStrVal(result)
if result.startsWith("card://"):
result = ""
2020-06-01 00:16:24 +00:00
proc getCardDomain*(js: JsonNode; kind: CardKind): string =
result = js{"vanity_url"}.getStrVal(js{"domain"}.getStr)
if kind == promoVideoConvo:
result = js{"thank_you_vanity_url"}.getStrVal(result)
proc getCardTitle*(js: JsonNode; kind: CardKind): string =
result = js{"title"}.getStrVal
if kind == promoVideoConvo:
result = js{"thank_you_text"}.getStrVal(result)
2020-06-10 14:13:40 +00:00
elif kind == liveEvent:
result = js{"event_category"}.getStrVal
2020-06-10 14:13:40 +00:00
elif kind in {videoDirectMessage, imageDirectMessage}:
result = js{"cta1"}.getStrVal
2020-06-01 00:16:24 +00:00
proc getBanner*(js: JsonNode): string =
let url = js{"profile_banner_url"}.getImageStr
2020-06-01 00:16:24 +00:00
if url.len > 0:
return url & "/1500x500"
let color = js{"profile_link_color"}.getStr
if color.len > 0:
return '#' & color
# use primary color from profile picture color histrogram
with p, js{"profile_image_extensions", "mediaColor", "r", "ok", "palette"}:
if p.len > 0:
2020-06-01 11:50:06 +00:00
let pal = p[0]{"rgb"}
2020-06-01 00:16:24 +00:00
result = "#"
2020-06-01 11:50:06 +00:00
result.add toHex(pal{"red"}.getInt, 2)
result.add toHex(pal{"green"}.getInt, 2)
result.add toHex(pal{"blue"}.getInt, 2)
2020-06-01 00:16:24 +00:00
return
return "#161616"
proc getTombstone*(js: JsonNode): string =
result = js{"tombstoneInfo", "richText", "text"}.getStr
result.removeSuffix(" Learn more")
2020-06-01 00:16:24 +00:00
2020-11-14 23:01:13 +00:00
proc extractSlice(js: JsonNode): Slice[int] =
result = js["indices"][0].getInt ..< js["indices"][1].getInt
2020-06-01 00:16:24 +00:00
2020-11-14 23:01:13 +00:00
proc extractUrls(result: var seq[ReplaceSlice]; js: JsonNode;
textLen: int; hideTwitter = false) =
2020-06-01 00:16:24 +00:00
let
2020-11-14 23:01:13 +00:00
url = js["expanded_url"].getStr
slice = js.extractSlice
2020-06-01 00:16:24 +00:00
2020-11-14 23:01:13 +00:00
if hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl:
if slice.a < textLen:
result.add ReplaceSlice(kind: rkRemove, slice: slice)
2020-06-01 00:16:24 +00:00
else:
2020-11-14 23:01:13 +00:00
result.add ReplaceSlice(kind: rkUrl, url: url,
display: url.shortLink, slice: slice)
proc extractHashtags(result: var seq[ReplaceSlice]; js: JsonNode) =
result.add ReplaceSlice(kind: rkHashtag, slice: js.extractSlice)
proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice];
textSlice: Slice[int]): string =
template extractLowerBound(i: int; idx): int =
if i > 0: repls[idx].slice.b.succ else: textSlice.a
result = newStringOfCap(runes.len)
for i, rep in repls:
result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a]
case rep.kind
of rkHashtag:
let
name = $runes[rep.slice.a.succ .. rep.slice.b]
symbol = $runes[rep.slice.a]
result.add a(symbol & name, href = "/search?q=%23" & name)
of rkMention:
result.add a($runes[rep.slice], href = rep.url, title = rep.display)
of rkUrl:
result.add a(rep.display, href = rep.url)
of rkRemove:
discard
let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b
if rest.a <= rest.b:
result.add $runes[rest]
proc deduplicate(s: var seq[ReplaceSlice]) =
var
len = s.len
i = 0
while i < len:
var j = i + 1
while j < len:
if s[i].slice.a == s[j].slice.a:
s.del j
dec len
else:
inc j
inc i
proc cmp(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
2020-06-01 00:16:24 +00:00
proc expandProfileEntities*(profile: var Profile; js: JsonNode) =
let
2020-11-14 23:01:13 +00:00
orig = profile.bio.toRunes
2020-06-01 00:16:24 +00:00
ent = ? js{"entities"}
with urls, ent{"url", "urls"}:
profile.website = urls[0]{"expanded_url"}.getStr
2020-11-14 23:01:13 +00:00
var replacements = newSeq[ReplaceSlice]()
2020-06-01 00:16:24 +00:00
with urls, ent{"description", "urls"}:
2020-11-14 23:01:13 +00:00
for u in urls:
replacements.extractUrls(u, orig.high)
replacements.deduplicate
replacements.sort(cmp)
2020-06-01 00:16:24 +00:00
2020-11-14 23:01:13 +00:00
profile.bio = orig.replacedWith(replacements, 0 .. orig.len)
2020-06-01 00:16:24 +00:00
profile.bio = profile.bio.replace(unRegex, unReplace)
.replace(htRegex, htReplace)
proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
let
2020-11-14 23:01:13 +00:00
orig = tweet.text.toRunes
textRange = js{"display_text_range"}
2020-11-14 23:01:13 +00:00
textSlice = textRange{0}.getInt .. textRange{1}.getInt
2020-06-01 00:16:24 +00:00
hasQuote = js{"is_quote_status"}.getBool
hasCard = tweet.card.isSome
var replyTo = ""
if tweet.replyId != 0:
with reply, js{"in_reply_to_screen_name"}:
tweet.reply.add reply.getStr
replyTo = reply.getStr
let ent = ? js{"entities"}
2020-11-14 23:01:13 +00:00
var replacements = newSeq[ReplaceSlice]()
2020-06-01 00:16:24 +00:00
with urls, ent{"urls"}:
for u in urls:
2020-11-14 23:01:13 +00:00
let urlStr = u["url"].getStr
if urlStr.len == 0 or urlStr notin tweet.text:
continue
replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote)
2020-06-01 00:16:24 +00:00
if hasCard and u{"url"}.getStr == get(tweet.card).url:
get(tweet.card).url = u{"expanded_url"}.getStr
with media, ent{"media"}:
2020-11-14 23:01:13 +00:00
for m in media:
replacements.extractUrls(m, textSlice.b, hideTwitter = true)
if "hashtags" in ent:
for hashtag in ent["hashtags"]:
replacements.extractHashtags(hashtag)
if "symbols" in ent:
for symbol in ent["symbols"]:
replacements.extractHashtags(symbol)
if "user_mentions" in ent:
for mention in ent["user_mentions"]:
let
name = mention{"screen_name"}.getStr
slice = mention.extractSlice
idx = tweet.reply.find(name)
if slice.a >= textSlice.a:
replacements.add ReplaceSlice(kind: rkMention, slice: slice,
url: "/" & name, display: mention["name"].getStr)
if idx > -1 and name != replyTo:
tweet.reply.delete idx
elif idx == -1 and tweet.replyId != 0:
tweet.reply.add name
replacements.deduplicate
replacements.sort(cmp)
tweet.text = orig.replacedWith(replacements, textSlice)