Rewrite text parsing to ensure correctness

src/formatters.nim

@@ -6,12 +6,6 @@ import types, utils, query
 from unicode import Rune, `$`
 
 const
-  urlRegex = re"((https?|ftp)://(-\.)?([^\s/?\.#]+\.?)+([/\?][^\s\)]*)?)"
-  emailRegex = re"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
-  usernameRegex = re"(^|[^A-z0-9_?\/])@([A-z0-9_]+)"
-  picRegex = re"pic.twitter.com/[^ ]+"
-  ellipsisRegex = re" ?…"
-  hashtagRegex = re"([^\S]|^)([#$]\w+)"
   ytRegex = re"(www.|m.)?youtu(be.com|.be)"
   twRegex = re"(www.|mobile.)?twitter.com"
   nbsp = $Rune(0x000A0)
@@ -26,75 +20,14 @@ proc shortLink*(text: string; length=28): string =
   if result.len > length:
     result = result[0 ..< length] & "…"
 
-proc toLink*(url, text: string): string =
+proc replaceUrl*(url: string; prefs: Prefs; rss=false): string =
-  a(text, href=url)
-
-proc reUrlToShortLink*(m: RegexMatch; s: string): string =
-  let url = s[m.group(0)[0]]
-  toLink(url, shortLink(url))
-
-proc reUrlToLink*(m: RegexMatch; s: string): string =
-  let url = s[m.group(0)[0]]
-  toLink(url, url.replace(re"https?://(www.)?", ""))
-
-proc reEmailToLink*(m: RegexMatch; s: string): string =
-  let url = s[m.group(0)[0]]
-  toLink("mailto://" & url, url)
-
-proc reHashtagToLink*(m: RegexMatch; s: string): string =
-  result = if m.group(0).len > 0: s[m.group(0)[0]] else: ""
-  let hash = s[m.group(1)[0]]
-  let link = toLink("/search?q=" & encodeUrl(hash), hash)
-  if hash.any(isAlphaAscii):
-    result &= link
-  else:
-    result &= hash
-
-proc reUsernameToLink*(m: RegexMatch; s: string): string =
-  var username = ""
-  var pretext = ""
-
-  let pre = m.group(0)
-  let match = m.group(1)
-
-  username = s[match[0]]
-
-  if pre.len > 0:
-    pretext = s[pre[0]]
-
-  pretext & toLink("/" & username, "@" & username)
-
-proc reUsernameToFullLink*(m: RegexMatch; s: string): string =
-  result = reUsernameToLink(m, s)
-  result = result.replace("href=\"/", &"href=\"https://{hostname}/")
-
-proc replaceUrl*(url: string; prefs: Prefs): string =
   result = url
   if prefs.replaceYouTube.len > 0:
     result = result.replace(ytRegex, prefs.replaceYouTube)
   if prefs.replaceTwitter.len > 0:
     result = result.replace(twRegex, prefs.replaceTwitter)
-
-proc linkifyText*(text: string; prefs: Prefs; rss=false): string =
-  result = xmltree.escape(stripText(text))
-  result = result.replace(ellipsisRegex, " ")
-  result = result.replace(emailRegex, reEmailToLink)
   if rss:
-    result = result.replace(urlRegex, reUrlToLink)
+    result = result.replace("href=\"/", "href=\"" & hostname & "/")
-    result = result.replace(usernameRegex, reUsernameToFullLink)
-  else:
-    result = result.replace(urlRegex, reUrlToShortLink)
-    result = result.replace(usernameRegex, reUsernameToLink)
-  result = result.replace(hashtagRegex, reHashtagToLink)
-  result = result.replace(re"([^\s\(\n%])<a", "$1 <a")
-  result = result.replace(re"</a>\s+([;.,!\)'%]|&apos;)", "</a>$1")
-  result = result.replace(re"^\. <a", ".<a")
-  result = result.replaceUrl(prefs)
-
-proc stripTwitterUrls*(text: string): string =
-  result = text
-  result = result.replace(picRegex, "")
-  result = result.replace(ellipsisRegex, "")
 
 proc proxifyVideo*(manifest: string; proxy: bool): string =
   proc cb(m: RegexMatch; s: string): string =

src/parserutils.nim

@@ -1,9 +1,10 @@
-import xmltree, strtabs, strformat, strutils, times
+import xmltree, strtabs, strformat, strutils, times, uri
 import regex
 
 import types, formatters
 
 from q import nil
+from htmlgen import a
 
 const
   thumbRegex = re".+:url\('([^']+)'\)"
@@ -41,29 +42,41 @@ proc isVerified*(profile: XmlNode): bool =
 proc isProtected*(profile: XmlNode): bool =
   getHeader(profile).select(".Icon.Icon--protected") != nil
 
-proc emojify*(node: XmlNode) =
+proc parseText*(text: XmlNode; skipLink=""): string =
-  for i in node.selectAll(".Emoji"):
+  for el in text:
-    i.add newText(i.attr("alt"))
+    case el.kind
+    of xnText:
+      result.add el
+    of xnElement:
+      if el.attrs == nil:
+        if el.tag == "strong":
+          result.add $el
+        continue
+
+      let class = el.attr("class")
+      if "data-expanded-url" in el.attrs:
+        let url = el.attr("data-expanded-url")
+        if url == skipLink: continue
+        elif "u-hidden" in class: result.add "\n"
+        result.add a(shortLink(url), href=url)
+      elif "ashtag" in class:
+        let hash = el.innerText()
+        result.add a(hash, href=("/search?q=" & encodeUrl(hash)))
+      elif "atreply" in class:
+        result.add a(el.innerText(), href=el.attr("href"))
+      elif "Emoji" in class:
+        result.add el.attr("alt")
+    else: discard
 
 proc getQuoteText*(tweet: XmlNode): string =
-  let text = tweet.select(".QuoteTweet-text")
+  parseText(tweet.select(".QuoteTweet-text"))
-  emojify(text)
-  result = stripText(text.innerText())
-  result = stripTwitterUrls(result)
 
 proc getTweetText*(tweet: XmlNode): string =
   let
     quote = tweet.select(".QuoteTweet")
     text = tweet.select(".tweet-text")
     link = text.selectAttr("a.twitter-timeline-link.u-hidden", "data-expanded-url")
-
+  parseText(text, if quote != nil: link else: "")
-  emojify(text)
-  result = stripText(text.innerText())
-
-  if quote != nil and link.len > 0:
-    result = result.replace(link, "")
-
-  result = stripTwitterUrls(result)
 
 proc getTime(tweet: XmlNode): XmlNode =
   tweet.select(".js-short-timestamp")
@@ -87,10 +100,10 @@ proc getUsername*(profile: XmlNode; selector: string): string =
   profile.selectText(selector).strip(chars={'@', ' ', '\n'})
 
 proc getBio*(profile: XmlNode; selector: string; fallback=""): string =
-  var bio = profile.selectText(selector)
+  var bio = profile.select(selector)
-  if bio.len == 0 and fallback.len > 0:
+  if bio == nil and fallback.len > 0:
-    bio = profile.selectText(fallback)
+    bio = profile.select(fallback)
-  stripText(bio)
+  parseText(bio)
 
 proc getLocation*(profile: XmlNode): string =
   let sel = ".ProfileHeaderCard-locationText"

src/views/profile.nim

@@ -25,7 +25,7 @@ proc renderProfileCard*(profile: Profile; prefs: Prefs): VNode =
     tdiv(class="profile-card-extra"):
       if profile.bio.len > 0:
         tdiv(class="profile-bio"):
-          p: verbatim linkifyText(profile.bio, prefs)
+          p: verbatim replaceUrl(profile.bio, prefs)
 
       if profile.location.len > 0:
         tdiv(class="profile-location"):
@@ -39,8 +39,9 @@ proc renderProfileCard*(profile: Profile; prefs: Prefs): VNode =
       if profile.website.len > 0:
         tdiv(class="profile-website"):
           span:
+            let url = replaceUrl(profile.website, prefs)
             icon "link"
-            verbatim linkifyText(profile.website, prefs)
+            a(href=url): text shortLink(url)
 
       tdiv(class="profile-joindate"):
         span(title=getJoinDateFull(profile)):

src/views/rss.nimf

@@ -7,7 +7,7 @@
 #if tweet.pinned: result = "Pinned: "
 #elif tweet.retweet.isSome: result = "RT: "
 #end if
-#result &= xmltree.escape(replaceUrl(tweet.text, prefs))
+#result &= xmltree.escape(replaceUrl(tweet.text, prefs, rss=true))
 #if result.len > 0: return
 #end if
 #if tweet.photos.len > 0:
@@ -20,7 +20,7 @@
 #end proc
 #
 #proc renderRssTweet(tweet: Tweet; prefs: Prefs): string =
-#let text = linkifyText(tweet.text, prefs, rss=true)
+#let text = replaceUrl(tweet.text, prefs, rss=true)
 #if tweet.quote.isSome and get(tweet.quote).available:
 #let quoteLink = hostname & getLink(get(tweet.quote))
 <p>${text}<br><a href="https://${quoteLink}">${quoteLink}</a></p>
@@ -58,7 +58,7 @@
 #end proc
 #
 #proc renderTimelineRss*(timeline: Timeline; profile: Profile): string =
-#let prefs = Prefs(replaceTwitter: hostname)
+#let prefs = Prefs(replaceTwitter: hostname, replaceYoutube: "invidio.us")
 #result = ""
 <?xml version="1.0" encoding="UTF-8"?>
 <rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
@@ -84,7 +84,7 @@
 #end proc
 #
 #proc renderListRss*(tweets: seq[Tweet]; name, list: string): string =
-#let prefs = Prefs(replaceTwitter: hostname)
+#let prefs = Prefs(replaceTwitter: hostname, replaceYoutube: "invidio.us")
 #let link = &"https://{hostname}/{name}/lists/{list}"
 #result = ""
 <?xml version="1.0" encoding="UTF-8"?>
@@ -102,7 +102,7 @@
 #end proc
 #
 #proc renderSearchRss*(tweets: seq[Tweet]; name, param: string): string =
-#let prefs = Prefs(replaceTwitter: hostname)
+#let prefs = Prefs(replaceTwitter: hostname, replaceYoutube: "invidio.us")
 #let link = &"https://{hostname}/search"
 #result = ""
 <?xml version="1.0" encoding="UTF-8"?>

src/views/timeline.nim

@@ -56,7 +56,7 @@ proc renderUser(user: Profile; prefs: Prefs): VNode =
         linkUser(user, class="username")
 
       tdiv(class="tweet-content media-body"):
-        verbatim linkifyText(user.bio, prefs)
+        verbatim replaceUrl(user.bio, prefs)
 
 proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs; path=""): VNode =
   buildHtml(tdiv(class="timeline")):

src/views/tweet.nim

@@ -215,7 +215,7 @@ proc renderQuote(quote: Quote; prefs: Prefs): VNode =
       renderReply(quote)
 
     tdiv(class="quote-text"):
-      verbatim linkifyText(quote.text, prefs)
+      verbatim replaceUrl(quote.text, prefs)
 
     if quote.hasThread:
       a(class="show-thread", href=getLink(quote)):
@@ -248,7 +248,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class="";
         renderReply(tweet)
 
       tdiv(class="tweet-content media-body"):
-        verbatim linkifyText(tweet.text, prefs)
+        verbatim replaceUrl(tweet.text, prefs)
 
       if tweet.quote.isSome:
         renderQuote(tweet.quote.get(), prefs)

tests/test_tweet.py

@@ -51,7 +51,7 @@ link = [
         'old.reddit.com/r/programming…'
     ]],
     ['nim_lang/status/1125887775151140864', [
-        'en.wikipedia.org/wiki/Nim_(p…)'
+        'en.wikipedia.org/wiki/Nim_(p…'
     ]],
     ['hiankun_taioan/status/1086916335215341570', [
         '(hackernoon.com/interview-wit…)'