From fe15003665cae90ec23784a325f9f376f7530b82 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 10 Oct 2019 11:35:48 +0200 Subject: [PATCH 1/4] Remove html from tweet preview --- src/formatters.nim | 7 ++++++- src/views/general.nim | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/formatters.nim b/src/formatters.nim index f40de94..09d4111 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -1,4 +1,5 @@ -import strutils, strformat, sequtils, htmlgen, xmltree, times, uri, tables +import strutils, strformat, sequtils, times, uri, tables +import xmltree, htmlparser, htmlgen import regex import types, utils, query @@ -92,3 +93,7 @@ proc getTwitterLink*(path: string; params: Table[string, string]): string = result = $(parseUri("https://twitter.com") / path ? p) if username.len > 0: result = result.replace("/" & username, "") + +proc getTweetPreview*(text: string): string = + let html = parseHtml(text) + html.innerText() diff --git a/src/views/general.nim b/src/views/general.nim index 0cd70db..dbb4bac 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -57,7 +57,7 @@ proc renderMain*(body: VNode; req: Request; title="Nitter"; titleText=""; desc=" meta(name="viewport", content="width=device-width, initial-scale=1.0") meta(property="og:type", content=`type`) meta(property="og:title", content=titleText) - meta(property="og:description", content=desc) + meta(property="og:description", content=getTweetPreview(desc)) meta(property="og:site_name", content="Nitter") for url in images: From a3303d6beffeeea71f9eea1cf1e657fc41e735e9 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 10 Oct 2019 12:16:50 +0200 Subject: [PATCH 2/4] Prepend "https://" to relative rss links --- src/formatters.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/formatters.nim b/src/formatters.nim index 09d4111..ea61ce0 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -28,7 +28,7 @@ proc replaceUrl*(url: string; prefs: Prefs; rss=false): string = if prefs.replaceTwitter.len > 0: result = result.replace(twRegex, prefs.replaceTwitter) if rss: - result = result.replace("href=\"/", "href=\"" & hostname & "/") + result = result.replace("href=\"/", "href=\"https://" & hostname & "/") proc proxifyVideo*(manifest: string; proxy: bool): string = proc cb(m: RegexMatch; s: string): string = From 4407651ed694abab26d68ec0b4989d79e739ca0d Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 10 Oct 2019 17:47:02 +0200 Subject: [PATCH 3/4] Minor cleanup, fix empty lines before card links --- src/formatters.nim | 8 ++++---- src/parserutils.nim | 3 ++- src/views/general.nim | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/formatters.nim b/src/formatters.nim index ea61ce0..c9aa369 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -16,6 +16,10 @@ const hostname {.strdefine.} = "nitter.net" proc stripText*(text: string): string = text.replace(nbsp, " ").strip() +proc stripHtml*(text: string): string = + let html = parseHtml(text) + html.innerText() + proc shortLink*(text: string; length=28): string = result = text.replace(re"https?://(www.)?", "") if result.len > length: @@ -93,7 +97,3 @@ proc getTwitterLink*(path: string; params: Table[string, string]): string = result = $(parseUri("https://twitter.com") / path ? p) if username.len > 0: result = result.replace("/" & username, "") - -proc getTweetPreview*(text: string): string = - let html = parseHtml(text) - html.innerText() diff --git a/src/parserutils.nim b/src/parserutils.nim index b2b5bec..37e03ca 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -57,7 +57,8 @@ proc parseText*(text: XmlNode; skipLink=""): string = if "data-expanded-url" in el.attrs: let url = el.attr("data-expanded-url") if url == skipLink: continue - elif "u-hidden" in class: result.add "\n" + if "u-hidden" in class and result.len > 0: + result.add "\n" result.add a(shortLink(url), href=url) elif "ashtag" in class: let hash = el.innerText() diff --git a/src/views/general.nim b/src/views/general.nim index dbb4bac..0ff46b1 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -57,7 +57,7 @@ proc renderMain*(body: VNode; req: Request; title="Nitter"; titleText=""; desc=" meta(name="viewport", content="width=device-width, initial-scale=1.0") meta(property="og:type", content=`type`) meta(property="og:title", content=titleText) - meta(property="og:description", content=getTweetPreview(desc)) + meta(property="og:description", content=stripHtml(desc)) meta(property="og:site_name", content="Nitter") for url in images: From 1faf976d7cc9a7d5830796f1e0d3fb749f5259e9 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 10 Oct 2019 18:22:14 +0200 Subject: [PATCH 4/4] Use int for tweet ids for correct thread sorting --- src/api/list.nim | 4 ++-- src/api/media.nim | 4 ++-- src/formatters.nim | 4 ++-- src/parser.nim | 12 ++++++------ src/parserutils.nim | 2 +- src/routes/status.nim | 2 +- src/types.nim | 8 ++++---- src/views/timeline.nim | 10 +++++----- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/api/list.nim b/src/api/list.nim index ba4ca09..ea495ac 100644 --- a/src/api/list.nim +++ b/src/api/list.nim @@ -23,8 +23,8 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {. let last = result.content[^1] result.minId = - if last.retweet.isNone: last.id - else: get(last.retweet).id + if last.retweet.isNone: $last.id + else: $(get(last.retweet).id) proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} = let diff --git a/src/api/media.nim b/src/api/media.nim index e3e5267..a010985 100644 --- a/src/api/media.nim +++ b/src/api/media.nim @@ -68,7 +68,7 @@ proc getVideoFetch(tweet: Tweet; agent, token: string) {.async.} = let headers = genHeaders({"authorization": auth, "x-guest-token": token}, agent, base / getLink(tweet), lang=false) - url = apiBase / (videoUrl % tweet.id) + url = apiBase / (videoUrl % $tweet.id) json = await fetchJson(url, headers) if json == nil: @@ -106,7 +106,7 @@ proc getPoll*(tweet: Tweet; agent: string) {.async.} = let headers = genHeaders(agent, base / getLink(tweet), auth=true) - url = base / (pollUrl % tweet.id) + url = base / (pollUrl % $tweet.id) html = await fetchHtml(url, headers) if html == nil: return diff --git a/src/formatters.nim b/src/formatters.nim index c9aa369..f7b2aa5 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -47,7 +47,7 @@ proc getUserpic*(userpic: string; style=""): string = proc getUserpic*(profile: Profile; style=""): string = getUserPic(profile.userpic, style) -proc getVideoEmbed*(id: string): string = +proc getVideoEmbed*(id: int): string = &"https://twitter.com/i/videos/{id}?embed_source=facebook" proc pageTitle*(profile: Profile): string = @@ -72,7 +72,7 @@ proc getTweetTime*(tweet: Tweet): string = tweet.time.format("h:mm tt' ยท 'MMM d', 'YYYY") proc getLink*(tweet: Tweet | Quote): string = - if tweet.id.len == 0: return + if tweet.id == 0: return &"/{tweet.profile.username}/status/{tweet.id}" proc getTombstone*(text: string): string = diff --git a/src/parser.nim b/src/parser.nim index a0ce710..e822e48 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -72,7 +72,7 @@ proc parseTweetProfile*(profile: XmlNode): Profile = proc parseQuote*(quote: XmlNode): Quote = result = Quote( - id: quote.attr("data-item-id"), + id: parseInt(quote.attr("data-item-id")), text: getQuoteText(quote), reply: parseTweetReply(quote), hasThread: quote.select(".self-thread-context") != nil, @@ -99,8 +99,8 @@ proc parseTweet*(node: XmlNode): Tweet = return Tweet() result = Tweet( - id: tweet.attr("data-item-id"), - threadId: tweet.attr("data-conversation-id"), + id: parseInt(tweet.attr("data-item-id")), + threadId: parseInt(tweet.attr("data-conversation-id")), text: getTweetText(tweet), time: getTimestamp(tweet), shortTime: getShortTime(tweet), @@ -119,7 +119,7 @@ proc parseTweet*(node: XmlNode): Tweet = if by.len > 0: result.retweet = some Retweet( by: stripText(by), - id: tweet.attr("data-retweet-id") + id: parseInt(tweet.attr("data-retweet-id")) ) let quote = tweet.select(".QuoteTweet-innerContainer") @@ -191,7 +191,7 @@ proc parseTimeline*(node: XmlNode; after: string): Timeline = beginning: after.len == 0 ) -proc parseVideo*(node: JsonNode; tweetId: string): Video = +proc parseVideo*(node: JsonNode; tweetId: int): Video = let track = node{"track"} cType = track["contentType"].to(string) @@ -216,7 +216,7 @@ proc parseVideo*(node: JsonNode; tweetId: string): Video = else: echo "Can't parse video of type ", cType - result.videoId = tweetId + result.videoId = $tweetId result.thumb = node["posterImage"].to(string) proc parsePoll*(node: XmlNode): Poll = diff --git a/src/parserutils.nim b/src/parserutils.nim index 37e03ca..90d9c47 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -241,7 +241,7 @@ proc getTweetCard*(tweet: Tweet; node: XmlNode) = if cardDiv == nil: return var card = Card( - id: tweet.id, + id: $tweet.id, query: cardDiv.attr("data-src") ) diff --git a/src/routes/status.nim b/src/routes/status.nim index 0798126..ac028e4 100644 --- a/src/routes/status.nim +++ b/src/routes/status.nim @@ -18,7 +18,7 @@ proc createStatusRouter*(cfg: Config) = let prefs = cookiePrefs() let conversation = await getTweet(@"name", @"id", @"max_position", getAgent()) - if conversation == nil or conversation.tweet.id.len == 0: + if conversation == nil or conversation.tweet.id == 0: var error = "Tweet not found" if conversation != nil and conversation.tweet.tombstone.len > 0: error = conversation.tweet.tombstone diff --git a/src/types.nim b/src/types.nim index 3fb362e..8ea7873 100644 --- a/src/types.nim +++ b/src/types.nim @@ -115,7 +115,7 @@ type video*: Option[Video] Quote* = object - id*: string + id*: int profile*: Profile text*: string reply*: seq[string] @@ -128,7 +128,7 @@ type Retweet* = object by*: string - id*: string + id*: int TweetStats* = object replies*: string @@ -136,8 +136,8 @@ type likes*: string Tweet* = ref object - id*: string - threadId*: string + id*: int + threadId*: int profile*: Profile text*: string time*: Time diff --git a/src/views/timeline.nim b/src/views/timeline.nim index ee8897c..adf630f 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -34,13 +34,13 @@ proc renderNoneFound(): VNode = proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode = buildHtml(tdiv(class="thread-line")): - for i, threadTweet in thread.sortedByIt(it.time): + for i, threadTweet in thread.sortedByIt(it.id): let show = i == thread.len and thread[0].id != threadTweet.threadId renderTweet(threadTweet, prefs, path, class="thread", index=i, total=thread.high, showThread=show) -proc threadFilter(it: Tweet; tweetThread: string): bool = - it.retweet.isNone and it.reply.len == 0 and it.threadId == tweetThread +proc threadFilter(it: Tweet; thread: int): bool = + it.retweet.isNone and it.reply.len == 0 and it.threadId == thread proc renderUser(user: Profile; prefs: Prefs): VNode = buildHtml(tdiv(class="timeline-item")): @@ -81,8 +81,8 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string): if results.content.len == 0: renderNoneFound() else: - var threads: seq[string] - var retweets: seq[string] + var threads: seq[int] + var retweets: seq[int] for tweet in results.content: if tweet.threadId in threads or tweet.id in retweets: continue let thread = results.content.filterIt(threadFilter(it, tweet.threadId))