Merge pull request #40 from zedeus/rss

Add timeline RSS support
2019-09-17 06:07:13 +02:00 · 2019-09-17 06:07:13 +02:00 · da0ad25c2c
parent 4ae5d4fda3 2677782286
commit da0ad25c2c
14 changed files with 192 additions and 54 deletions
--- a/README.md
+++ b/README.md
@ -10,6 +10,7 @@ Inspired by the [invidio.us](https://github.com/omarroth/invidious) project.
 - AGPLv3 licensed, no proprietary instances permitted
 - Dark theme
 - Lightweight (for [@nim_lang](https://twitter.com/nim_lang), 36KB vs 580KB from twitter.com)
+- Native RSS feeds

 ## Installation

@ -23,11 +24,12 @@ It is possible to install Nim system-wide or in the user directory you create be
 # su nitter
 $ git clone https://github.com/zedeus/nitter
 $ cd nitter
-$ nimble build -d:release
+$ nimble build -d:release -d:hostname="..."
 $ nimble scss
 $ mkdir ./tmp
 ```

+Change `-d:hostname="..."` to your instance's domain, eg. `-d:hostname:"nitter.net"`.
 Set your port and page title in `nitter.conf`, then run Nitter by executing `./nitter`.
 You should run Nitter behind a reverse proxy such as nginx or Apache for better
 security.
--- a/public/css/fontello.css
+++ b/public/css/fontello.css
@ -1,11 +1,11 @@
@font-face {
  font-family: 'fontello';
-  src: url('/fonts/fontello.eot?85902121');
-  src: url('/fonts/fontello.eot?85902121#iefix') format('embedded-opentype'),
-       url('/fonts/fontello.woff2?85902121') format('woff2'),
-       url('/fonts/fontello.woff?85902121') format('woff'),
-       url('/fonts/fontello.ttf?85902121') format('truetype'),
-       url('/fonts/fontello.svg?85902121#fontello') format('svg');
+  src: url('/fonts/fontello.eot?33844470');
+  src: url('/fonts/fontello.eot?33844470#iefix') format('embedded-opentype'),
+       url('/fonts/fontello.woff2?33844470') format('woff2'),
+       url('/fonts/fontello.woff?33844470') format('woff'),
+       url('/fonts/fontello.ttf?33844470') format('truetype'),
+       url('/fonts/fontello.svg?33844470#fontello') format('svg');
  font-weight: normal;
  font-style: normal;
 }
@ -50,4 +50,5 @@
 .icon-search:before { content: '\e80e'; } /* '' */
 .icon-pin:before { content: '\e80f'; } /* '' */
 .icon-cog:before { content: '\e812'; } /* '' */
+.icon-rss:before { content: '\f143'; } /* '' */
 .icon-thumbs-up:before { content: '\f164'; } /* '' */
--- a/public/fonts/fontello.eot
+++ b/public/fonts/fontello.eot
--- a/public/fonts/fontello.svg
+++ b/public/fonts/fontello.svg
@ -40,6 +40,8 @@

 <glyph glyph-name="cog" unicode="&#xe812;" d="M0 272l0 156 150 16q14 45 38 88l-96 117 109 109 117-95q41 23 88 37l16 150 156 0 16-150q45-14 88-37l117 95 109-109-96-117q24-43 38-88l150-16 0-156-150-16q-14-47-38-88l96-117-109-109-117 96q-43-24-88-38l-16-150-156 0-16 150q-47 14-88 38l-117-96-109 109 96 117q-24 41-38 88z m355 78q0-60 42-102t103-42 103 42 42 102-42 103-103 42-103-42-42-103z" horiz-adv-x="1000" />

+<glyph glyph-name="rss-squared" unicode="&#xf143;" d="M286 136q0 29-21 50t-51 21-50-21-21-50 21-51 50-21 51 21 21 51z m196-53q-8 130-99 222t-221 98q-8 1-14-5t-5-13v-71q0-7 5-12t12-6q86-6 147-68t67-147q1-7 6-12t12-5h72q7 0 13 6t5 13z m214 0q-3 86-31 166t-78 145-115 114-145 78-166 31q-7 1-13-5-5-5-5-13v-71q0-7 5-12t12-6q114-4 211-62t156-155 62-211q0-8 5-13t13-5h71q7 0 13 6 6 5 5 13z m161 535v-536q0-66-47-113t-114-48h-535q-67 0-114 48t-47 113v536q0 66 47 113t114 48h535q67 0 114-48t47-113z" horiz-adv-x="857.1" />
+
 <glyph glyph-name="thumbs-up" unicode="&#xf164;" d="M143 100q0 15-11 25t-25 11q-15 0-25-11t-11-25q0-15 11-25t25-11q15 0 25 11t11 25z m89 286v-357q0-15-10-25t-26-11h-160q-15 0-25 11t-11 25v357q0 14 11 25t25 10h160q15 0 26-10t10-25z m661 0q0-48-31-83 9-25 9-43 1-42-24-76 9-31 0-66-9-31-31-52 5-62-27-101-36-43-110-44h-72q-37 0-80 9t-68 16-67 22q-69 24-88 25-15 0-25 11t-11 25v357q0 14 10 25t24 11q13 1 42 33t57 67q38 49 56 67 10 10 17 27t10 27 8 34q4 22 7 34t11 29 19 28q10 11 25 11 25 0 46-6t33-15 22-22 14-25 7-28 2-25 1-22q0-21-6-43t-10-33-16-31q-1-4-5-10t-6-13-5-13h155q43 0 75-32t32-75z" horiz-adv-x="928.6" />
 </font>
 </defs>
--- a/public/fonts/fontello.ttf
+++ b/public/fonts/fontello.ttf
--- a/public/fonts/fontello.woff
+++ b/public/fonts/fontello.woff
--- a/public/fonts/fontello.woff2
+++ b/public/fonts/fontello.woff2
--- a/src/formatters.nim
+++ b/src/formatters.nim
@ -15,6 +15,8 @@ const
  twRegex = re"(www.|mobile.)?twitter.com"
  nbsp = $Rune(0x000A0)

+const hostname {.strdefine.} = "nitter.net"
+
 proc stripText*(text: string): string =
  text.replace(nbsp, " ").strip()

@ -23,12 +25,16 @@ proc shortLink*(text: string; length=28): string =
  if result.len > length:
    result = result[0 ..< length] & "…"

-proc toLink*(url, text: string; class="timeline-link"): string =
-  a(text, class=class, href=url)
+proc toLink*(url, text: string): string =
+  a(text, href=url)
+
+proc reUrlToShortLink*(m: RegexMatch; s: string): string =
+  let url = s[m.group(0)[0]]
+  toLink(url, shortLink(url))

 proc reUrlToLink*(m: RegexMatch; s: string): string =
  let url = s[m.group(0)[0]]
-  toLink(url, shortLink(url))
+  toLink(url, url.replace(re"https?://(www.)?", ""))

 proc reEmailToLink*(m: RegexMatch; s: string): string =
  let url = s[m.group(0)[0]]
@ -48,19 +54,9 @@ proc reUsernameToLink*(m: RegexMatch; s: string): string =

  pretext & toLink("/" & username, "@" & username)

-proc linkifyText*(text: string; prefs: Prefs): string =
-  result = xmltree.escape(stripText(text))
-  result = result.replace(ellipsisRegex, "")
-  result = result.replace(emailRegex, reEmailToLink)
-  result = result.replace(urlRegex, reUrlToLink)
-  result = result.replace(usernameRegex, reUsernameToLink)
-  result = result.replace(re"([^\s\(\n%])<a", "$1 <a")
-  result = result.replace(re"</a>\s+([;.,!\)'%]|&apos;)", "</a>$1")
-  result = result.replace(re"^\. <a", ".<a")
-  if prefs.replaceYouTube.len > 0:
-    result = result.replace(ytRegex, prefs.replaceYouTube)
-  if prefs.replaceTwitter.len > 0:
-    result = result.replace(twRegex, prefs.replaceTwitter)
+proc reUsernameToFullLink*(m: RegexMatch; s: string): string =
+  result = reUsernameToLink(m, s)
+  result = result.replace("href=\"/", &"href=\"https://{hostname}/")

 proc replaceUrl*(url: string; prefs: Prefs): string =
  result = url
@ -69,6 +65,21 @@ proc replaceUrl*(url: string; prefs: Prefs): string =
  if prefs.replaceTwitter.len > 0:
    result = result.replace(twRegex, prefs.replaceTwitter)

+proc linkifyText*(text: string; prefs: Prefs; rss=false): string =
+  result = xmltree.escape(stripText(text))
+  result = result.replace(ellipsisRegex, "")
+  result = result.replace(emailRegex, reEmailToLink)
+  if rss:
+    result = result.replace(urlRegex, reUrlToLink)
+    result = result.replace(usernameRegex, reUsernameToFullLink)
+  else:
+    result = result.replace(urlRegex, reUrlToShortLink)
+    result = result.replace(usernameRegex, reUsernameToLink)
+  result = result.replace(re"([^\s\(\n%])<a", "$1 <a")
+  result = result.replace(re"</a>\s+([;.,!\)'%]|&apos;)", "</a>$1")
+  result = result.replace(re"^\. <a", ".<a")
+  result = result.replaceUrl(prefs)
+
 proc stripTwitterUrls*(text: string): string =
  result = text
  result = result.replace(picRegex, "")
@ -105,6 +116,9 @@ proc getJoinDateFull*(profile: Profile): string =
 proc getTime*(tweet: Tweet): string =
  tweet.time.format("d/M/yyyy', 'HH:mm:ss")

+proc getRfc822Time*(tweet: Tweet): string =
+  tweet.time.format("ddd', 'd MMM yyyy HH:mm:ss 'GMT'")
+
 proc getLink*(tweet: Tweet | Quote): string =
  &"/{tweet.profile.username}/status/{tweet.id}"

--- a/src/nitter.nim
+++ b/src/nitter.nim
@ -5,7 +5,7 @@ import jester

 import types, config, prefs
 import views/[general, about]
-import routes/[preferences, timeline, media]
+import routes/[preferences, timeline, media, rss]

 const configPath {.strdefine.} = "./nitter.conf"
 let cfg = getConfig(configPath)
@ -13,6 +13,7 @@ let cfg = getConfig(configPath)
 createPrefRouter(cfg)
 createTimelineRouter(cfg)
 createMediaRouter(cfg)
+createRssRouter(cfg)

 settings:
  port = Port(cfg.port)
@ -32,6 +33,7 @@ routes:
    redirect("/" & @"query")

  extend preferences, ""
+  extend rss, ""
  extend timeline, ""
  extend media, ""

--- a/src/routes/rss.nim
+++ b/src/routes/rss.nim
@ -0,0 +1,32 @@
+import asyncdispatch, strutils
+
+import jester
+
+import router_utils, timeline
+import ".."/[cache, agents, search]
+import ../views/general
+
+include "../views/rss.nimf"
+
+proc showRss*(name: string; query: Option[Query]): Future[string] {.async.} =
+  let (profile, timeline, _) = await fetchSingleTimeline(name, "", getAgent(), query)
+  return renderTimelineRss(timeline.content, profile)
+
+template respRss*(rss: typed) =
+  if rss.len == 0:
+    resp Http404, showError("User \"" & @"name" & "\" not found", cfg.title)
+  resp rss, "application/rss+xml;charset=utf-8"
+
+proc createRssRouter*(cfg: Config) =
+  router rss:
+    get "/@name/rss":
+      cond '.' notin @"name"
+      respRss(await showRss(@"name", none(Query)))
+
+    get "/@name/replies/rss":
+      cond '.' notin @"name"
+      respRss(await showRss(@"name", some(getReplyQuery(@"name"))))
+
+    get "/@name/media/rss":
+      cond '.' notin @"name"
+      respRss(await showRss(@"name", some(getMediaQuery(@"name"))))
--- a/src/routes/timeline.nim
+++ b/src/routes/timeline.nim
@ -6,13 +6,17 @@ import router_utils
 import ".."/[api, prefs, types, utils, cache, formatters, agents, search]
 import ../views/[general, profile, timeline, status]

+include "../views/rss.nimf"
+
 export uri, sequtils
 export router_utils
 export api, cache, formatters, search, agents
 export profile, timeline, status

-proc showSingleTimeline(name, after, agent: string; query: Option[Query];
-                        prefs: Prefs; path, title: string): Future[string] {.async.} =
+type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
+
+proc fetchSingleTimeline*(name, after, agent: string;
+                          query: Option[Query]): Future[ProfileTimeline] {.async.} =
  let railFut = getPhotoRail(name, agent)

  var timeline: Timeline
@ -34,35 +38,34 @@ proc showSingleTimeline(name, after, agent: string; query: Option[Query];
      profile = await getCachedProfile(name, agent)
    timeline = await timelineFut

-  if profile.username.len == 0:
-    return ""
+  if profile.username.len == 0: return
+  return (profile, timeline, await railFut)

-  let profileHtml = renderProfile(profile, timeline, await railFut, prefs, path)
-  return renderMain(profileHtml, prefs, title, pageTitle(profile),
-                    pageDesc(profile), path)
-
-proc showMultiTimeline(names: seq[string]; after, agent: string; query: Option[Query];
-                       prefs: Prefs; path, title: string): Future[string] {.async.} =
+proc fetchMultiTimeline*(names: seq[string]; after, agent: string;
+                         query: Option[Query]): Future[Timeline] {.async.} =
  var q = query
  if q.isSome:
    get(q).fromUser = names
  else:
    q = some(Query(kind: multi, fromUser: names, excludes: @["replies"]))

-  var timeline = renderMulti(await getTimelineSearch(get(q), after, agent),
-                             names.join(","), prefs, path)
-
-  return renderMain(timeline, prefs, title, "Multi")
+  return await getTimelineSearch(get(q), after, agent)

 proc showTimeline*(name, after: string; query: Option[Query];
-                  prefs: Prefs; path, title: string): Future[string] {.async.} =
+                   prefs: Prefs; path, title, rss: string): Future[string] {.async.} =
  let agent = getAgent()
  let names = name.strip(chars={'/'}).split(",").filterIt(it.len > 0)

  if names.len == 1:
-    return await showSingleTimeline(names[0], after, agent, query, prefs, path, title)
+    let (p, t, r) = await fetchSingleTimeline(names[0], after, agent, query)
+    if p.username.len == 0: return
+    let pHtml = renderProfile(p, t, r, prefs, path)
+    return renderMain(pHtml, prefs, title, pageTitle(p), pageDesc(p), path, rss=rss)
  else:
-    return await showMultiTimeline(names, after, agent, query, prefs, path, title)
+    let
+      timeline = await fetchMultiTimeline(names, after, agent, query)
+      html = renderMulti(timeline, names.join(","), prefs, path)
+    return renderMain(html, prefs, title, "Multi")

 template respTimeline*(timeline: typed) =
  if timeline.len == 0:
@ -75,24 +78,27 @@ proc createTimelineRouter*(cfg: Config) =
  router timeline:
    get "/@name/?":
      cond '.' notin @"name"
-      respTimeline(await showTimeline(@"name", @"after", none(Query),
-                                      cookiePrefs(), getPath(), cfg.title))
+      let rss = "/$1/rss" % @"name"
+      respTimeline(await showTimeline(@"name", @"after", none(Query), cookiePrefs(),
+                                      getPath(), cfg.title, rss))

    get "/@name/search":
      cond '.' notin @"name"
      let query = initQuery(@"filter", @"include", @"not", @"sep", @"name")
      respTimeline(await showTimeline(@"name", @"after", some(query),
-                                      cookiePrefs(), getPath(), cfg.title))
+                                      cookiePrefs(), getPath(), cfg.title, ""))

    get "/@name/replies":
      cond '.' notin @"name"
+      let rss = "/$1/replies/rss" % @"name"
      respTimeline(await showTimeline(@"name", @"after", some(getReplyQuery(@"name")),
-                                      cookiePrefs(), getPath(), cfg.title))
+                                      cookiePrefs(), getPath(), cfg.title, rss))

    get "/@name/media":
      cond '.' notin @"name"
+      let rss = "/$1/media/rss" % @"name"
      respTimeline(await showTimeline(@"name", @"after", some(getMediaQuery(@"name")),
-                                      cookiePrefs(), getPath(), cfg.title))
+                                      cookiePrefs(), getPath(), cfg.title, rss))

    get "/@name/status/@id":
      cond '.' notin @"name"
@ -121,7 +127,8 @@ proc createTimelineRouter*(cfg: Config) =
        resp renderMain(html, prefs, cfg.title, title, desc, path, images = @[thumb],
                        `type`="video", video=vidUrl)
      else:
-        resp renderMain(html, prefs, cfg.title, title, desc, path, images=conversation.tweet.photos)
+        resp renderMain(html, prefs, cfg.title, title, desc, path,
+                        images=conversation.tweet.photos, `type`="photo")

    get "/i/web/status/@id":
      redirect("/i/status/" & @"id")
--- a/src/sass/tweet/thread.scss
+++ b/src/sass/tweet/thread.scss
@ -50,7 +50,7 @@
        width: 5px;
        top: 2px;
        margin-bottom: 0;
-        margin-left: -5px;
+        margin-left: -2.5px;
    }
 }

--- a/src/views/general.nim
+++ b/src/views/general.nim
@ -5,7 +5,7 @@ import ../utils, ../types

 const doctype = "<!DOCTYPE html>\n"

-proc renderNavbar*(title, path: string): VNode =
+proc renderNavbar*(title, path, rss: string): VNode =
  buildHtml(nav(id="nav", class="nav-bar container")):
    tdiv(class="inner-nav"):
      tdiv(class="item"):
@ -14,16 +14,21 @@ proc renderNavbar*(title, path: string): VNode =
      a(href="/"): img(class="site-logo", src="/logo.png")

      tdiv(class="item right"):
+        if rss.len > 0:
+          icon "rss", title="RSS Feed", href=rss
        icon "info-circled", title="About", href="/about"
        iconReferer "cog", "/settings", path, title="Preferences"

-proc renderMain*(body: VNode; prefs: Prefs; title="Nitter"; titleText=""; desc="";
-                 path="/"; `type`="article"; video=""; images: seq[string] = @[]): string =
+proc renderMain*(body: VNode; prefs: Prefs; title="Nitter"; titleText=""; desc=""; path="/";
+                 rss=""; `type`="article"; video=""; images: seq[string] = @[]): string =
  let node = buildHtml(html(lang="en")):
    head:
      link(rel="stylesheet", `type`="text/css", href="/css/style.css")
      link(rel="stylesheet", `type`="text/css", href="/css/fontello.css")

+      if rss.len > 0:
+        link(rel="alternate", `type`="application/rss+xml", href=rss, title="RSS feed")
+
      if prefs.hlsPlayback:
        script(src="/js/hls.light.min.js")
        script(src="/js/hlsPlayback.js")
@ -38,7 +43,7 @@ proc renderMain*(body: VNode; prefs: Prefs; title="Nitter"; titleText=""; desc="
      meta(property="og:type", content=`type`)
      meta(property="og:title", content=titleText)
      meta(property="og:description", content=desc)
-      meta(property="og:site_name", content="Twitter")
+      meta(property="og:site_name", content="Nitter")

      for url in images:
        meta(property="og:image", content=getPicUrl(url))
@ -48,7 +53,7 @@ proc renderMain*(body: VNode; prefs: Prefs; title="Nitter"; titleText=""; desc="
        meta(property="og:video:secure_url", content=video)

    body:
-      renderNavbar(title, path)
+      renderNavbar(title, path, rss)

      tdiv(id="content", class="container"):
        body
--- a/src/views/rss.nimf
+++ b/src/views/rss.nimf
@ -0,0 +1,73 @@
+#? stdtmpl(subsChar = '$', metaChad = '#')
+#import strutils, xmltree, strformat
+#import ../types, ../utils, ../formatters
+#const hostname {.strdefine.} = "nitter.net"
+#
+#proc renderRssTweet(tweet: Tweet; prefs: Prefs): string =
+#let text = linkifyText(tweet.text, prefs, rss=true)
+#if tweet.quote.isSome and get(tweet.quote).available:
+#let quoteLink = hostname & getLink(get(tweet.quote))
+<p>${text}<br><a href="https://${quoteLink}">${quoteLink}</a></p>
+#else:
+<p>${text}</p>
+#end if
+#if tweet.photos.len > 0:
+<img src="https://${hostname}${getPicUrl(tweet.photos[0])}" width="250" />
+#elif tweet.video.isSome:
+<img src="https://${hostname}${getPicUrl(get(tweet.video).thumb)}" width="250" />
+#elif tweet.gif.isSome:
+#let thumb = &"https://{hostname}{getPicUrl(get(tweet.gif).thumb)}"
+#let url = &"https://{hostname}{getGifUrl(get(tweet.gif).url)}"
+<video poster="${thumb}" autoplay muted loop width="250">
+  <source src="${url}" type="video/mp4"</source></video>
+#end if
+#end proc
+#
+#proc getTitle(tweet: Tweet; prefs: Prefs): string =
+#if tweet.pinned: result = "Pinned: "
+#elif tweet.retweet.isSome: result = "RT: "
+#end if
+#result &= xmltree.escape(replaceUrl(tweet.text, prefs))
+#if result.len > 0: return
+#end if
+#if tweet.photos.len > 0:
+#  result &= "Image"
+#elif tweet.video.isSome:
+#  result &= "Video"
+#elif tweet.gif.isSome:
+#  result &= "Gif"
+#end if
+#end proc
+#
+#proc renderTimelineRss*(tweets: seq[Tweet]; profile: Profile): string =
+#let prefs = Prefs(replaceTwitter: hostname)
+#result = ""
+<?xml version="1.0" encoding="UTF-8"?>
+<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
+  <channel>
+    <atom:link href="https://${hostname}/${profile.username}/rss" rel="self" type="application/rss+xml" />
+    <title>${profile.fullname} / @${profile.username}</title>
+    <link>https://${hostname}/${profile.username}</link>
+    <description>Twitter feed for: @${profile.username}. Generated by ${hostname}</description>
+    <language>en-us</language>
+    <ttl>40</ttl>
+    <image>
+      <title>${profile.fullname} / @${profile.username}</title>
+      <link>https://${hostname}/${profile.username}</link>
+      <url>https://${hostname}${getPicUrl(profile.getUserPic(style="_400x400"))}</url>
+      <width>128</width>
+      <height>128</height>
+    </image>
+    #for tweet in tweets:
+    <item>
+      <title>${getTitle(tweet, prefs)}</title>
+      <dc:creator>@${tweet.profile.username}</dc:creator>
+      <description><![CDATA[${renderRssTweet(tweet, prefs).strip(chars={'\n'})}]]></description>
+      <pubDate>${getRfc822Time(tweet)}</pubDate>
+      <guid>https://${hostname}${getLink(tweet)}</guid>
+      <link>https://${hostname}${getLink(tweet)}</link>
+    </item>
+    #end for
+  </channel>
+</rss>
+#end proc