Add timeline filters

Custom filter menu is still WIP

README.md

@@ -26,7 +26,7 @@ is on implementing missing features.
 ## Todo (roughly in this order)
 
 - Search (images/videos, hashtags, etc.)
-- Hiding retweets, showing replies, etc.
+- Custom timeline filter
 - Media carousel below profile
 - Media-only/gallery view
 - Nitter link previews

public/style.css

@@ -497,6 +497,39 @@ video {
     word-wrap: break-word;
 }
 
+.tab {
+    align-items: center;
+    display: flex;
+    flex-wrap: wrap;
+    list-style: none;
+    margin: 0 0 5px 0;
+    background-color: #161616;
+    padding: 0;
+}
+
+.tab .tab-item {
+    margin-top: 0;
+}
+
+.tab-item {
+    flex: 1 1 0;
+    text-align: center;
+}
+
+.tab .tab-item a.active, .tab .tab-item.active a {
+    border-bottom-color: #ff6c60;
+    color: #ff6c60;
+}
+
+.tab .tab-item a {
+    border-bottom: .1rem solid transparent;
+    color: inherit;
+    display: block;
+    padding: 8px 0;
+    text-decoration: none;
+    font-weight: bold;
+}
+
 .conversation {
     max-width: 580px;
     margin: 0 auto;

src/api.nim

@@ -1,21 +1,20 @@
 import httpclient, asyncdispatch, htmlparser, times
-import sequtils, strutils, strformat, json, xmltree, uri
+import sequtils, strutils, json, xmltree, uri
-import regex
 
-import ./types, ./parser, ./parserutils, ./formatters
+import types, parser, parserutils, formatters, search
 
 const
   agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
   lang = "en-US,en;q=0.9"
   auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
   cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
+  jsonAccept = "application/json, text/javascript, */*; q=0.01"
 
   base = parseUri("https://twitter.com/")
   apiBase = parseUri("https://api.twitter.com/1.1/")
 
-  timelineParams = "?include_available_features=1&include_entities=1&include_new_items_bar=false&reset_error_state=false"
+  timelineUrl = "i/profiles/show/$1/timeline/tweets"
-  showUrl = "i/profiles/show/$1" & timelineParams
+  timelineSearchUrl = "i/search/timeline"
-  timelineUrl = showUrl % "$1/timeline/tweets"
   profilePopupUrl = "i/profiles/popup"
   profileIntentUrl = "intent/user"
   tweetUrl = "status"
@@ -70,7 +69,7 @@ proc getGuestToken(force=false): Future[string] {.async.} =
   tokenUses = 0
 
   let headers = newHttpHeaders({
-    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Accept": jsonAccept,
     "Referer": $base,
     "User-Agent": agent,
     "Authorization": auth
@@ -89,7 +88,7 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
   if tweet.video.isNone(): return
 
   let headers = newHttpHeaders({
-    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Accept": jsonAccept,
     "Referer": $(base / getLink(tweet)),
     "User-Agent": agent,
     "Authorization": auth,
@@ -196,45 +195,9 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
 
   result = parsePopupProfile(html)
 
-proc getTimeline*(username: string; after=""): Future[Timeline] {.async.} =
+proc getTweet*(username, id: string): Future[Conversation] {.async.} =
   let headers = newHttpHeaders({
-    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Accept": jsonAccept,
-    "Referer": $(base / username),
-    "User-Agent": agent,
-    "X-Twitter-Active-User": "yes",
-    "X-Requested-With": "XMLHttpRequest",
-    "Accept-Language": lang
-  })
-
-  var url = timelineUrl % username
-  let cleanAfter = after.replace(re"[^\d]*(\d+)[^\d]*", "$1")
-  if cleanAfter.len > 0:
-    url &= "&max_position=" & cleanAfter
-
-  let json = await fetchJson(base / url, headers)
-  if json == nil: return Timeline()
-
-  result = Timeline(
-    hasMore: json["has_more_items"].to(bool),
-    maxId: json.getOrDefault("max_position").getStr(""),
-    minId: json.getOrDefault("min_position").getStr(""),
-  )
-
-  if json["new_latent_count"].to(int) == 0: return
-  if not json.hasKey("items_html"): return
-
-  let
-    html = parseHtml(json["items_html"].to(string))
-    thread = parseThread(html)
-    vidsFut = getVideos(thread)
-    pollFut = getPolls(thread)
-
-  await all(vidsFut, pollFut)
-  result.tweets = thread.tweets
-
-proc getTweet*(username: string; id: string): Future[Conversation] {.async.} =
-  let headers = newHttpHeaders({
-    "Accept": "application/json, text/javascript, */*; q=0.01",
     "Referer": $base,
     "User-Agent": agent,
     "X-Twitter-Active-User": "yes",
@@ -255,3 +218,72 @@ proc getTweet*(username: string; id: string): Future[Conversation] {.async.} =
   let vidsFut = getConversationVideos(result)
   let pollFut = getConversationPolls(result)
   await all(vidsFut, pollFut)
+
+proc finishTimeline(json: JsonNode; query: Option[Query]): Future[Timeline] {.async.} =
+  if json == nil: return Timeline()
+
+  result = Timeline(
+    hasMore: json["has_more_items"].to(bool),
+    maxId: json.getOrDefault("max_position").getStr(""),
+    minId: json.getOrDefault("min_position").getStr("").cleanPos(),
+  )
+
+  if json["new_latent_count"].to(int) == 0: return
+  if not json.hasKey("items_html"): return
+
+  let
+    html = parseHtml(json["items_html"].to(string))
+    thread = parseThread(html)
+    vidsFut = getVideos(thread)
+    pollFut = getPolls(thread)
+
+  await all(vidsFut, pollFut)
+  result.tweets = thread.tweets
+  result.query = query
+
+proc getTimeline*(username, after: string): Future[Timeline] {.async.} =
+  let headers = newHttpHeaders({
+    "Accept": jsonAccept,
+    "Referer": $(base / username),
+    "User-Agent": agent,
+    "X-Twitter-Active-User": "yes",
+    "X-Requested-With": "XMLHttpRequest",
+    "Accept-Language": lang
+  })
+
+  var params = toSeq({
+    "include_available_features": "1",
+    "include_entities": "1",
+    "include_new_items_bar": "false",
+    "reset_error_state": "false"
+  })
+
+  if after.len > 0:
+    params.add {"max_position": after}
+
+  let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
+  result = await finishTimeline(json, none(Query))
+
+proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline] {.async.} =
+  let headers = newHttpHeaders({
+    "Accept": jsonAccept,
+    "Referer": $(base / ("search?f=tweets&q=from%3A$1&src=typd" % username)),
+    "User-Agent": agent,
+    "X-Requested-With": "XMLHttpRequest",
+    "Authority": "twitter.com",
+    "Accept-Language": lang
+  })
+
+  let params = {
+    "f": "tweets",
+    "vertical": "default",
+    "q": genQueryParam(query),
+    "src": "typd",
+    "include_available_features": "1",
+    "include_entities": "1",
+    "max_position": if after.len > 0: genPos(after) else: "0",
+    "reset_error_state": "false"
+  }
+
+  let json = await fetchJson(base / timelineSearchUrl ? params, headers)
+  result = await finishTimeline(json, some(query))

src/formatters.nim

@@ -1,7 +1,7 @@
 import strutils, strformat, htmlgen, xmltree, times
 import regex
 
-import ./types, ./utils
+import types, utils
 
 from unicode import Rune, `$`
 

src/nitter.nim

@@ -1,26 +1,36 @@
 import asyncdispatch, asyncfile, httpclient, strutils, strformat, uri, os
-import jester
+import jester, regex
 
-import api, utils, types, cache, formatters
+import api, utils, types, cache, formatters, search
 
 include views/"user.nimf"
 include views/"general.nimf"
 
 const cacheDir {.strdefine.} = "/tmp/nitter"
 
-proc showTimeline(name: string; num=""): Future[string] {.async.} =
+proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} =
   let
     username = name.strip(chars={'/'})
     profileFut = getCachedProfile(username)
-    tweetsFut = getTimeline(username, after=num)
+
+  var timelineFut: Future[Timeline]
+  if query.isNone:
+     timelineFut = getTimeline(username, after)
+  else:
+    timelineFut = getTimelineSearch(username, after, get(query))
 
   let profile = await profileFut
   if profile.username.len == 0:
     return ""
 
-  let profileHtml = renderProfile(profile, await tweetsFut, num.len == 0)
+  let profileHtml = renderProfile(profile, await timelineFut, after.len == 0)
   return renderMain(profileHtml, title=pageTitle(profile))
 
+template respTimeline(timeline: typed) =
+  if timeline.len == 0:
+    resp Http404, showError("User \"" & @"name" & "\" not found")
+  resp timeline
+
 routes:
   get "/":
     resp renderMain(renderSearchPanel(), title=pageTitle("Search"))
@@ -28,17 +38,24 @@ routes:
   post "/search":
     if @"query".len == 0:
       resp Http404, showError("Please enter a username.")
-
     redirect("/" & @"query")
 
   get "/@name/?":
     cond '.' notin @"name"
+    respTimeline(await showTimeline(@"name", @"after", none(Query)))
 
-    let timeline = await showTimeline(@"name", @"after")
+  get "/@name/search/?":
-    if timeline.len == 0:
+    cond '.' notin @"name"
-      resp Http404, showError("User \"" & @"name" & "\" not found")
+    let query = initQuery(@"filter", @"sep", @"name")
+    respTimeline(await showTimeline(@"name", @"after", some(query)))
 
-    resp timeline
+  get "/@name/replies":
+    cond '.' notin @"name"
+    respTimeline(await showTimeline(@"name", @"after", some(getReplyQuery(@"name"))))
+
+  get "/@name/media":
+    cond '.' notin @"name"
+    respTimeline(await showTimeline(@"name", @"after", some(getMediaQuery(@"name"))))
 
   get "/@name/status/@id":
     cond '.' notin @"name"

src/parser.nim

@@ -1,6 +1,6 @@
 import xmltree, sequtils, strtabs, strutils, strformat, json
 
-import ./types, ./parserutils, ./formatters
+import types, parserutils, formatters
 
 proc parsePopupProfile*(node: XmlNode): Profile =
   let profile = node.select(".profile-card")

src/parserutils.nim

@@ -1,7 +1,7 @@
 import xmltree, htmlparser, strtabs, strformat, times
 import regex
 
-import ./types, ./formatters, ./api
+import types, formatters, api
 
 from q import nil
 

src/search.nim

@@ -0,0 +1,80 @@
+import asyncdispatch, strutils, strformat, uri, tables
+
+import types
+
+const
+  separators = @["AND", "OR"]
+  validFilters = @[
+    "media", "images", "videos", "native_video", "twimg",
+    "links", "quote", "replies", "mentions",
+    "news", "verified", "safe"
+  ]
+
+# Experimental, this might break in the future
+# Till then, it results in shorter urls
+const
+  posPrefix = "thGAVUV0VFVBa"
+  posSuffix = "EjUAFQAlAFUAFQAA"
+
+proc initQuery*(filter, separator: string; name=""): Query =
+  var sep = separator.strip().toUpper()
+  Query(
+    filter: filter.split(",").filterIt(it in validFilters),
+    sep: if sep in separators: sep else: "AND",
+    fromUser: name,
+    queryType: custom
+  )
+
+proc getMediaQuery*(name: string): Query =
+  Query(
+    filter: @["twimg", "native_video"],
+    sep: "OR",
+    fromUser: name,
+    queryType: media
+  )
+
+proc getReplyQuery*(name: string): Query =
+  Query(fromUser: name, queryType: replies)
+
+proc genQueryParam*(query: Query): string =
+  var filters: seq[string]
+  var param: string
+
+  if query.fromUser.len > 0:
+    param = &"from:{query.fromUser} "
+
+  for f in query.filter:
+    filters.add "filter:" & f
+  for e in query.exclude:
+    filters.add "-filter:" & e
+
+  return strip(param & filters.join(&" {query.sep} "))
+
+proc genQueryUrl*(query: Query): string =
+  result = &"/{query.queryType}?"
+  if query.queryType != custom: return
+
+  var params: seq[string]
+  if query.filter.len > 0:
+    params &= "filter=" & query.filter.join(",")
+  if query.exclude.len > 0:
+    params &= "not=" & query.exclude.join(",")
+  if query.sep.len > 0:
+    params &= "sep=" & query.sep
+  if params.len > 0:
+    result &= params.join("&") & "&"
+
+proc cleanPos*(pos: string): string =
+  pos.multiReplace((posPrefix, ""), (posSuffix, ""))
+
+proc genPos*(pos: string): string =
+  posPrefix & pos & posSuffix
+
+proc tabClass*(timeline: Timeline; tab: string): string =
+  result = '"' & "tab-item"
+  if timeline.query.isNone:
+    if tab == "tweets":
+      result &= " active"
+  elif $timeline.query.get().queryType == tab:
+    result &= " active"
+  result &= '"'

src/types.nim

@@ -31,6 +31,16 @@ db("cache.db", "", "", ""):
         .}: Time
 
 type
+  QueryType* = enum
+    replies, media, custom = "search"
+
+  Query* = object
+    filter*: seq[string]
+    exclude*: seq[string]
+    sep*: string
+    fromUser*: string
+    queryType*: QueryType
+
   VideoType* = enum
     vmap, m3u8, mp4
 
@@ -106,6 +116,7 @@ type
     minId*: string
     maxId*: string
     hasMore*: bool
+    query*: Option[Query]
 
 proc contains*(thread: Thread; tweet: Tweet): bool =
   thread.tweets.anyIt(it.id == tweet.id)

src/views/general.nimf

@@ -45,5 +45,5 @@
 #end proc
 #
 #proc showError*(error: string): string =
-${renderMain(renderError(error), title="Error | Nitter")}
+#renderMain(renderError(error), title="Error | Nitter")
 #end proc

src/views/user.nimf

@@ -1,6 +1,6 @@
 #? stdtmpl(subsChar = '$', metaChar = '#')
 #import xmltree, strutils, uri
-#import ../types, ../formatters, ../utils
+#import ../types, ../formatters, ../utils, ../search
 #include "tweet.nimf"
 #
 #proc renderProfileCard*(profile: Profile): string =
@@ -52,10 +52,13 @@
 #
 #proc renderTimeline*(timeline: Timeline; profile: Profile; beginning: bool): string =
 #var retweets: seq[string]
+#var query = "?"
+#if timeline.query.isSome: query = genQueryUrl(get(timeline.query))
+#end if
 <div id="tweets">
   #if not beginning:
   <div class="show-more status-el">
-    <a href="/${profile.username}">Load newest tweets</a>
+    <a href="/${profile.username}${query.strip(chars={'?'})}">Load newest tweets</a>
   </div>
   #end if
   #
@@ -66,9 +69,9 @@
   ${renderTweet(tweet, "timeline-tweet")}
   #end for
   #
-  #if timeline.hasMore:
+  #if timeline.hasMore or timeline.query.isSome and timeline.tweets.len > 0:
   <div class="show-more">
-    <a href="/${profile.username}?after=${timeline.minId}">Load older tweets</a>
+    <a href="/${profile.username}${query}after=${timeline.minId}">Load older tweets</a>
   </div>
   #elif timeline.tweets.len > 0:
   <div class="timeline-footer">
@@ -96,6 +99,13 @@
     ${renderProfileCard(profile)}
   </div>
   <div class="timeline-tab">
+    #let link = "/" & profile.username
+    <ul class="tab">
+      <li class=${timeline.tabClass("tweets")}><a href="${link}">Tweets</a></li>
+      <li class=${timeline.tabClass("replies")}><a href="${link}/replies">Tweets & Replies</a></li>
+      <li class=${timeline.tabClass("media")}><a href="${link}/media">Media</a></li>
+      #discard "<li class=tab-item><a href=${link}/search>Custom</a></li>"
+    </ul>
     ${renderTimeline(timeline, profile, beginning)}
   </div>
 </div>