Add user search

src/api.nim

@@ -1,2 +1,2 @@
-import api/[media, profile, timeline, tweet, search]
+import api/[profile, timeline, tweet, search, media]
 export profile, timeline, tweet, search, media

src/api/search.nim

@@ -1,32 +1,56 @@
 import httpclient, asyncdispatch, htmlparser
 import sequtils, strutils, json, xmltree, uri
 
-import ".."/[types, parser, parserutils, formatters, search]
+import ".."/[types, parser, parserutils, formatters, query]
 import utils, consts, media, timeline
 
-proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.async.} =
-  let queryParam = genQueryParam(query)
-  let queryEncoded = encodeUrl(queryParam, usePlus=false)
+proc getResult[T](json: JsonNode; query: Query; after: string): Result[T] =
+  Result[T](
+    hasMore: json["has_more_items"].to(bool),
+    maxId: json.getOrDefault("max_position").getStr(""),
+    minId: json.getOrDefault("min_position").getStr("").cleanPos(),
+    query: query.some,
+    beginning: after.len == 0
+  )
 
-  let headers = newHttpHeaders({
+proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
+  let
+    kind = if query.kind == users: "users" else: "tweets"
+    pos = when T is Tweet: genPos(after) else: after
+
+    param = genQueryParam(query)
+    encoded = encodeUrl(param, usePlus=false)
+
+    headers = newHttpHeaders({
       "Accept": jsonAccept,
-    "Referer": $(base / ("search?f=tweets&vertical=default&q=$1&src=typd" % queryEncoded)),
+      "Referer": $(base / ("search?f=$1&q=$2&src=typd" % [kind, encoded])),
       "User-Agent": agent,
       "X-Requested-With": "XMLHttpRequest",
       "Authority": "twitter.com",
       "Accept-Language": lang
     })
 
-  let params = {
-    "f": "tweets",
+    params = {
+      "f": kind,
       "vertical": "default",
-    "q": queryParam,
+      "q": param,
       "src": "typd",
       "include_available_features": "1",
       "include_entities": "1",
-    "max_position": if after.len > 0: genPos(after) else: "0",
+      "max_position": if pos.len > 0: pos else: "0",
       "reset_error_state": "false"
     }
 
   let json = await fetchJson(base / searchUrl ? params, headers)
+  if json == nil: return Result[T]()
+
+  result = getResult[T](json, query, after)
+  if not json.hasKey("items_html"): return
+  let html = parseHtml(json["items_html"].to(string))
+
+  when T is Tweet:
     result = await finishTimeline(json, some(query), after, agent)
+  elif T is Profile:
+    result.hasMore = json["items_html"].to(string) != "\n"
+    for p in html.selectAll(".js-stream-item"):
+      result.content.add parsePopupProfile(p, ".ProfileCard")

src/api/timeline.nim

@@ -1,7 +1,7 @@
 import httpclient, asyncdispatch, htmlparser
 import sequtils, strutils, json, xmltree, uri
 
-import ".."/[types, parser, parserutils, formatters, search]
+import ".."/[types, parser, parserutils, formatters, query]
 import utils, consts, media
 
 proc finishTimeline*(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} =

src/nitter.nim

@@ -5,13 +5,14 @@ import jester
 
 import types, config, prefs
 import views/[general, about]
-import routes/[preferences, timeline, media, rss]
+import routes/[preferences, timeline, media, search, rss]
 
 const configPath {.strdefine.} = "./nitter.conf"
 let cfg = getConfig(configPath)
 
 createPrefRouter(cfg)
 createTimelineRouter(cfg)
+createSearchRouter(cfg)
 createMediaRouter(cfg)
 createRssRouter(cfg)
 
@@ -27,13 +28,9 @@ routes:
   get "/about":
     resp renderMain(renderAbout(), Prefs(), cfg.title)
 
-  post "/search":
-    if @"query".len == 0:
-      resp Http404, showError("Please enter a username.", cfg.title)
-    redirect("/" & @"query")
-
   extend preferences, ""
   extend rss, ""
+  extend search, ""
   extend timeline, ""
   extend media, ""
 

src/parser.nim

@@ -23,14 +23,14 @@ proc parseTimelineProfile*(node: XmlNode): Profile =
 
   result.getProfileStats(node.select(".ProfileNav-list"))
 
-proc parsePopupProfile*(node: XmlNode): Profile =
-  let profile = node.select(".profile-card")
+proc parsePopupProfile*(node: XmlNode; selector=".profile-card"): Profile =
+  let profile = node.select(selector)
   if profile == nil: return
 
   result = Profile(
     fullname:  profile.getName(".fullname"),
     username:  profile.getUsername(".username"),
-    bio:       profile.getBio(".bio"),
+    bio:       profile.getBio(".bio", fallback=".ProfileCard-bio"),
     userpic:   profile.getAvatar(".ProfileCard-avatarImage"),
     verified:  isVerified(profile),
     protected: isProtected(profile),

src/parserutils.nim

@@ -86,8 +86,11 @@ proc getName*(profile: XmlNode; selector: string): string =
 proc getUsername*(profile: XmlNode; selector: string): string =
   profile.selectText(selector).strip(chars={'@', ' ', '\n'})
 
-proc getBio*(profile: XmlNode; selector: string): string =
-  profile.selectText(selector).stripText()
+proc getBio*(profile: XmlNode; selector: string; fallback=""): string =
+  var bio = profile.selectText(selector)
+  if bio.len == 0 and fallback.len > 0:
+    bio = profile.selectText(fallback)
+  stripText(bio)
 
 proc getAvatar*(profile: XmlNode; selector: string): string =
   profile.selectAttr(selector, "src").getUserpic()

src/query.nim

@@ -18,8 +18,7 @@ const
   posPrefix = "thGAVUV0VFVBa"
   posSuffix = "EjUAFQAlAFUAFQAA"
 
-proc initQuery*(filters, includes, excludes, separator, text: string;
-                name=""): Query =
+proc initQuery*(filters, includes, excludes, separator, text: string; name=""): Query =
   var sep = separator.strip().toUpper()
   Query(
     kind: custom,
@@ -50,6 +49,9 @@ proc genQueryParam*(query: Query): string =
   var filters: seq[string]
   var param: string
 
+  if query.kind == users:
+    return query.text
+
   for i, user in query.fromUser:
     param &= &"from:{user} "
     if i < query.fromUser.high:
@@ -67,12 +69,18 @@ proc genQueryParam*(query: Query): string =
     result &= " " & query.text
 
 proc genQueryUrl*(query: Query): string =
-  if query.kind == multi: return "?"
+  if query.fromUser.len > 0:
+    result = "/" & query.fromUser.join(",")
 
-  result = &"/{query.kind}?"
-  if query.kind != custom: return
+  if query.kind == multi:
+    return result & "?"
 
-  var params: seq[string]
+  if query.kind notin {custom, users}:
+    return result & &"/{query.kind}?"
+
+  result &= &"/search?"
+
+  var params = @[&"kind={query.kind}"]
   if query.filters.len > 0:
     params &= "filter=" & query.filters.join(",")
   if query.includes.len > 0:
@@ -84,7 +92,7 @@ proc genQueryUrl*(query: Query): string =
   if query.text.len > 0:
     params &= "text=" & query.text
   if params.len > 0:
-    result &= params.join("&") & "&"
+    result &= params.join("&")
 
 proc cleanPos*(pos: string): string =
   pos.multiReplace((posPrefix, ""), (posSuffix, ""))

src/routes/search.nim

@@ -0,0 +1,30 @@
+import strutils, uri
+
+import jester
+
+import router_utils
+import ".."/[query, types, utils, api, agents]
+import ../views/[general, search]
+
+export search
+
+proc createSearchRouter*(cfg: Config) =
+  router search:
+    get "/search":
+      if @"text".len == 0 or "." in @"text":
+        resp Http404, showError("Please enter a valid username.", cfg.title)
+
+      if "," in @"text":
+        redirect("/" & @"text")
+
+      let query = Query(kind: parseEnum[QueryKind](@"kind", custom), text: @"text")
+
+      case query.kind
+      of users:
+        let users = await getSearch[Profile](query, @"after", getAgent())
+        resp renderMain(renderUserSearch(users, Prefs()), Prefs(), path=getPath())
+      of custom:
+        let tweets = await getSearch[Tweet](query, @"after", getAgent())
+        resp renderMain(renderTweetSearch(tweets, Prefs(), getPath()), Prefs(), path=getPath())
+      else:
+        resp Http404

src/routes/timeline.nim

@@ -3,14 +3,14 @@ import asyncdispatch, strutils, sequtils, uri
 import jester
 
 import router_utils
-import ".."/[api, prefs, types, utils, cache, formatters, agents, search]
-import ../views/[general, profile, timeline, status]
+import ".."/[api, prefs, types, utils, cache, formatters, agents, query]
+import ../views/[general, profile, timeline, status, search]
 
 include "../views/rss.nimf"
 
 export uri, sequtils
 export router_utils
-export api, cache, formatters, search, agents
+export api, cache, formatters, query, agents
 export profile, timeline, status
 
 type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
@@ -33,7 +33,7 @@ proc fetchSingleTimeline*(name, after, agent: string;
       (profile, timeline) = await getProfileAndTimeline(name, agent, after)
       cache(profile)
   else:
-    var timelineFut = getTimelineSearch(get(query), after, agent)
+    var timelineFut = getSearch[Tweet](get(query), after, agent)
     if cachedProfile.isNone:
       profile = await getCachedProfile(name, agent)
     timeline = await timelineFut
@@ -49,7 +49,7 @@ proc fetchMultiTimeline*(names: seq[string]; after, agent: string;
   else:
     q = some(Query(kind: multi, fromUser: names, excludes: @["replies"]))
 
-  return await getTimelineSearch(get(q), after, agent)
+  return await getSearch[Tweet](get(q), after, agent)
 
 proc showTimeline*(name, after: string; query: Option[Query];
                    prefs: Prefs; path, title, rss: string): Future[string] {.async.} =

src/sass/profile/_base.scss

@@ -43,6 +43,14 @@
     top: 50px;
 }
 
+.profile-result .username {
+    margin: 0 !important;
+}
+
+.profile-result .tweet-header {
+    margin-bottom: unset;
+}
+
 @media(max-width: 600px) {
     .profile-tabs {
         width: 100vw;

src/sass/timeline.scss

@@ -20,6 +20,14 @@
     display: block;
     font-weight: bold;
     margin-bottom: 5px;
+
+    input[type="text"] {
+        height: 20px;
+    }
+
+    button {
+        float: unset;
+    }
 }
 
 .tab {

src/types.nim

@@ -57,7 +57,7 @@ dbFromTypes("cache.db", "", "", "", [Profile, Video])
 
 type
   QueryKind* = enum
-    replies, media, multi, custom = "search"
+    replies, media, multi, users, custom
 
   Query* = object
     kind*: QueryKind

src/views/general.nim

@@ -60,13 +60,6 @@ proc renderMain*(body: VNode; prefs: Prefs; title="Nitter"; titleText=""; desc="
 
   result = doctype & $node
 
-proc renderSearch*(): VNode =
-  buildHtml(tdiv(class="panel-container")):
-    tdiv(class="search-panel"):
-      form(`method`="post", action="/search"):
-        input(`type`="text", name="query", autofocus="", placeholder="Enter usernames...")
-        button(`type`="submit"): icon "search"
-
 proc renderError*(error: string): VNode =
   buildHtml(tdiv(class="panel-container")):
     tdiv(class="error-panel"):

src/views/profile.nim

@@ -98,5 +98,5 @@ proc renderProfile*(profile: Profile; timeline: Timeline;
       if profile.protected:
         renderProtected(profile.username)
       else:
-        renderProfileTabs(timeline, profile.username)
+        renderProfileTabs(timeline.query, profile.username)
         renderTimelineTweets(timeline, prefs, path)

src/views/search.nim

@@ -0,0 +1,38 @@
+import strutils, strformat
+import karax/[karaxdsl, vdom, vstyles]
+
+import renderutils, timeline
+import ".."/[types, formatters, query]
+
+proc renderSearch*(): VNode =
+  buildHtml(tdiv(class="panel-container")):
+    tdiv(class="search-panel"):
+      form(`method`="get", action="/search"):
+        verbatim "<input name=\"kind\" style=\"display: none\" value=\"users\"/>"
+        input(`type`="text", name="text", autofocus="", placeholder="Enter username...")
+        button(`type`="submit"): icon "search"
+
+proc renderTweetSearch*(timeline: Timeline; prefs: Prefs; path: string): VNode =
+  let users = get(timeline.query).fromUser
+  buildHtml(tdiv(class="timeline-container")):
+    tdiv(class="timeline-header"):
+      text users.join(" | ")
+
+    renderProfileTabs(timeline.query, users.join(","))
+    renderTimelineTweets(timeline, prefs, path)
+
+proc renderUserSearch*(users: Result[Profile]; prefs: Prefs): VNode =
+  let searchText =
+    if users.query.isSome: get(users.query).text
+    else: ""
+
+  buildHtml(tdiv(class="timeline-container")):
+    tdiv(class="timeline-header"):
+      form(`method`="get", action="/search"):
+        verbatim "<input name=\"kind\" style=\"display: none\" value=\"users\"/>"
+        verbatim "<input type=\"text\" name=\"text\" value=\"$1\"/>" % searchText
+        button(`type`="submit"): icon "search"
+
+    renderSearchTabs(users.query)
+
+    renderTimelineUsers(users, prefs)

src/views/timeline.nim

@@ -12,27 +12,35 @@ proc getQuery(query: Option[Query]): string =
     if result[^1] != '?':
       result &= "&"
 
-proc getTabClass(results: Result; tab: string): string =
+proc getTabClass(query: Option[Query]; tab: string): string =
   var classes = @["tab-item"]
 
-  if results.query.isNone or get(results.query).kind == multi:
+  if query.isNone or get(query).kind == multi:
     if tab == "posts":
       classes.add "active"
-  elif $get(results.query).kind == tab:
+  elif $get(query).kind == tab:
     classes.add "active"
 
   return classes.join(" ")
 
-proc renderProfileTabs*(timeline: Timeline; username: string): VNode =
+proc renderProfileTabs*(query: Option[Query]; username: string): VNode =
   let link = "/" & username
   buildHtml(ul(class="tab")):
-    li(class=timeline.getTabClass("posts")):
+    li(class=query.getTabClass("posts")):
       a(href=link): text "Tweets"
-    li(class=timeline.getTabClass("replies")):
+    li(class=query.getTabClass("replies")):
       a(href=(link & "/replies")): text "Tweets & Replies"
-    li(class=timeline.getTabClass("media")):
+    li(class=query.getTabClass("media")):
       a(href=(link & "/media")): text "Media"
 
+proc renderSearchTabs*(query: Option[Query]): VNode =
+  var q = if query.isSome: get(query) else: Query()
+
+  buildHtml(ul(class="tab")):
+    li(class=query.getTabClass("users")):
+      q.kind = users
+      a(href=genQueryUrl(q)): text "Users"
+
 proc renderNewer(query: Option[Query]): VNode =
   buildHtml(tdiv(class="timeline-item show-more")):
     a(href=(getQuery(query).strip(chars={'?', '&'}))):
@@ -62,6 +70,34 @@ proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode =
 proc threadFilter(it: Tweet; tweetThread: string): bool =
   it.retweet.isNone and it.reply.len == 0 and it.threadId == tweetThread
 
+proc renderUser(user: Profile; prefs: Prefs): VNode =
+  buildHtml(tdiv(class="timeline-item")):
+    tdiv(class="tweet-body profile-result"):
+      tdiv(class="tweet-header"):
+        a(class="tweet-avatar", href=("/" & user.username)):
+          genImg(user.getUserpic("_bigger"), class="avatar")
+
+        tdiv(class="tweet-name-row"):
+          tdiv(class="fullname-and-username"):
+            linkUser(user, class="fullname")
+        linkUser(user, class="username")
+
+      tdiv(class="tweet-content media-body"):
+        verbatim linkifyText(user.bio, prefs)
+
+proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs): VNode =
+  buildHtml(tdiv(class="timeline")):
+    if not results.beginning:
+      renderNewer(results.query)
+
+    if results.content.len > 0:
+      for user in results.content:
+        renderUser(user, prefs)
+      renderOlder(results.query, results.minId)
+    elif results.beginning:
+      renderNoneFound()
+    else:
+      renderNoMore()
 
 proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string): VNode =
   buildHtml(tdiv(class="timeline")):