From 30bab22dae6a9910be5ff12e222e892cfd5027c0 Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 13 Sep 2019 22:24:58 +0200 Subject: [PATCH] Add user search --- src/api.nim | 2 +- src/api/search.nim | 70 +++++++++++++++++++++++------------ src/api/timeline.nim | 2 +- src/nitter.nim | 9 ++--- src/parser.nim | 6 +-- src/parserutils.nim | 7 +++- src/{search.nim => query.nim} | 22 +++++++---- src/routes/search.nim | 30 +++++++++++++++ src/routes/timeline.nim | 10 ++--- src/sass/profile/_base.scss | 8 ++++ src/sass/timeline.scss | 8 ++++ src/types.nim | 2 +- src/views/general.nim | 7 ---- src/views/profile.nim | 2 +- src/views/search.nim | 38 +++++++++++++++++++ src/views/timeline.nim | 50 +++++++++++++++++++++---- 16 files changed, 209 insertions(+), 64 deletions(-) rename src/{search.nim => query.nim} (85%) create mode 100644 src/routes/search.nim create mode 100644 src/views/search.nim diff --git a/src/api.nim b/src/api.nim index 2cffc66..3b45b0f 100644 --- a/src/api.nim +++ b/src/api.nim @@ -1,2 +1,2 @@ -import api/[media, profile, timeline, tweet, search] +import api/[profile, timeline, tweet, search, media] export profile, timeline, tweet, search, media diff --git a/src/api/search.nim b/src/api/search.nim index f07a864..21cf9b8 100644 --- a/src/api/search.nim +++ b/src/api/search.nim @@ -1,32 +1,56 @@ import httpclient, asyncdispatch, htmlparser import sequtils, strutils, json, xmltree, uri -import ".."/[types, parser, parserutils, formatters, search] +import ".."/[types, parser, parserutils, formatters, query] import utils, consts, media, timeline -proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.async.} = - let queryParam = genQueryParam(query) - let queryEncoded = encodeUrl(queryParam, usePlus=false) +proc getResult[T](json: JsonNode; query: Query; after: string): Result[T] = + Result[T]( + hasMore: json["has_more_items"].to(bool), + maxId: json.getOrDefault("max_position").getStr(""), + minId: json.getOrDefault("min_position").getStr("").cleanPos(), + query: query.some, + beginning: after.len == 0 + ) - let headers = newHttpHeaders({ - "Accept": jsonAccept, - "Referer": $(base / ("search?f=tweets&vertical=default&q=$1&src=typd" % queryEncoded)), - "User-Agent": agent, - "X-Requested-With": "XMLHttpRequest", - "Authority": "twitter.com", - "Accept-Language": lang - }) +proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} = + let + kind = if query.kind == users: "users" else: "tweets" + pos = when T is Tweet: genPos(after) else: after - let params = { - "f": "tweets", - "vertical": "default", - "q": queryParam, - "src": "typd", - "include_available_features": "1", - "include_entities": "1", - "max_position": if after.len > 0: genPos(after) else: "0", - "reset_error_state": "false" - } + param = genQueryParam(query) + encoded = encodeUrl(param, usePlus=false) + + headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $(base / ("search?f=$1&q=$2&src=typd" % [kind, encoded])), + "User-Agent": agent, + "X-Requested-With": "XMLHttpRequest", + "Authority": "twitter.com", + "Accept-Language": lang + }) + + params = { + "f": kind, + "vertical": "default", + "q": param, + "src": "typd", + "include_available_features": "1", + "include_entities": "1", + "max_position": if pos.len > 0: pos else: "0", + "reset_error_state": "false" + } let json = await fetchJson(base / searchUrl ? params, headers) - result = await finishTimeline(json, some(query), after, agent) + if json == nil: return Result[T]() + + result = getResult[T](json, query, after) + if not json.hasKey("items_html"): return + let html = parseHtml(json["items_html"].to(string)) + + when T is Tweet: + result = await finishTimeline(json, some(query), after, agent) + elif T is Profile: + result.hasMore = json["items_html"].to(string) != "\n" + for p in html.selectAll(".js-stream-item"): + result.content.add parsePopupProfile(p, ".ProfileCard") diff --git a/src/api/timeline.nim b/src/api/timeline.nim index e917152..1f56cb7 100644 --- a/src/api/timeline.nim +++ b/src/api/timeline.nim @@ -1,7 +1,7 @@ import httpclient, asyncdispatch, htmlparser import sequtils, strutils, json, xmltree, uri -import ".."/[types, parser, parserutils, formatters, search] +import ".."/[types, parser, parserutils, formatters, query] import utils, consts, media proc finishTimeline*(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} = diff --git a/src/nitter.nim b/src/nitter.nim index 7a4f746..41dc3b9 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -5,13 +5,14 @@ import jester import types, config, prefs import views/[general, about] -import routes/[preferences, timeline, media, rss] +import routes/[preferences, timeline, media, search, rss] const configPath {.strdefine.} = "./nitter.conf" let cfg = getConfig(configPath) createPrefRouter(cfg) createTimelineRouter(cfg) +createSearchRouter(cfg) createMediaRouter(cfg) createRssRouter(cfg) @@ -27,13 +28,9 @@ routes: get "/about": resp renderMain(renderAbout(), Prefs(), cfg.title) - post "/search": - if @"query".len == 0: - resp Http404, showError("Please enter a username.", cfg.title) - redirect("/" & @"query") - extend preferences, "" extend rss, "" + extend search, "" extend timeline, "" extend media, "" diff --git a/src/parser.nim b/src/parser.nim index 17956df..0285c10 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -23,14 +23,14 @@ proc parseTimelineProfile*(node: XmlNode): Profile = result.getProfileStats(node.select(".ProfileNav-list")) -proc parsePopupProfile*(node: XmlNode): Profile = - let profile = node.select(".profile-card") +proc parsePopupProfile*(node: XmlNode; selector=".profile-card"): Profile = + let profile = node.select(selector) if profile == nil: return result = Profile( fullname: profile.getName(".fullname"), username: profile.getUsername(".username"), - bio: profile.getBio(".bio"), + bio: profile.getBio(".bio", fallback=".ProfileCard-bio"), userpic: profile.getAvatar(".ProfileCard-avatarImage"), verified: isVerified(profile), protected: isProtected(profile), diff --git a/src/parserutils.nim b/src/parserutils.nim index 6de7bdf..90ebd17 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -86,8 +86,11 @@ proc getName*(profile: XmlNode; selector: string): string = proc getUsername*(profile: XmlNode; selector: string): string = profile.selectText(selector).strip(chars={'@', ' ', '\n'}) -proc getBio*(profile: XmlNode; selector: string): string = - profile.selectText(selector).stripText() +proc getBio*(profile: XmlNode; selector: string; fallback=""): string = + var bio = profile.selectText(selector) + if bio.len == 0 and fallback.len > 0: + bio = profile.selectText(fallback) + stripText(bio) proc getAvatar*(profile: XmlNode; selector: string): string = profile.selectAttr(selector, "src").getUserpic() diff --git a/src/search.nim b/src/query.nim similarity index 85% rename from src/search.nim rename to src/query.nim index c7b118b..2baa3d9 100644 --- a/src/search.nim +++ b/src/query.nim @@ -18,8 +18,7 @@ const posPrefix = "thGAVUV0VFVBa" posSuffix = "EjUAFQAlAFUAFQAA" -proc initQuery*(filters, includes, excludes, separator, text: string; - name=""): Query = +proc initQuery*(filters, includes, excludes, separator, text: string; name=""): Query = var sep = separator.strip().toUpper() Query( kind: custom, @@ -50,6 +49,9 @@ proc genQueryParam*(query: Query): string = var filters: seq[string] var param: string + if query.kind == users: + return query.text + for i, user in query.fromUser: param &= &"from:{user} " if i < query.fromUser.high: @@ -67,12 +69,18 @@ proc genQueryParam*(query: Query): string = result &= " " & query.text proc genQueryUrl*(query: Query): string = - if query.kind == multi: return "?" + if query.fromUser.len > 0: + result = "/" & query.fromUser.join(",") - result = &"/{query.kind}?" - if query.kind != custom: return + if query.kind == multi: + return result & "?" - var params: seq[string] + if query.kind notin {custom, users}: + return result & &"/{query.kind}?" + + result &= &"/search?" + + var params = @[&"kind={query.kind}"] if query.filters.len > 0: params &= "filter=" & query.filters.join(",") if query.includes.len > 0: @@ -84,7 +92,7 @@ proc genQueryUrl*(query: Query): string = if query.text.len > 0: params &= "text=" & query.text if params.len > 0: - result &= params.join("&") & "&" + result &= params.join("&") proc cleanPos*(pos: string): string = pos.multiReplace((posPrefix, ""), (posSuffix, "")) diff --git a/src/routes/search.nim b/src/routes/search.nim new file mode 100644 index 0000000..b22af77 --- /dev/null +++ b/src/routes/search.nim @@ -0,0 +1,30 @@ +import strutils, uri + +import jester + +import router_utils +import ".."/[query, types, utils, api, agents] +import ../views/[general, search] + +export search + +proc createSearchRouter*(cfg: Config) = + router search: + get "/search": + if @"text".len == 0 or "." in @"text": + resp Http404, showError("Please enter a valid username.", cfg.title) + + if "," in @"text": + redirect("/" & @"text") + + let query = Query(kind: parseEnum[QueryKind](@"kind", custom), text: @"text") + + case query.kind + of users: + let users = await getSearch[Profile](query, @"after", getAgent()) + resp renderMain(renderUserSearch(users, Prefs()), Prefs(), path=getPath()) + of custom: + let tweets = await getSearch[Tweet](query, @"after", getAgent()) + resp renderMain(renderTweetSearch(tweets, Prefs(), getPath()), Prefs(), path=getPath()) + else: + resp Http404 diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 09c2958..eb2dcb7 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -3,14 +3,14 @@ import asyncdispatch, strutils, sequtils, uri import jester import router_utils -import ".."/[api, prefs, types, utils, cache, formatters, agents, search] -import ../views/[general, profile, timeline, status] +import ".."/[api, prefs, types, utils, cache, formatters, agents, query] +import ../views/[general, profile, timeline, status, search] include "../views/rss.nimf" export uri, sequtils export router_utils -export api, cache, formatters, search, agents +export api, cache, formatters, query, agents export profile, timeline, status type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto]) @@ -33,7 +33,7 @@ proc fetchSingleTimeline*(name, after, agent: string; (profile, timeline) = await getProfileAndTimeline(name, agent, after) cache(profile) else: - var timelineFut = getTimelineSearch(get(query), after, agent) + var timelineFut = getSearch[Tweet](get(query), after, agent) if cachedProfile.isNone: profile = await getCachedProfile(name, agent) timeline = await timelineFut @@ -49,7 +49,7 @@ proc fetchMultiTimeline*(names: seq[string]; after, agent: string; else: q = some(Query(kind: multi, fromUser: names, excludes: @["replies"])) - return await getTimelineSearch(get(q), after, agent) + return await getSearch[Tweet](get(q), after, agent) proc showTimeline*(name, after: string; query: Option[Query]; prefs: Prefs; path, title, rss: string): Future[string] {.async.} = diff --git a/src/sass/profile/_base.scss b/src/sass/profile/_base.scss index 4d4cb49..4e069af 100644 --- a/src/sass/profile/_base.scss +++ b/src/sass/profile/_base.scss @@ -43,6 +43,14 @@ top: 50px; } +.profile-result .username { + margin: 0 !important; +} + +.profile-result .tweet-header { + margin-bottom: unset; +} + @media(max-width: 600px) { .profile-tabs { width: 100vw; diff --git a/src/sass/timeline.scss b/src/sass/timeline.scss index fa536bd..ff8b048 100644 --- a/src/sass/timeline.scss +++ b/src/sass/timeline.scss @@ -20,6 +20,14 @@ display: block; font-weight: bold; margin-bottom: 5px; + + input[type="text"] { + height: 20px; + } + + button { + float: unset; + } } .tab { diff --git a/src/types.nim b/src/types.nim index 772c8ed..de28cfe 100644 --- a/src/types.nim +++ b/src/types.nim @@ -57,7 +57,7 @@ dbFromTypes("cache.db", "", "", "", [Profile, Video]) type QueryKind* = enum - replies, media, multi, custom = "search" + replies, media, multi, users, custom Query* = object kind*: QueryKind diff --git a/src/views/general.nim b/src/views/general.nim index b5b1db3..fbbd17e 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -60,13 +60,6 @@ proc renderMain*(body: VNode; prefs: Prefs; title="Nitter"; titleText=""; desc=" result = doctype & $node -proc renderSearch*(): VNode = - buildHtml(tdiv(class="panel-container")): - tdiv(class="search-panel"): - form(`method`="post", action="/search"): - input(`type`="text", name="query", autofocus="", placeholder="Enter usernames...") - button(`type`="submit"): icon "search" - proc renderError*(error: string): VNode = buildHtml(tdiv(class="panel-container")): tdiv(class="error-panel"): diff --git a/src/views/profile.nim b/src/views/profile.nim index 7002248..87445ef 100644 --- a/src/views/profile.nim +++ b/src/views/profile.nim @@ -98,5 +98,5 @@ proc renderProfile*(profile: Profile; timeline: Timeline; if profile.protected: renderProtected(profile.username) else: - renderProfileTabs(timeline, profile.username) + renderProfileTabs(timeline.query, profile.username) renderTimelineTweets(timeline, prefs, path) diff --git a/src/views/search.nim b/src/views/search.nim new file mode 100644 index 0000000..f3043bf --- /dev/null +++ b/src/views/search.nim @@ -0,0 +1,38 @@ +import strutils, strformat +import karax/[karaxdsl, vdom, vstyles] + +import renderutils, timeline +import ".."/[types, formatters, query] + +proc renderSearch*(): VNode = + buildHtml(tdiv(class="panel-container")): + tdiv(class="search-panel"): + form(`method`="get", action="/search"): + verbatim "" + input(`type`="text", name="text", autofocus="", placeholder="Enter username...") + button(`type`="submit"): icon "search" + +proc renderTweetSearch*(timeline: Timeline; prefs: Prefs; path: string): VNode = + let users = get(timeline.query).fromUser + buildHtml(tdiv(class="timeline-container")): + tdiv(class="timeline-header"): + text users.join(" | ") + + renderProfileTabs(timeline.query, users.join(",")) + renderTimelineTweets(timeline, prefs, path) + +proc renderUserSearch*(users: Result[Profile]; prefs: Prefs): VNode = + let searchText = + if users.query.isSome: get(users.query).text + else: "" + + buildHtml(tdiv(class="timeline-container")): + tdiv(class="timeline-header"): + form(`method`="get", action="/search"): + verbatim "" + verbatim "" % searchText + button(`type`="submit"): icon "search" + + renderSearchTabs(users.query) + + renderTimelineUsers(users, prefs) diff --git a/src/views/timeline.nim b/src/views/timeline.nim index eccb16f..aabdf53 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -12,27 +12,35 @@ proc getQuery(query: Option[Query]): string = if result[^1] != '?': result &= "&" -proc getTabClass(results: Result; tab: string): string = +proc getTabClass(query: Option[Query]; tab: string): string = var classes = @["tab-item"] - if results.query.isNone or get(results.query).kind == multi: + if query.isNone or get(query).kind == multi: if tab == "posts": classes.add "active" - elif $get(results.query).kind == tab: + elif $get(query).kind == tab: classes.add "active" return classes.join(" ") -proc renderProfileTabs*(timeline: Timeline; username: string): VNode = +proc renderProfileTabs*(query: Option[Query]; username: string): VNode = let link = "/" & username buildHtml(ul(class="tab")): - li(class=timeline.getTabClass("posts")): + li(class=query.getTabClass("posts")): a(href=link): text "Tweets" - li(class=timeline.getTabClass("replies")): + li(class=query.getTabClass("replies")): a(href=(link & "/replies")): text "Tweets & Replies" - li(class=timeline.getTabClass("media")): + li(class=query.getTabClass("media")): a(href=(link & "/media")): text "Media" +proc renderSearchTabs*(query: Option[Query]): VNode = + var q = if query.isSome: get(query) else: Query() + + buildHtml(ul(class="tab")): + li(class=query.getTabClass("users")): + q.kind = users + a(href=genQueryUrl(q)): text "Users" + proc renderNewer(query: Option[Query]): VNode = buildHtml(tdiv(class="timeline-item show-more")): a(href=(getQuery(query).strip(chars={'?', '&'}))): @@ -62,6 +70,34 @@ proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode = proc threadFilter(it: Tweet; tweetThread: string): bool = it.retweet.isNone and it.reply.len == 0 and it.threadId == tweetThread +proc renderUser(user: Profile; prefs: Prefs): VNode = + buildHtml(tdiv(class="timeline-item")): + tdiv(class="tweet-body profile-result"): + tdiv(class="tweet-header"): + a(class="tweet-avatar", href=("/" & user.username)): + genImg(user.getUserpic("_bigger"), class="avatar") + + tdiv(class="tweet-name-row"): + tdiv(class="fullname-and-username"): + linkUser(user, class="fullname") + linkUser(user, class="username") + + tdiv(class="tweet-content media-body"): + verbatim linkifyText(user.bio, prefs) + +proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs): VNode = + buildHtml(tdiv(class="timeline")): + if not results.beginning: + renderNewer(results.query) + + if results.content.len > 0: + for user in results.content: + renderUser(user, prefs) + renderOlder(results.query, results.minId) + elif results.beginning: + renderNoneFound() + else: + renderNoMore() proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string): VNode = buildHtml(tdiv(class="timeline")):