From 9e3138e51bbfdec88604d0ea3ff8c3419f440c4b Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 21 Sep 2019 01:08:30 +0200 Subject: [PATCH] Add list support --- src/api.nim | 4 +- src/api/consts.nim | 2 + src/api/list.nim | 83 +++++++++++++++++++++++++++++++++++++ src/api/search.nim | 7 ++-- src/api/timeline.nim | 2 +- src/nitter.nim | 8 ++-- src/parser.nim | 10 +++++ src/query.nim | 4 +- src/routes/list.nim | 34 +++++++++++++++ src/routes/media.nim | 2 +- src/routes/preferences.nim | 2 +- src/routes/router_utils.nim | 4 +- src/routes/rss.nim | 5 +++ src/routes/search.nim | 4 +- src/routes/status.nim | 2 +- src/routes/timeline.nim | 2 +- src/sass/tweet/_base.scss | 5 ++- src/types.nim | 2 +- src/utils.nim | 2 +- src/views/general.nim | 4 +- src/views/list.nim | 20 +++++++++ src/views/renderutils.nim | 13 ++++-- src/views/rss.nimf | 26 ++++++++++++ src/views/search.nim | 13 ++---- src/views/timeline.nim | 3 +- 25 files changed, 224 insertions(+), 39 deletions(-) create mode 100644 src/api/list.nim create mode 100644 src/routes/list.nim create mode 100644 src/views/list.nim diff --git a/src/api.nim b/src/api.nim index 3b45b0f..df6d586 100644 --- a/src/api.nim +++ b/src/api.nim @@ -1,2 +1,2 @@ -import api/[profile, timeline, tweet, search, media] -export profile, timeline, tweet, search, media +import api/[profile, timeline, tweet, search, media, list] +export profile, timeline, tweet, search, media, list diff --git a/src/api/consts.nim b/src/api/consts.nim index 0be86d3..61b7508 100644 --- a/src/api/consts.nim +++ b/src/api/consts.nim @@ -11,6 +11,8 @@ const timelineUrl* = "i/profiles/show/$1/timeline/tweets" timelineMediaUrl* = "i/profiles/show/$1/media_timeline" + listUrl* = "$1/lists/$2/timeline" + listMembersUrl* = "$1/lists/$2/members" profilePopupUrl* = "i/profiles/popup" profileIntentUrl* = "intent/user" searchUrl* = "i/search/timeline" diff --git a/src/api/list.nim b/src/api/list.nim new file mode 100644 index 0000000..249ac5c --- /dev/null +++ b/src/api/list.nim @@ -0,0 +1,83 @@ +import httpclient, asyncdispatch, htmlparser, strformat +import sequtils, strutils, json, uri + +import ".."/[types, parser, parserutils, query] +import utils, consts, timeline, search + +proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} = + let url = base / (listUrl % [username, list]) + + let headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $url, + "User-Agent": agent, + "X-Twitter-Active-User": "yes", + "X-Requested-With": "XMLHttpRequest", + "Accept-Language": lang + }) + + var params = toSeq({ + "include_available_features": "1", + "include_entities": "1", + "reset_error_state": "false" + }) + + if after.len > 0: + params.add {"max_position": after} + + let json = await fetchJson(url ? params, headers) + result = await finishTimeline(json, Query(), after, agent) + if result.content.len > 0: + result.minId = result.content[^1].id + +proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} = + let url = base / (listMembersUrl % [username, list]) + + let headers = newHttpHeaders({ + "Accept": htmlAccept, + "Referer": $(base / &"{username}/lists/{list}/members"), + "User-Agent": agent, + "Accept-Language": lang + }) + + let html = await fetchHtml(url, headers) + + result = Result[Profile]( + minId: html.selectAttr(".stream-container", "data-min-position"), + hasMore: html.select(".has-more-items") != nil, + beginning: true, + query: Query(kind: users), + content: html.selectAll(".account").map(parseListProfile) + ) + +proc getListMembersSearch*(username, list, agent, after: string): Future[Result[Profile]] {.async.} = + let url = base / ((listMembersUrl & "/timeline") % [username, list]) + + let headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $(base / &"{username}/lists/{list}/members"), + "User-Agent": agent, + "X-Twitter-Active-User": "yes", + "X-Requested-With": "XMLHttpRequest", + "X-Push-With": "XMLHttpRequest", + "Accept-Language": lang + }) + + var params = toSeq({ + "include_available_features": "1", + "include_entities": "1", + "reset_error_state": "false" + }) + + if after.len > 0: + params.add {"max_position": after} + + let json = await fetchJson(url ? params, headers) + + result = getResult[Profile](json, Query(kind: users), after) + if json == nil or not json.hasKey("items_html"): return + + let html = json["items_html"].to(string) + result.hasMore = html != "\n" + for p in parseHtml(html).selectAll(".account"): + result.content.add parseListProfile(p) diff --git a/src/api/search.nim b/src/api/search.nim index 480df1b..36e7a41 100644 --- a/src/api/search.nim +++ b/src/api/search.nim @@ -7,7 +7,7 @@ import utils, consts, timeline proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] = if json == nil: return Result[T](beginning: true, query: query) Result[T]( - hasMore: json["has_more_items"].to(bool), + hasMore: json.getOrDefault("has_more_items").getBool(false), maxId: json.getOrDefault("max_position").getStr(""), minId: json.getOrDefault("min_position").getStr("").cleanPos(), query: query, @@ -16,7 +16,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] = proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} = let - kind = if query.kind == users: "users" else: "tweets" + kind = if query.kind == userSearch: "users" else: "tweets" pos = when T is Tweet: genPos(after) else: after param = genQueryParam(query) @@ -46,10 +46,9 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn return Result[T](query: query, beginning: true) let json = await fetchJson(base / searchUrl ? params, headers) - if json == nil: return Result[T](query: query, beginning: true) result = getResult[T](json, query, after) - if not json.hasKey("items_html"): return + if json == nil or not json.hasKey("items_html"): return when T is Tweet: result = await finishTimeline(json, query, after, agent) diff --git a/src/api/timeline.nim b/src/api/timeline.nim index ada0c67..b2a98d1 100644 --- a/src/api/timeline.nim +++ b/src/api/timeline.nim @@ -1,4 +1,4 @@ -import httpclient, asyncdispatch, htmlparser +import httpclient, asyncdispatch, htmlparser, strformat import sequtils, strutils, json, xmltree, uri import ".."/[types, parser, parserutils, formatters, query] diff --git a/src/nitter.nim b/src/nitter.nim index cece195..543da28 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -5,13 +5,14 @@ import jester import types, config, prefs import views/[general, about] -import routes/[preferences, timeline, status, media, search, rss] +import routes/[preferences, timeline, status, media, search, rss, list] const configPath {.strdefine.} = "./nitter.conf" let cfg = getConfig(configPath) createPrefRouter(cfg) createTimelineRouter(cfg) +createListRouter(cfg) createStatusRouter(cfg) createSearchRouter(cfg) createMediaRouter(cfg) @@ -24,15 +25,16 @@ settings: routes: get "/": - resp renderMain(renderSearch(), Prefs(), cfg.title) + resp renderMain(renderSearch(), request, cfg.title) get "/about": - resp renderMain(renderAbout(), Prefs(), cfg.title) + resp renderMain(renderAbout(), request, cfg.title) extend preferences, "" extend rss, "" extend search, "" extend timeline, "" + extend list, "" extend status, "" extend media, "" diff --git a/src/parser.nim b/src/parser.nim index f59ce2d..22413eb 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -39,6 +39,16 @@ proc parsePopupProfile*(node: XmlNode; selector=".profile-card"): Profile = result.getPopupStats(profile) +proc parseListProfile*(profile: XmlNode): Profile = + result = Profile( + fullname: profile.getName(".fullname"), + username: profile.getUsername(".username"), + bio: profile.getBio(".bio"), + userpic: profile.getAvatar(".avatar"), + verified: isVerified(profile), + protected: isProtected(profile), + ) + proc parseIntentProfile*(profile: XmlNode): Profile = result = Profile( fullname: profile.getName("a.fn.url.alternate-context"), diff --git a/src/query.nim b/src/query.nim index 402a194..5561169 100644 --- a/src/query.nim +++ b/src/query.nim @@ -58,7 +58,7 @@ proc genQueryParam*(query: Query): string = var filters: seq[string] var param: string - if query.kind == users: + if query.kind == userSearch: return query.text for i, user in query.fromUser: @@ -84,7 +84,7 @@ proc genQueryParam*(query: Query): string = result &= " " & query.text proc genQueryUrl*(query: Query): string = - if query.kind notin {custom, users}: return + if query.kind notin {custom, userSearch}: return var params = @[&"kind={query.kind}"] if query.text.len > 0: diff --git a/src/routes/list.nim b/src/routes/list.nim new file mode 100644 index 0000000..312e56c --- /dev/null +++ b/src/routes/list.nim @@ -0,0 +1,34 @@ +import strutils + +import jester + +import router_utils +import ".."/[query, types, api, agents] +import ../views/[general, timeline, list] + +template respList*(list, timeline: typed) = + if list.minId.len == 0: + resp Http404, showError("List \"" & @"list" & "\" not found", cfg.title) + let html = renderList(timeline, list.query, @"name", @"list") + let rss = "/$1/lists/$2/rss" % [@"name", @"list"] + resp renderMain(html, request, cfg.title, rss=rss) + +proc createListRouter*(cfg: Config) = + router list: + get "/@name/lists/@list": + cond '.' notin @"name" + let + list = await getListTimeline(@"name", @"list", getAgent(), @"after") + tweets = renderTimelineTweets(list, cookiePrefs(), request.path) + respList list, tweets + + get "/@name/lists/@list/members": + cond '.' notin @"name" + let list = + if @"after".len == 0: + await getListMembers(@"name", @"list", getAgent()) + else: + await getListMembersSearch(@"name", @"list", getAgent(), @"after") + + let users = renderTimelineUsers(list, cookiePrefs(), request.path) + respList list, users diff --git a/src/routes/media.nim b/src/routes/media.nim index 1fa1915..f187853 100644 --- a/src/routes/media.nim +++ b/src/routes/media.nim @@ -3,7 +3,7 @@ import asyncfile, uri, strutils, httpclient, os import jester, regex import router_utils -import ".."/[types, formatters, prefs] +import ".."/[types, formatters] import ../views/general export asyncfile, httpclient, os, strutils diff --git a/src/routes/preferences.nim b/src/routes/preferences.nim index 4058348..d824771 100644 --- a/src/routes/preferences.nim +++ b/src/routes/preferences.nim @@ -3,7 +3,7 @@ import strutils, uri import jester import router_utils -import ".."/[prefs, types] +import ".."/[types] import ../views/[general, preferences] export preferences diff --git a/src/routes/router_utils.nim b/src/routes/router_utils.nim index 164a28b..391ba6a 100644 --- a/src/routes/router_utils.nim +++ b/src/routes/router_utils.nim @@ -1,5 +1,5 @@ -import ../utils -export utils +import ../utils, ../prefs +export utils, prefs template cookiePrefs*(): untyped {.dirty.} = getPrefs(request.cookies.getOrDefault("preferences")) diff --git a/src/routes/rss.nim b/src/routes/rss.nim index fa4acb8..f40d939 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -34,3 +34,8 @@ proc createRssRouter*(cfg: Config) = get "/@name/search/rss": cond '.' notin @"name" respRss(await showRss(@"name", initQuery(params(request), name=(@"name")))) + + get "/@name/lists/@list/rss": + cond '.' notin @"name" + let list = await getListTimeline(@"name", @"list", getAgent(), "") + respRss(renderListRss(list.content, @"name", @"list")) diff --git a/src/routes/search.nim b/src/routes/search.nim index 700ca69..89497cd 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -3,7 +3,7 @@ import strutils, sequtils, uri import jester import router_utils -import ".."/[query, types, api, agents, prefs] +import ".."/[query, types, api, agents] import ../views/[general, search] export search @@ -18,7 +18,7 @@ proc createSearchRouter*(cfg: Config) = let query = initQuery(params(request)) case query.kind - of users: + of userSearch: if "," in @"text": redirect("/" & @"text") let users = await getSearch[Profile](query, @"after", getAgent()) diff --git a/src/routes/status.nim b/src/routes/status.nim index 2b37367..bfe0ec3 100644 --- a/src/routes/status.nim +++ b/src/routes/status.nim @@ -3,7 +3,7 @@ import asyncdispatch, strutils, sequtils, uri import jester import router_utils -import ".."/[api, prefs, types, formatters, agents] +import ".."/[api, types, formatters, agents] import ../views/[general, status] export uri, sequtils diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 4448913..2de73a1 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -3,7 +3,7 @@ import asyncdispatch, strutils, sequtils, uri import jester import router_utils -import ".."/[api, prefs, types, cache, formatters, agents, query] +import ".."/[api, types, cache, formatters, agents, query] import ../views/[general, profile, timeline, status, search] export uri, sequtils diff --git a/src/sass/tweet/_base.scss b/src/sass/tweet/_base.scss index a1320fd..1afc9e2 100644 --- a/src/sass/tweet/_base.scss +++ b/src/sass/tweet/_base.scss @@ -85,7 +85,10 @@ .replying-to { color: $fg_dark; margin: -2px 0 4px; - pointer-events: all; + + a { + pointer-events: all; + } } .retweet, .pinned, .tweet-stats { diff --git a/src/types.nim b/src/types.nim index be3ee20..3e431b9 100644 --- a/src/types.nim +++ b/src/types.nim @@ -57,7 +57,7 @@ dbFromTypes("cache.db", "", "", "", [Profile, Video]) type QueryKind* = enum - posts, replies, media, users, custom + posts, replies, media, users, userSearch, custom Query* = object kind*: QueryKind diff --git a/src/utils.nim b/src/utils.nim index a65e9cb..d49752b 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -41,7 +41,7 @@ proc cleanFilename*(filename: string): string = filename.replace(reg, "_") proc filterParams*(params: Table): seq[(string, string)] = - let filter = ["name", "id"] + let filter = ["name", "id", "list"] toSeq(params.pairs()).filterIt(it[0] notin filter and it[1].len > 0) proc isTwitterUrl*(url: string): bool = diff --git a/src/views/general.nim b/src/views/general.nim index fad9c85..f424ffc 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -71,5 +71,5 @@ proc renderError*(error: string): VNode = tdiv(class="error-panel"): span: text error -proc showError*(error, title: string): string = - renderMain(renderError(error), Request(), title, "Error") +template showError*(error, title: string): string = + renderMain(renderError(error), request, title, "Error") diff --git a/src/views/list.nim b/src/views/list.nim new file mode 100644 index 0000000..8020423 --- /dev/null +++ b/src/views/list.nim @@ -0,0 +1,20 @@ +import strformat +import karax/[karaxdsl, vdom] + +import renderutils +import ".."/[types] + +proc renderListTabs*(query: Query; path: string): VNode = + buildHtml(ul(class="tab")): + li(class=query.getTabClass(posts)): + a(href=(path)): text "Tweets" + li(class=query.getTabClass(users)): + a(href=(path & "/members")): text "Members" + +proc renderList*(body: VNode; query: Query; name, list: string): VNode = + buildHtml(tdiv(class="timeline-container")): + tdiv(class="timeline-header"): + text &"\"{list}\" by @{name}" + + renderListTabs(query, &"/{name}/lists/{list}") + body diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index 60445dd..7612acb 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -30,10 +30,6 @@ proc linkUser*(profile: Profile, class=""): VNode = text " " icon "lock-circled", title="Protected account" -proc genImg*(url: string; class=""): VNode = - buildHtml(): - img(src=getPicUrl(url), class=class, alt="Image") - proc linkText*(text: string; class=""): VNode = let url = if "http" notin text: "http://" & text else: text buildHtml(): @@ -91,3 +87,12 @@ proc genDate*(pref, state: string): VNode = else: verbatim &"" icon "calendar" + +proc genImg*(url: string; class=""): VNode = + buildHtml(): + img(src=getPicUrl(url), class=class, alt="Image") + +proc getTabClass*(query: Query; tab: QueryKind): string = + result = "tab-item" + if query.kind == tab: + result &= " active" diff --git a/src/views/rss.nimf b/src/views/rss.nimf index f9879c7..ac8bfc4 100644 --- a/src/views/rss.nimf +++ b/src/views/rss.nimf @@ -71,3 +71,29 @@ #end proc +# +#proc renderListRss*(tweets: seq[Tweet]; name, list: string): string = +#let prefs = Prefs(replaceTwitter: hostname) +#result = "" + + + + + ${list} / @${name} + https://${hostname}/${name}/lists/${list} + Twitter feed for: ${list} by @${name}. Generated by ${hostname} + en-us + 40 + #for tweet in tweets: + + ${getTitle(tweet, prefs)} + @${tweet.profile.username} + + ${getRfc822Time(tweet)} + https://${hostname}${getLink(tweet)} + https://${hostname}${getLink(tweet)} + + #end for + + +#end proc diff --git a/src/views/search.nim b/src/views/search.nim index 9b60b38..319c9bc 100644 --- a/src/views/search.nim +++ b/src/views/search.nim @@ -23,15 +23,10 @@ proc renderSearch*(): VNode = buildHtml(tdiv(class="panel-container")): tdiv(class="search-bar"): form(`method`="get", action="/search"): - hiddenField("kind", "users") + hiddenField("kind", "userSearch") input(`type`="text", name="text", autofocus="", placeholder="Enter username...") button(`type`="submit"): icon "search" -proc getTabClass(query: Query; tab: QueryKind): string = - result = "tab-item" - if query.kind == tab: - result &= " active" - proc renderProfileTabs*(query: Query; username: string): VNode = let link = "/" & username buildHtml(ul(class="tab")): @@ -50,8 +45,8 @@ proc renderSearchTabs*(query: Query): VNode = li(class=query.getTabClass(custom)): q.kind = custom a(href=("?" & genQueryUrl(q))): text "Tweets" - li(class=query.getTabClass(users)): - q.kind = users + li(class=query.getTabClass(userSearch)): + q.kind = userSearch a(href=("?" & genQueryUrl(q))): text "Users" proc isPanelOpen(q: Query): bool = @@ -114,7 +109,7 @@ proc renderUserSearch*(users: Result[Profile]; prefs: Prefs): VNode = buildHtml(tdiv(class="timeline-container")): tdiv(class="timeline-header"): form(`method`="get", action="/search", class="search-field"): - hiddenField("kind", "users") + hiddenField("kind", "userSearch") genInput("text", "", users.query.text, "Enter username...", class="pref-inline") button(`type`="submit"): icon "search" diff --git a/src/views/timeline.nim b/src/views/timeline.nim index 0fde744..a4072b9 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -64,7 +64,8 @@ proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs; path=""): VNod if results.content.len > 0: for user in results.content: renderUser(user, prefs) - renderMore(results.query, results.minId) + if results.minId != "0": + renderMore(results.query, results.minId) elif results.beginning: renderNoneFound() else: