Add support for loading more tweet replies

This commit is contained in:
Zed 2019-09-24 15:39:04 +02:00
parent 14e544500d
commit 9038645bc1
9 changed files with 44 additions and 22 deletions

View File

@ -35,12 +35,12 @@ macro genMediaGet(media: untyped; token=false) =
futs.add `single`(convo.tweet, agent, token) futs.add `single`(convo.tweet, agent, token)
futs.add `multi`(convo.before, agent, token=token) futs.add `multi`(convo.before, agent, token=token)
futs.add `multi`(convo.after, agent, token=token) futs.add `multi`(convo.after, agent, token=token)
futs.add convo.replies.mapIt(`multi`(it, agent, token=token)) futs.add convo.replies.content.mapIt(`multi`(it, agent, token=token))
else: else:
futs.add `single`(convo.tweet, agent) futs.add `single`(convo.tweet, agent)
futs.add `multi`(convo.before, agent) futs.add `multi`(convo.before, agent)
futs.add `multi`(convo.after, agent) futs.add `multi`(convo.after, agent)
futs.add convo.replies.mapIt(`multi`(it, agent)) futs.add convo.replies.content.mapIt(`multi`(it, agent))
await all(futs) await all(futs)
proc getGuestToken(agent: string; force=false): Future[string] {.async.} = proc getGuestToken(agent: string; force=false): Future[string] {.async.} =

View File

@ -7,9 +7,9 @@ import utils, consts, timeline
proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] = proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
if json == nil: return Result[T](beginning: true, query: query) if json == nil: return Result[T](beginning: true, query: query)
Result[T]( Result[T](
hasMore: json.getOrDefault("has_more_items").getBool(false), hasMore: json{"has_more_items"}.getBool(false),
maxId: json.getOrDefault("max_position").getStr(""), maxId: json{"max_position"}.getStr(""),
minId: json.getOrDefault("min_position").getStr("").cleanPos(), minId: json{"min_position"}.getStr("").cleanPos(),
query: query, query: query,
beginning: after.len == 0 beginning: after.len == 0
) )

View File

@ -3,7 +3,7 @@ import httpclient, asyncdispatch, strutils, uri
import ".."/[types, parser] import ".."/[types, parser]
import utils, consts, media import utils, consts, media
proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} = proc getTweet*(username, id, after, agent: string): Future[Conversation] {.async.} =
let headers = newHttpHeaders({ let headers = newHttpHeaders({
"Accept": jsonAccept, "Accept": jsonAccept,
"Referer": $base, "Referer": $base,
@ -11,17 +11,17 @@ proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} =
"X-Twitter-Active-User": "yes", "X-Twitter-Active-User": "yes",
"X-Requested-With": "XMLHttpRequest", "X-Requested-With": "XMLHttpRequest",
"Accept-Language": lang, "Accept-Language": lang,
"pragma": "no-cache", "Pragma": "no-cache",
"x-previous-page-name": "profile" "X-Previous-Page-Name": "profile"
}) })
let let
url = base / username / tweetUrl / id url = base / username / tweetUrl / id ? {"max_position": after}
html = await fetchHtml(url, headers) html = await fetchHtml(url, headers)
if html == nil: return if html == nil: return
result = parseConversation(html) result = parseConversation(html, after)
let let
vidsFut = getConversationVideos(result, agent) vidsFut = getConversationVideos(result, agent)

View File

@ -141,7 +141,7 @@ proc parseThread*(nodes: XmlNode): Thread =
else: else:
result.content.add parseTweet(n) result.content.add parseTweet(n)
proc parseConversation*(node: XmlNode): Conversation = proc parseConversation*(node: XmlNode; after: string): Conversation =
let tweet = node.select(".permalink-tweet-container") let tweet = node.select(".permalink-tweet-container")
if tweet == nil: if tweet == nil:
@ -149,9 +149,21 @@ proc parseConversation*(node: XmlNode): Conversation =
result = Conversation( result = Conversation(
tweet: parseTweet(tweet), tweet: parseTweet(tweet),
before: parseThread(node.select(".in-reply-to .stream-items")) before: parseThread(node.select(".in-reply-to .stream-items")),
replies: Result[Thread](
minId: node.selectAttr(".replies-to .stream-container", "data-min-position"),
hasMore: node.select(".stream-footer .has-more-items") != nil,
beginning: after.len == 0
)
) )
let showMore = node.selectAttr(".ThreadedConversation-showMoreThreads button",
"data-cursor")
if showMore.len > 0:
result.replies.minId = showMore
result.replies.hasMore = true
let replies = node.select(".replies-to .stream-items") let replies = node.select(".replies-to .stream-items")
if replies == nil: return if replies == nil: return
@ -162,9 +174,9 @@ proc parseConversation*(node: XmlNode): Conversation =
if i == 0 and "self" in class: if i == 0 and "self" in class:
result.after = parseThread(thread) result.after = parseThread(thread)
elif "lone" in class: elif "lone" in class:
result.replies.add parseThread(reply) result.replies.content.add parseThread(reply)
else: else:
result.replies.add parseThread(thread) result.replies.content.add parseThread(thread)
proc parseTimeline*(node: XmlNode; after: string): Timeline = proc parseTimeline*(node: XmlNode; after: string): Timeline =
if node == nil: return Timeline() if node == nil: return Timeline()

View File

@ -17,7 +17,7 @@ proc createStatusRouter*(cfg: Config) =
cond '.' notin @"name" cond '.' notin @"name"
let prefs = cookiePrefs() let prefs = cookiePrefs()
let conversation = await getTweet(@"name", @"id", getAgent()) let conversation = await getTweet(@"name", @"id", @"after", getAgent())
if conversation == nil or conversation.tweet.id.len == 0: if conversation == nil or conversation.tweet.id.len == 0:
if conversation != nil and conversation.tweet.tombstone.len > 0: if conversation != nil and conversation.tweet.tombstone.len > 0:
resp Http404, showError(conversation.tweet.tombstone, cfg.title) resp Http404, showError(conversation.tweet.tombstone, cfg.title)

View File

@ -3,6 +3,10 @@
.conversation { .conversation {
@include panel(100%, 600px); @include panel(100%, 600px);
.show-more {
margin-bottom: 10px;
}
} }
.main-thread { .main-thread {

View File

@ -164,7 +164,7 @@ type
tweet*: Tweet tweet*: Tweet
before*: Thread before*: Thread
after*: Thread after*: Thread
replies*: seq[Thread] replies*: Result[Thread]
Timeline* = Result[Tweet] Timeline* = Result[Tweet]

View File

@ -1,7 +1,7 @@
import karax/[karaxdsl, vdom] import karax/[karaxdsl, vdom]
import ../types import ".."/[types, formatters]
import tweet import tweet, timeline
proc renderMoreReplies(thread: Thread): VNode = proc renderMoreReplies(thread: Thread): VNode =
let num = if thread.more != -1: $thread.more & " " else: "" let num = if thread.more != -1: $thread.more & " " else: ""
@ -42,8 +42,14 @@ proc renderConversation*(conversation: Conversation; prefs: Prefs; path: string)
if more != 0: if more != 0:
renderMoreReplies(conversation.after) renderMoreReplies(conversation.after)
if conversation.replies.len > 0: if not conversation.replies.beginning:
renderNewer(Query(), getLink(conversation.tweet))
if conversation.replies.content.len > 0:
tdiv(class="replies"): tdiv(class="replies"):
for thread in conversation.replies: for thread in conversation.replies.content:
if thread == nil: continue if thread == nil: continue
renderReplyThread(thread, prefs, path) renderReplyThread(thread, prefs, path)
if conversation.replies.hasMore:
renderMore(Query(), conversation.replies.minId)

View File

@ -10,14 +10,14 @@ proc getQuery(query: Query): string =
if result.len > 0: if result.len > 0:
result &= "&" result &= "&"
proc renderNewer(query: Query; path: string): VNode = proc renderNewer*(query: Query; path: string): VNode =
let q = genQueryUrl(query) let q = genQueryUrl(query)
let url = if q.len > 0: "?" & q else: "" let url = if q.len > 0: "?" & q else: ""
buildHtml(tdiv(class="timeline-item show-more")): buildHtml(tdiv(class="timeline-item show-more")):
a(href=(path & url)): a(href=(path & url)):
text "Load newest" text "Load newest"
proc renderMore(query: Query; minId: string): VNode = proc renderMore*(query: Query; minId: string): VNode =
buildHtml(tdiv(class="show-more")): buildHtml(tdiv(class="show-more")):
a(href=(&"?{getQuery(query)}after={minId}")): a(href=(&"?{getQuery(query)}after={minId}")):
text "Load more" text "Load more"