Add support for loading more tweet replies

src/api/media.nim

@@ -35,12 +35,12 @@ macro genMediaGet(media: untyped; token=false) =
         futs.add `single`(convo.tweet, agent, token)
         futs.add `multi`(convo.before, agent, token=token)
         futs.add `multi`(convo.after, agent, token=token)
-        futs.add convo.replies.mapIt(`multi`(it, agent, token=token))
+        futs.add convo.replies.content.mapIt(`multi`(it, agent, token=token))
       else:
         futs.add `single`(convo.tweet, agent)
         futs.add `multi`(convo.before, agent)
         futs.add `multi`(convo.after, agent)
-        futs.add convo.replies.mapIt(`multi`(it, agent))
+        futs.add convo.replies.content.mapIt(`multi`(it, agent))
       await all(futs)
 
 proc getGuestToken(agent: string; force=false): Future[string] {.async.} =

src/api/search.nim

@@ -7,9 +7,9 @@ import utils, consts, timeline
 proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
   if json == nil: return Result[T](beginning: true, query: query)
   Result[T](
-    hasMore: json.getOrDefault("has_more_items").getBool(false),
-    maxId: json.getOrDefault("max_position").getStr(""),
-    minId: json.getOrDefault("min_position").getStr("").cleanPos(),
+    hasMore: json{"has_more_items"}.getBool(false),
+    maxId: json{"max_position"}.getStr(""),
+    minId: json{"min_position"}.getStr("").cleanPos(),
     query: query,
     beginning: after.len == 0
   )

src/api/tweet.nim

@@ -3,7 +3,7 @@ import httpclient, asyncdispatch, strutils, uri
 import ".."/[types, parser]
 import utils, consts, media
 
-proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} =
+proc getTweet*(username, id, after, agent: string): Future[Conversation] {.async.} =
   let headers = newHttpHeaders({
     "Accept": jsonAccept,
     "Referer": $base,
@@ -11,17 +11,17 @@ proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} =
     "X-Twitter-Active-User": "yes",
     "X-Requested-With": "XMLHttpRequest",
     "Accept-Language": lang,
-    "pragma": "no-cache",
-    "x-previous-page-name": "profile"
+    "Pragma": "no-cache",
+    "X-Previous-Page-Name": "profile"
   })
 
   let
-    url = base / username / tweetUrl / id
+    url = base / username / tweetUrl / id ? {"max_position": after}
     html = await fetchHtml(url, headers)
 
   if html == nil: return
 
-  result = parseConversation(html)
+  result = parseConversation(html, after)
 
   let
     vidsFut = getConversationVideos(result, agent)

src/parser.nim

@@ -141,7 +141,7 @@ proc parseThread*(nodes: XmlNode): Thread =
     else:
       result.content.add parseTweet(n)
 
-proc parseConversation*(node: XmlNode): Conversation =
+proc parseConversation*(node: XmlNode; after: string): Conversation =
   let tweet = node.select(".permalink-tweet-container")
 
   if tweet == nil:
@@ -149,8 +149,20 @@ proc parseConversation*(node: XmlNode): Conversation =
 
   result = Conversation(
     tweet:  parseTweet(tweet),
-    before: parseThread(node.select(".in-reply-to .stream-items"))
+    before: parseThread(node.select(".in-reply-to .stream-items")),
+    replies: Result[Thread](
+      minId: node.selectAttr(".replies-to .stream-container", "data-min-position"),
+      hasMore: node.select(".stream-footer .has-more-items") != nil,
+      beginning: after.len == 0
     )
+  )
+
+  let showMore = node.selectAttr(".ThreadedConversation-showMoreThreads button",
+                                 "data-cursor")
+
+  if showMore.len > 0:
+    result.replies.minId = showMore
+    result.replies.hasMore = true
 
   let replies = node.select(".replies-to .stream-items")
   if replies == nil: return
@@ -162,9 +174,9 @@ proc parseConversation*(node: XmlNode): Conversation =
     if i == 0 and "self" in class:
       result.after = parseThread(thread)
     elif "lone" in class:
-      result.replies.add parseThread(reply)
+      result.replies.content.add parseThread(reply)
     else:
-      result.replies.add parseThread(thread)
+      result.replies.content.add parseThread(thread)
 
 proc parseTimeline*(node: XmlNode; after: string): Timeline =
   if node == nil: return Timeline()

src/routes/status.nim

@@ -17,7 +17,7 @@ proc createStatusRouter*(cfg: Config) =
       cond '.' notin @"name"
       let prefs = cookiePrefs()
 
-      let conversation = await getTweet(@"name", @"id", getAgent())
+      let conversation = await getTweet(@"name", @"id", @"after", getAgent())
       if conversation == nil or conversation.tweet.id.len == 0:
         if conversation != nil and conversation.tweet.tombstone.len > 0:
           resp Http404, showError(conversation.tweet.tombstone, cfg.title)

src/sass/tweet/thread.scss

@@ -3,6 +3,10 @@
 
 .conversation {
     @include panel(100%, 600px);
+
+    .show-more {
+        margin-bottom: 10px;
+    }
 }
 
 .main-thread {

src/types.nim

@@ -164,7 +164,7 @@ type
     tweet*: Tweet
     before*: Thread
     after*: Thread
-    replies*: seq[Thread]
+    replies*: Result[Thread]
 
   Timeline* = Result[Tweet]
 

src/views/status.nim

@@ -1,7 +1,7 @@
 import karax/[karaxdsl, vdom]
 
-import ../types
-import tweet
+import ".."/[types, formatters]
+import tweet, timeline
 
 proc renderMoreReplies(thread: Thread): VNode =
   let num = if thread.more != -1: $thread.more & " " else: ""
@@ -42,8 +42,14 @@ proc renderConversation*(conversation: Conversation; prefs: Prefs; path: string)
           if more != 0:
             renderMoreReplies(conversation.after)
 
-    if conversation.replies.len > 0:
+    if not conversation.replies.beginning:
+      renderNewer(Query(), getLink(conversation.tweet))
+
+    if conversation.replies.content.len > 0:
       tdiv(class="replies"):
-        for thread in conversation.replies:
+        for thread in conversation.replies.content:
           if thread == nil: continue
           renderReplyThread(thread, prefs, path)
+
+    if conversation.replies.hasMore:
+      renderMore(Query(), conversation.replies.minId)

src/views/timeline.nim

@@ -10,14 +10,14 @@ proc getQuery(query: Query): string =
   if result.len > 0:
     result &= "&"
 
-proc renderNewer(query: Query; path: string): VNode =
+proc renderNewer*(query: Query; path: string): VNode =
   let q = genQueryUrl(query)
   let url = if q.len > 0: "?" & q else: ""
   buildHtml(tdiv(class="timeline-item show-more")):
     a(href=(path & url)):
       text "Load newest"
 
-proc renderMore(query: Query; minId: string): VNode =
+proc renderMore*(query: Query; minId: string): VNode =
   buildHtml(tdiv(class="show-more")):
     a(href=(&"?{getQuery(query)}after={minId}")):
       text "Load more"