Implement proper infinite scroll for replies

Fixes #125

public/js/infiniteScroll.js

@@ -6,8 +6,16 @@ function getLoadMore(doc) {
     return doc.querySelector('.show-more:not(.timeline-item)');
 }
 
+function isDuplicate(item, itemClass) {
+    const tweet = item.querySelector(".tweet-link");
+    if (tweet == null) return false;
+    const href = tweet.getAttribute("href");
+    return document.querySelector(itemClass + " .tweet-link[href='" + href + "']") != null;
+}
+
 window.onload = function() {
-    const isTweet = window.location.pathname.indexOf("/status/") !== -1;
+    const url = window.location.pathname;
+    const isTweet = url.indexOf("/status/") !== -1;
     const containerClass = isTweet ? ".replies" : ".timeline";
     const itemClass = isTweet ? ".thread-line" : ".timeline-item";
 
@@ -36,13 +44,16 @@ window.onload = function() {
 
                 for (var item of doc.querySelectorAll(itemClass)) {
                     if (item.className == "timeline-item show-more") continue;
+                    if (isDuplicate(item, itemClass)) continue;
                     if (isTweet) container.appendChild(item);
                     else insertBeforeLast(container, item);
                 }
 
-                if (isTweet) container.appendChild(getLoadMore(doc));
-                else insertBeforeLast(container, getLoadMore(doc));
                 loading = false;
+                const newLoadMore = getLoadMore(doc);
+                if (newLoadMore == null) return;
+                if (isTweet) container.appendChild(newLoadMore);
+                else insertBeforeLast(container, newLoadMore);
             }).catch(function (err) {
                 console.warn('Something went wrong.', err);
                 loading = true;

src/api/consts.nim

@@ -17,6 +17,7 @@ const
   profileIntentUrl* = "intent/user"
   searchUrl* = "i/search/timeline"
   tweetUrl* = "status"
+  repliesUrl* = "i/$1/conversation/$2"
   videoUrl* = "videos/tweet/config/$1.json"
   tokenUrl* = "guest/activate.json"
   cardUrl* = "i/cards/tfw/v1/$1"

src/api/media.nim

@@ -16,6 +16,7 @@ macro genMediaGet(media: untyped; token=false) =
     mediaName = capitalizeAscii($media)
     multi = ident("get" & mediaName & "s")
     convo = ident("getConversation" & mediaName & "s")
+    replies = ident("getReplies" & mediaName & "s")
     single = ident("get" & mediaName)
 
   quote do:
@@ -29,6 +30,14 @@ macro genMediaGet(media: untyped; token=false) =
       else:
         await all(`media`.mapIt(`single`(it, agent)))
 
+    proc `replies`*(replies: Result[Chain]; agent: string; token="") {.async.} =
+      when `token`:
+        var gToken = token
+        if gToken.len == 0: gToken = await getGuestToken(agent)
+        await all(replies.content.mapIt(`multi`(it, agent, token=gToken)))
+      else:
+        await all(replies.content.mapIt(`multi`(it, agent)))
+
     proc `convo`*(convo: Conversation; agent: string) {.async.} =
       var futs: seq[Future[void]]
       when `token`:
@@ -37,13 +46,13 @@ macro genMediaGet(media: untyped; token=false) =
         futs.add `multi`(convo.before, agent, token=token)
         futs.add `multi`(convo.after, agent, token=token)
         if convo.replies != nil:
-          futs.add convo.replies.content.mapIt(`multi`(it, agent, token=token))
+          futs.add `replies`(convo.replies, agent, token=token)
       else:
         futs.add `single`(convo.tweet, agent)
         futs.add `multi`(convo.before, agent)
         futs.add `multi`(convo.after, agent)
         if convo.replies != nil:
-          futs.add convo.replies.content.mapIt(`multi`(it, agent))
+          futs.add `replies`(convo.replies, agent)
       await all(futs)
 
 proc getGuestToken(agent: string; force=false): Future[string] {.async.} =

src/api/search.nim

@@ -8,8 +8,8 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
   if json == nil: return Result[T](beginning: true, query: query)
   Result[T](
     hasMore: json{"has_more_items"}.getBool(false),
-    maxId: json{"max_position"}.getStr(""),
+    maxId: json{"max_position"}.getStr,
-    minId: json{"min_position"}.getStr(""),
+    minId: json{"min_position"}.getStr,
     query: query,
     beginning: after.len == 0
   )

src/api/tweet.nim

@@ -30,3 +30,32 @@ proc getTweet*(username, id, after, agent: string): Future[Conversation] {.async
   await all(getConversationVideos(result, agent),
             getConversationCards(result, agent),
             getConversationPolls(result, agent))
+
+proc getReplies*(username, id, after, agent: string): Future[Result[Chain]] {.async.} =
+  let
+    headers = genHeaders({
+      "pragma": "no-cache",
+      "x-previous-page-name": "permalink",
+      "accept": htmlAccept
+      }, agent, base, xml=true)
+
+    params = {
+      "include_available_features": "1",
+      "include_entities": "1",
+      "max_position": after,
+    }
+
+    url = base / (repliesUrl % [username, id]) ? params
+
+  let json = await fetchJson(url, headers)
+  if json == nil or not json.hasKey("items_html"): return
+  let html = parseHtml(json{"items_html"}.getStr)
+
+  result = parseReplies(html)
+  result.minId = json{"min_position"}.getStr(result.minId)
+  if result.minId.len > 0:
+    result.hasMore = true
+
+  await all(getRepliesVideos(result, agent),
+            getRepliesCards(result, agent),
+            getRepliesPolls(result, agent))

src/parser.nim

@@ -160,6 +160,19 @@ proc parseChain*(nodes: XmlNode): Chain =
     else:
       result.content.add parseTweet(n)
 
+proc parseReplies*(replies: XmlNode; skipFirst=false): Result[Chain] =
+  new(result)
+  for i, reply in replies.filterIt(it.kind != xnText):
+    if skipFirst and i == 0: continue
+    let class = reply.attr("class").toLower()
+    if "lone" in class:
+      result.content.add parseChain(reply)
+    elif "showmore" in class:
+      result.minId = reply.selectAttr("button", "data-cursor")
+      result.hasMore = true
+    else:
+      result.content.add parseChain(reply.select(".stream-items"))
+
 proc parseConversation*(node: XmlNode; after: string): Conversation =
   let tweet = node.select(".permalink-tweet-container")
 
@@ -169,11 +182,6 @@ proc parseConversation*(node: XmlNode; after: string): Conversation =
   result = Conversation(
     tweet:  parseTweet(tweet),
     before: parseChain(node.select(".in-reply-to .stream-items")),
-    replies: Result[Chain](
-      minId: node.selectAttr(".replies-to .stream-container", "data-min-position"),
-      hasMore: node.select(".stream-footer .has-more-items") != nil,
-      beginning: after.len == 0
-    )
   )
 
   if result.before != nil:
@@ -181,26 +189,19 @@ proc parseConversation*(node: XmlNode; after: string): Conversation =
     if maxId.len > 0:
       result.before.more = -1
 
-  let showMore = node.selectAttr(".ThreadedConversation-showMoreThreads button",
-                                 "data-cursor")
-
-  if showMore.len > 0:
-    result.replies.minId = showMore
-    result.replies.hasMore = true
-
   let replies = node.select(".replies-to .stream-items")
   if replies == nil: return
 
-  for i, reply in replies.filterIt(it.kind != xnText):
+  let nodes = replies.filterIt(it.kind != xnText and "self" in it.attr("class"))
-    let class = reply.attr("class").toLower()
+  if nodes.len > 0 and "self" in nodes[0].attr("class"):
-    let thread = reply.select(".stream-items")
+    result.after = parseChain(nodes[0].select(".stream-items"))
 
-    if i == 0 and "self" in class:
+  result.replies = parseReplies(replies, result.after != nil)
-      result.after = parseChain(thread)
+
-    elif "lone" in class:
+  result.replies.beginning = after.len == 0
-      result.replies.content.add parseChain(reply)
+  if result.replies.minId.len == 0:
-    else:
+    result.replies.minId = node.selectAttr(".replies-to .stream-container", "data-min-position")
-      result.replies.content.add parseChain(thread)
+    result.replies.hasMore = node.select(".stream-footer .has-more-items") != nil
 
 proc parseTimeline*(node: XmlNode; after: string): Timeline =
   if node == nil: return Timeline()

src/routes/status.nim

@@ -1,6 +1,6 @@
 import asyncdispatch, strutils, sequtils, uri, options
 
-import jester
+import jester, karax/vdom
 
 import router_utils
 import ".."/[api, types, formatters, agents]
@@ -17,6 +17,10 @@ proc createStatusRouter*(cfg: Config) =
       cond '.' notin @"name"
       let prefs = cookiePrefs()
 
+      if @"scroll".len > 0:
+        let replies = await getReplies(@"name", @"id", @"max_position", getAgent())
+        resp $renderReplies(replies, prefs, getPath())
+
       let conversation = await getTweet(@"name", @"id", @"max_position", getAgent())
       if conversation == nil or conversation.tweet.id == 0:
         var error = "Tweet not found"

src/types.nim

@@ -77,14 +77,6 @@ type
     near*: string
     sep*: string
 
-  Result*[T] = ref object
-    content*: seq[T]
-    minId*: string
-    maxId*: string
-    hasMore*: bool
-    beginning*: bool
-    query*: Query
-
   Gif* = object
     url*: string
     thumb*: string
@@ -166,6 +158,14 @@ type
     photos*: seq[string]
     poll*: Option[Poll]
 
+  Result*[T] = ref object
+    content*: seq[T]
+    minId*: string
+    maxId*: string
+    hasMore*: bool
+    beginning*: bool
+    query*: Query
+
   Chain* = ref object
     content*: seq[Tweet]
     more*: int64

src/views/status.nim

@@ -29,6 +29,15 @@ proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode =
     if thread.more != 0:
       renderMoreReplies(thread)
 
+proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode =
+  buildHtml(tdiv(class="replies", id="r")):
+    for thread in replies.content:
+      if thread == nil: continue
+      renderReplyThread(thread, prefs, path)
+
+    if replies.hasMore:
+      renderMore(Query(), replies.minId, focus="#r")
+
 proc renderConversation*(conversation: Conversation; prefs: Prefs; path: string): VNode =
   let hasAfter = conversation.after != nil
   let showReplies = not prefs.hideReplies
@@ -60,10 +69,4 @@ proc renderConversation*(conversation: Conversation; prefs: Prefs; path: string)
       renderNewer(Query(), getLink(conversation.tweet))
 
     if conversation.replies.content.len > 0 and showReplies:
-      tdiv(class="replies", id="r"):
+      renderReplies(conversation.replies, prefs, path)
-        for thread in conversation.replies.content:
-          if thread == nil: continue
-          renderReplyThread(thread, prefs, path)
-
-    if conversation.replies.hasMore and showReplies:
-      renderMore(Query(), conversation.replies.minId, focus="#r")