More API refactoring

src/api/list.nim

@@ -1,4 +1,4 @@
-import httpclient, asyncdispatch, htmlparser, strformat
+import httpclient, asyncdispatch, htmlparser
 import sequtils, strutils, json, uri
 
 import ".."/[types, parser, parserutils, query]
@@ -29,8 +29,7 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.
 proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
   let
     url = base / (listMembersUrl % [username, list])
-    referer = base / &"{username}/lists/{list}/members"
+    html = await fetchHtml(url, genHeaders(agent, url))
-    html = await fetchHtml(url, genHeaders(agent, referer))
 
   result = Result[Profile](
     minId: html.selectAttr(".stream-container", "data-min-position"),
@@ -42,8 +41,8 @@ proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.a
 
 proc getListMembersSearch*(username, list, agent, after: string): Future[Result[Profile]] {.async.} =
   let
-    url = base / ((listMembersUrl & "/timeline") % [username, list])
+    referer = base / (listMembersUrl % [username, list])
-    referer = base / &"{username}/lists/{list}/members"
+    url = referer / "timeline"
     headers = genHeaders({"x-push-with": "XMLHttpRequest"}, agent, referer, xml=true)
 
   var params = toSeq({

src/api/media.nim

@@ -53,8 +53,7 @@ proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
 
   let
     headers = genHeaders({"authorization": auth}, agent, base, lang=false)
-    url = apiBase / tokenUrl
+    json = await fetchJson(apiBase / tokenUrl, headers)
-    json = await fetchJson(url, headers)
 
   if json != nil:
     result = json["guest_token"].to(string)

src/api/timeline.nim

@@ -1,9 +1,14 @@
 import httpclient, asyncdispatch, htmlparser, strformat
-import sequtils, strutils, json, xmltree, uri
+import sequtils, strutils, json, uri
 
 import ".."/[types, parser, parserutils, formatters, query]
 import utils, consts, media, search
 
+proc getMedia(thread: Thread | Timeline; agent: string) {.async.} =
+  await all(getVideos(thread, agent),
+            getCards(thread, agent),
+            getPolls(thread, agent))
+
 proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future[Timeline] {.async.} =
   result = getResult[Tweet](json, query, after)
   if json == nil: return
@@ -11,19 +16,13 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future
   if json["new_latent_count"].to(int) == 0: return
   if not json.hasKey("items_html"): return
 
-  let
+  let html = parseHtml(json["items_html"].to(string))
-    html = parseHtml(json["items_html"].to(string))
+  let thread = parseThread(html)
-    thread = parseThread(html)
-    vidsFut = getVideos(thread, agent)
-    pollFut = getPolls(thread, agent)
-    cardFut = getCards(thread, agent)
 
-  await all(vidsFut, pollFut, cardFut)
+  await getMedia(thread, agent)
   result.content = thread.content
 
 proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
-  let headers = genHeaders(agent, base / username, xml=true)
-
   var params = toSeq({
     "include_available_features": "1",
     "include_entities": "1",
@@ -34,7 +33,9 @@ proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
   if after.len > 0:
     params.add {"max_position": after}
 
+  let headers = genHeaders(agent, base / username, xml=true)
   let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
+
   result = await finishTimeline(json, Query(), after, agent)
 
 proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
@@ -48,9 +49,5 @@ proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Ti
     timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
     profile = parseTimelineProfile(html)
 
-    vidsFut = getVideos(timeline, agent)
+  await getMedia(timeline, agent)
-    pollFut = getPolls(timeline, agent)
-    cardFut = getCards(timeline, agent)
-
-  await all(vidsFut, pollFut, cardFut)
   result = (profile, timeline)

src/api/tweet.nim

@@ -17,9 +17,6 @@ proc getTweet*(username, id, after, agent: string): Future[Conversation] {.async
 
   result = parseConversation(html, after)
 
-  let
+  await all(getConversationVideos(result, agent),
-    vidsFut = getConversationVideos(result, agent)
+            getConversationCards(result, agent),
-    pollFut = getConversationPolls(result, agent)
+            getConversationPolls(result, agent))
-    cardFut = getConversationCards(result, agent)
-
-  await all(vidsFut, pollFut, cardFut)

tests/test_thread.py

@@ -18,7 +18,7 @@ thread = [
 
     ['d0m96/status/1141811379407425537', [], 'I\'m',
      ['The', 'The', 'Today', 'Some', 'If', 'There', 'Above'],
-     [['Thank', 'Also,', 'Awesome.', 'Thank']]],
+     [['Awesome.']]],
 
     ['gmpreussner/status/999766552546299904', [], 'A', [],
      [['I'], ['I']]]