Revert "Replace tweet endpoint with GraphQL"

This reverts commit 19adc658c3.
2023-02-02 12:38:58 +01:00 · 2023-02-02 12:38:58 +01:00 · 36c72f9860
parent 19adc658c3
commit 36c72f9860
7 changed files with 67 additions and 111 deletions
--- a/src/api.nim
+++ b/src/api.nim
@ -101,21 +101,16 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
  except InternalError:
    return Result[T](beginning: true, query: query)

-proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
-  if id.len == 0: return
-  let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = tweetVariables % [id, cursor]
-    params = {"variables": variables, "features": tweetFeatures}
-    js = await fetch(graphTweet ? params, Api.tweetDetail)
-  result = parseGraphConversation(js, id)
+proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
+  let url = tweet / (id & ".json") ? genParams(cursor=after)
+  result = parseConversation(await fetch(url, Api.tweet), id)

 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
-  result = (await getGraphTweet(id, after)).replies
+  result = (await getTweetImpl(id, after)).replies
  result.beginning = after.len == 0

 proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
-  result = await getGraphTweet(id)
+  result = await getTweetImpl(id)
  if after.len > 0:
    result.replies = await getReplies(id, after)

--- a/src/consts.nim
+++ b/src/consts.nim
@ -19,7 +19,6 @@ const
  tweet* = timelineApi / "conversation"

  graphql = api / "graphql"
-  graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
  graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
  graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
  graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
@ -59,34 +58,3 @@ const
  ## user:   "result_filter: user"
  ## photos: "result_filter: photos"
  ## videos: "result_filter: videos"
-
-  tweetVariables* = """{
-  "focalTweetId": "$1",
-  $2
-  "includePromotedContent": false,
-  "withBirdwatchNotes": false,
-  "withDownvotePerspective": false,
-  "withReactionsMetadata": false,
-  "withReactionsPerspective": false,
-  "withSuperFollowsTweetFields": false,
-  "withSuperFollowsUserFields": false,
-  "withVoice": false,
-  "withV2Timeline": true
-}"""
-
-  tweetFeatures* = """{
-  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
-  "responsive_web_graphql_timeline_navigation_enabled": false,
-  "standardized_nudges_misinfo": false,
-  "verified_phone_label_enabled": false,
-  "responsive_web_twitter_blue_verified_badge_is_enabled": false,
-  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
-  "view_counts_everywhere_api_enabled": false,
-  "responsive_web_edit_tweet_api_enabled": false,
-  "tweetypie_unmention_optimization_enabled": false,
-  "vibe_api_enabled": false,
-  "longform_notetweets_consumption_enabled": false,
-  "responsive_web_text_conversations_enabled": false,
-  "responsive_web_enhance_cards_enabled": false,
-  "interactive_text_enabled": false
-}"""
--- a/src/parser.nim
+++ b/src/parser.nim
@ -72,8 +72,8 @@ proc parseGif(js: JsonNode): Gif =
 proc parseVideo(js: JsonNode): Video =
  result = Video(
    thumb: js{"media_url_https"}.getImageStr,
-    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt),
-    available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available",
+    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
+    available: js{"ext_media_availability", "status"}.getStr == "available",
    title: js{"ext_alt_text"}.getStr,
    durationMs: js{"video_info", "duration_millis"}.getInt
    # playbackType: mp4
@ -185,7 +185,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
     result.url.len == 0 or result.url.startsWith("card://"):
    result.url = getPicUrl(result.image)

-proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
+proc parseTweet(js: JsonNode): Tweet =
  if js.isNull: return
  result = Tweet(
    id: js{"id_str"}.getId,
@ -193,6 +193,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
    replyId: js{"in_reply_to_status_id_str"}.getId,
    text: js{"full_text"}.getStr,
    time: js{"created_at"}.getTime,
+    source: getSource(js),
    hasThread: js{"self_thread"}.notNull,
    available: true,
    user: User(id: js{"user_id_str"}.getStr),
@ -217,7 +218,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
    result.retweet = some Tweet(id: rt.getId)
    return

-  if jsCard.kind != JNull:
+  with jsCard, js{"card"}:
    let name = jsCard{"name"}.getStr
    if "poll" in name:
      if "image" in name:
@ -294,17 +295,64 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
    result.users[k] = parseUser(v, k)

  for k, v in tweets:
-    var tweet = parseTweet(v, v{"card"})
+    var tweet = parseTweet(v)
    if tweet.user.id in result.users:
      tweet.user = result.users[tweet.user.id]
    result.tweets[k] = tweet

+proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
+  result.thread = Chain()
+
+  let thread = js{"content", "item", "content", "conversationThread"}
+  with cursor, thread{"showMoreCursor"}:
+    result.thread.cursor = cursor{"value"}.getStr
+    result.thread.hasMore = true
+
+  for t in thread{"conversationComponents"}:
+    let content = t{"conversationTweetComponent", "tweet"}
+
+    if content{"displayType"}.getStr == "SelfThread":
+      result.self = true
+
+    var tweet = finalizeTweet(global, content{"id"}.getStr)
+    if not tweet.available:
+      tweet.tombstone = getTombstone(content{"tombstone"})
+    result.thread.content.add tweet
+
+proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
+  result = Conversation(replies: Result[Chain](beginning: true))
+  let global = parseGlobalObjects(? js)
+
+  let instructions = ? js{"timeline", "instructions"}
+  if instructions.len == 0:
+    return
+
+  for e in instructions[0]{"addEntries", "entries"}:
+    let entry = e{"entryId"}.getStr
+    if "tweet" in entry or "tombstone" in entry:
+      let tweet = finalizeTweet(global, e.getEntryId)
+      if $tweet.id != tweetId:
+        result.before.content.add tweet
+      else:
+        result.tweet = tweet
+    elif "conversationThread" in entry:
+      let (thread, self) = parseThread(e, global)
+      if thread.content.len > 0:
+        if self:
+          result.after = thread
+        else:
+          result.replies.content.add thread
+    elif "cursor-showMore" in entry:
+      result.replies.bottom = e.getCursor
+    elif "cursor-bottom" in entry:
+      result.replies.bottom = e.getCursor
+
 proc parseStatus*(js: JsonNode): Tweet =
  with e, js{"errors"}:
    if e.getError == tweetNotFound:
      return

-  result = parseTweet(js, js{"card"})
+  result = parseTweet(js)
  if not result.isNil:
    result.user = parseUser(js{"user"})

@ -361,7 +409,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
 proc parsePhotoRail*(js: JsonNode): PhotoRail =
  for tweet in js:
    let
-      t = parseTweet(tweet, js{"card"})
+      t = parseTweet(tweet)
      url = if t.photos.len > 0: t.photos[0]
            elif t.video.isSome: get(t.video).thumb
            elif t.gif.isSome: get(t.gif).thumb
@ -370,61 +418,3 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =

    if url.len == 0: continue
    result.add GalleryPhoto(url: url, tweetId: $t.id)
-
-proc parseGraphTweet(js: JsonNode): Tweet =
-  if js.kind == JNull:
-    return Tweet(available: false)
-
-  var jsCard = copy(js{"card", "legacy"})
-  if jsCard.kind != JNull:
-    var values = newJObject()
-    for val in jsCard["binding_values"]:
-      values[val["key"].getStr] = val["value"]
-    jsCard["binding_values"] = values
-
-  result = parseTweet(js{"legacy"}, jsCard)
-  result.user = parseUser(js{"core", "user_results", "result", "legacy"})
-
-  if result.quote.isSome:
-    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
-
-proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
-  let thread = js{"content", "items"}
-  for t in js{"content", "items"}:
-    let entryId = t{"entryId"}.getStr
-    if "cursor-showmore" in entryId:
-      let cursor = t{"item", "itemContent", "value"}
-      result.thread.cursor = cursor.getStr
-      result.thread.hasMore = true
-    elif "tweet" in entryId:
-      let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"})
-      result.thread.content.add tweet
-
-      if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
-        result.self = true
-
-proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
-  result = Conversation(replies: Result[Chain](beginning: true))
-
-  let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
-  if instructions.len == 0:
-    return
-
-  for e in instructions[0]{"entries"}:
-    let entryId = e{"entryId"}.getStr
-    # echo entryId
-    if entryId.startsWith("tweet"):
-      let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
-
-      if $tweet.id == tweetId:
-        result.tweet = tweet
-      else:
-        result.before.content.add tweet
-    elif entryId.startsWith("conversationthread"):
-      let (thread, self) = parseGraphThread(e)
-      if self:
-        result.after = thread
-      else:
-        result.replies.content.add thread
-    elif entryId.startsWith("cursor-bottom"):
-      result.replies.bottom = e{"content", "itemContent", "value"}.getStr
--- a/src/parserutils.nim
+++ b/src/parserutils.nim
@ -133,6 +133,10 @@ proc getTombstone*(js: JsonNode): string =
  result = js{"tombstoneInfo", "richText", "text"}.getStr
  result.removeSuffix(" Learn more")

+proc getSource*(js: JsonNode): string =
+  let src = js{"source"}.getStr
+  result = src.substr(src.find('>') + 1, src.rfind('<') - 1)
+
 proc getMp4Resolution*(url: string): int =
  # parses the height out of a URL like this one:
  # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4
--- a/src/tokens.nim
+++ b/src/tokens.nim
@ -41,8 +41,7 @@ proc getPoolJson*(): JsonNode =
      let
        maxReqs =
          case api
-          of Api.listMembers, Api.listBySlug, Api.list,
-             Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
+          of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId, Api.userScreenName: 500
          of Api.timeline: 187
          else: 180
        reqs = maxReqs - token.apis[api].remaining
--- a/src/types.nim
+++ b/src/types.nim
@ -9,7 +9,6 @@ type
  InternalError* = object of CatchableError

  Api* {.pure.} = enum
-    tweetDetail
    userShow
    timeline
    search
@ -177,6 +176,7 @@ type
    available*: bool
    tombstone*: string
    location*: string
+    source*: string
    stats*: TweetStats
    retweet*: Option[Tweet]
    attribution*: Option[User]
--- a/src/views/tweet.nim
+++ b/src/views/tweet.nim
@ -347,7 +347,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
        renderQuote(tweet.quote.get(), prefs, path)

      if mainTweet:
-        p(class="tweet-published"): text &"{getTime(tweet)}"
+        p(class="tweet-published"): text &"{getTime(tweet)} · {tweet.source}"

      if tweet.mediaTags.len > 0:
        renderMediaTags(tweet.mediaTags)