Revert "Replace tweet endpoint with GraphQL"

This reverts commit 19adc658c3.
This commit is contained in:
Zed 2023-02-02 12:38:58 +01:00
parent 19adc658c3
commit 36c72f9860
7 changed files with 67 additions and 111 deletions

View File

@ -101,21 +101,16 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
except InternalError: except InternalError:
return Result[T](beginning: true, query: query) return Result[T](beginning: true, query: query)
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} = proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
if id.len == 0: return let url = tweet / (id & ".json") ? genParams(cursor=after)
let result = parseConversation(await fetch(url, Api.tweet), id)
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = tweetVariables % [id, cursor]
params = {"variables": variables, "features": tweetFeatures}
js = await fetch(graphTweet ? params, Api.tweetDetail)
result = parseGraphConversation(js, id)
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} = proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
result = (await getGraphTweet(id, after)).replies result = (await getTweetImpl(id, after)).replies
result.beginning = after.len == 0 result.beginning = after.len == 0
proc getTweet*(id: string; after=""): Future[Conversation] {.async.} = proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
result = await getGraphTweet(id) result = await getTweetImpl(id)
if after.len > 0: if after.len > 0:
result.replies = await getReplies(id, after) result.replies = await getReplies(id, after)

View File

@ -19,7 +19,6 @@ const
tweet* = timelineApi / "conversation" tweet* = timelineApi / "conversation"
graphql = api / "graphql" graphql = api / "graphql"
graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName" graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId" graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List" graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
@ -59,34 +58,3 @@ const
## user: "result_filter: user" ## user: "result_filter: user"
## photos: "result_filter: photos" ## photos: "result_filter: photos"
## videos: "result_filter: videos" ## videos: "result_filter: videos"
tweetVariables* = """{
"focalTweetId": "$1",
$2
"includePromotedContent": false,
"withBirdwatchNotes": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withSuperFollowsTweetFields": false,
"withSuperFollowsUserFields": false,
"withVoice": false,
"withV2Timeline": true
}"""
tweetFeatures* = """{
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": false,
"standardized_nudges_misinfo": false,
"verified_phone_label_enabled": false,
"responsive_web_twitter_blue_verified_badge_is_enabled": false,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"view_counts_everywhere_api_enabled": false,
"responsive_web_edit_tweet_api_enabled": false,
"tweetypie_unmention_optimization_enabled": false,
"vibe_api_enabled": false,
"longform_notetweets_consumption_enabled": false,
"responsive_web_text_conversations_enabled": false,
"responsive_web_enhance_cards_enabled": false,
"interactive_text_enabled": false
}"""

View File

@ -72,8 +72,8 @@ proc parseGif(js: JsonNode): Gif =
proc parseVideo(js: JsonNode): Video = proc parseVideo(js: JsonNode): Video =
result = Video( result = Video(
thumb: js{"media_url_https"}.getImageStr, thumb: js{"media_url_https"}.getImageStr,
views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt), views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available", available: js{"ext_media_availability", "status"}.getStr == "available",
title: js{"ext_alt_text"}.getStr, title: js{"ext_alt_text"}.getStr,
durationMs: js{"video_info", "duration_millis"}.getInt durationMs: js{"video_info", "duration_millis"}.getInt
# playbackType: mp4 # playbackType: mp4
@ -185,7 +185,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
result.url.len == 0 or result.url.startsWith("card://"): result.url.len == 0 or result.url.startsWith("card://"):
result.url = getPicUrl(result.image) result.url = getPicUrl(result.image)
proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = proc parseTweet(js: JsonNode): Tweet =
if js.isNull: return if js.isNull: return
result = Tweet( result = Tweet(
id: js{"id_str"}.getId, id: js{"id_str"}.getId,
@ -193,6 +193,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
replyId: js{"in_reply_to_status_id_str"}.getId, replyId: js{"in_reply_to_status_id_str"}.getId,
text: js{"full_text"}.getStr, text: js{"full_text"}.getStr,
time: js{"created_at"}.getTime, time: js{"created_at"}.getTime,
source: getSource(js),
hasThread: js{"self_thread"}.notNull, hasThread: js{"self_thread"}.notNull,
available: true, available: true,
user: User(id: js{"user_id_str"}.getStr), user: User(id: js{"user_id_str"}.getStr),
@ -217,7 +218,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
result.retweet = some Tweet(id: rt.getId) result.retweet = some Tweet(id: rt.getId)
return return
if jsCard.kind != JNull: with jsCard, js{"card"}:
let name = jsCard{"name"}.getStr let name = jsCard{"name"}.getStr
if "poll" in name: if "poll" in name:
if "image" in name: if "image" in name:
@ -294,17 +295,64 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result.users[k] = parseUser(v, k) result.users[k] = parseUser(v, k)
for k, v in tweets: for k, v in tweets:
var tweet = parseTweet(v, v{"card"}) var tweet = parseTweet(v)
if tweet.user.id in result.users: if tweet.user.id in result.users:
tweet.user = result.users[tweet.user.id] tweet.user = result.users[tweet.user.id]
result.tweets[k] = tweet result.tweets[k] = tweet
proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
result.thread = Chain()
let thread = js{"content", "item", "content", "conversationThread"}
with cursor, thread{"showMoreCursor"}:
result.thread.cursor = cursor{"value"}.getStr
result.thread.hasMore = true
for t in thread{"conversationComponents"}:
let content = t{"conversationTweetComponent", "tweet"}
if content{"displayType"}.getStr == "SelfThread":
result.self = true
var tweet = finalizeTweet(global, content{"id"}.getStr)
if not tweet.available:
tweet.tombstone = getTombstone(content{"tombstone"})
result.thread.content.add tweet
proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))
let global = parseGlobalObjects(? js)
let instructions = ? js{"timeline", "instructions"}
if instructions.len == 0:
return
for e in instructions[0]{"addEntries", "entries"}:
let entry = e{"entryId"}.getStr
if "tweet" in entry or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId)
if $tweet.id != tweetId:
result.before.content.add tweet
else:
result.tweet = tweet
elif "conversationThread" in entry:
let (thread, self) = parseThread(e, global)
if thread.content.len > 0:
if self:
result.after = thread
else:
result.replies.content.add thread
elif "cursor-showMore" in entry:
result.replies.bottom = e.getCursor
elif "cursor-bottom" in entry:
result.replies.bottom = e.getCursor
proc parseStatus*(js: JsonNode): Tweet = proc parseStatus*(js: JsonNode): Tweet =
with e, js{"errors"}: with e, js{"errors"}:
if e.getError == tweetNotFound: if e.getError == tweetNotFound:
return return
result = parseTweet(js, js{"card"}) result = parseTweet(js)
if not result.isNil: if not result.isNil:
result.user = parseUser(js{"user"}) result.user = parseUser(js{"user"})
@ -361,7 +409,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
proc parsePhotoRail*(js: JsonNode): PhotoRail = proc parsePhotoRail*(js: JsonNode): PhotoRail =
for tweet in js: for tweet in js:
let let
t = parseTweet(tweet, js{"card"}) t = parseTweet(tweet)
url = if t.photos.len > 0: t.photos[0] url = if t.photos.len > 0: t.photos[0]
elif t.video.isSome: get(t.video).thumb elif t.video.isSome: get(t.video).thumb
elif t.gif.isSome: get(t.gif).thumb elif t.gif.isSome: get(t.gif).thumb
@ -370,61 +418,3 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =
if url.len == 0: continue if url.len == 0: continue
result.add GalleryPhoto(url: url, tweetId: $t.id) result.add GalleryPhoto(url: url, tweetId: $t.id)
proc parseGraphTweet(js: JsonNode): Tweet =
if js.kind == JNull:
return Tweet(available: false)
var jsCard = copy(js{"card", "legacy"})
if jsCard.kind != JNull:
var values = newJObject()
for val in jsCard["binding_values"]:
values[val["key"].getStr] = val["value"]
jsCard["binding_values"] = values
result = parseTweet(js{"legacy"}, jsCard)
result.user = parseUser(js{"core", "user_results", "result", "legacy"})
if result.quote.isSome:
result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
let thread = js{"content", "items"}
for t in js{"content", "items"}:
let entryId = t{"entryId"}.getStr
if "cursor-showmore" in entryId:
let cursor = t{"item", "itemContent", "value"}
result.thread.cursor = cursor.getStr
result.thread.hasMore = true
elif "tweet" in entryId:
let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"})
result.thread.content.add tweet
if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
result.self = true
proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))
let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
if instructions.len == 0:
return
for e in instructions[0]{"entries"}:
let entryId = e{"entryId"}.getStr
# echo entryId
if entryId.startsWith("tweet"):
let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
if $tweet.id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
elif entryId.startsWith("conversationthread"):
let (thread, self) = parseGraphThread(e)
if self:
result.after = thread
else:
result.replies.content.add thread
elif entryId.startsWith("cursor-bottom"):
result.replies.bottom = e{"content", "itemContent", "value"}.getStr

View File

@ -133,6 +133,10 @@ proc getTombstone*(js: JsonNode): string =
result = js{"tombstoneInfo", "richText", "text"}.getStr result = js{"tombstoneInfo", "richText", "text"}.getStr
result.removeSuffix(" Learn more") result.removeSuffix(" Learn more")
proc getSource*(js: JsonNode): string =
let src = js{"source"}.getStr
result = src.substr(src.find('>') + 1, src.rfind('<') - 1)
proc getMp4Resolution*(url: string): int = proc getMp4Resolution*(url: string): int =
# parses the height out of a URL like this one: # parses the height out of a URL like this one:
# https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4 # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4

View File

@ -41,8 +41,7 @@ proc getPoolJson*(): JsonNode =
let let
maxReqs = maxReqs =
case api case api
of Api.listMembers, Api.listBySlug, Api.list, of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId, Api.userScreenName: 500
Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
of Api.timeline: 187 of Api.timeline: 187
else: 180 else: 180
reqs = maxReqs - token.apis[api].remaining reqs = maxReqs - token.apis[api].remaining

View File

@ -9,7 +9,6 @@ type
InternalError* = object of CatchableError InternalError* = object of CatchableError
Api* {.pure.} = enum Api* {.pure.} = enum
tweetDetail
userShow userShow
timeline timeline
search search
@ -177,6 +176,7 @@ type
available*: bool available*: bool
tombstone*: string tombstone*: string
location*: string location*: string
source*: string
stats*: TweetStats stats*: TweetStats
retweet*: Option[Tweet] retweet*: Option[Tweet]
attribution*: Option[User] attribution*: Option[User]

View File

@ -347,7 +347,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
renderQuote(tweet.quote.get(), prefs, path) renderQuote(tweet.quote.get(), prefs, path)
if mainTweet: if mainTweet:
p(class="tweet-published"): text &"{getTime(tweet)}" p(class="tweet-published"): text &"{getTime(tweet)} · {tweet.source}"
if tweet.mediaTags.len > 0: if tweet.mediaTags.len > 0:
renderMediaTags(tweet.mediaTags) renderMediaTags(tweet.mediaTags)