From cc5841df308506356d329662d0f0c2ec4713a35c Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 21 Jul 2023 18:56:39 +0200 Subject: [PATCH] Use old timeline endpoint --- src/api.nim | 9 ++++++++- src/consts.nim | 14 ++++++++++++++ src/parser.nim | 36 +++++++++++++++++++++++++----------- src/routes/timeline.nim | 2 +- src/tokens.nim | 6 ++++-- src/types.nim | 1 + 6 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/api.nim b/src/api.nim index 60af68d..e8d0830 100644 --- a/src/api.nim +++ b/src/api.nim @@ -33,6 +33,13 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi js = await fetch(url ? params, apiId) result = parseGraphTimeline(js, "user", after) +proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} = + if id.len == 0: return + let + ps = genParams({"userId": id, "include_tweet_replies": $replies}, after) + url = oldUserTweets / (id & ".json") ? ps + result = parseTimeline(await fetch(url, Api.timeline), after) + proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let @@ -155,7 +162,7 @@ proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} = ps = genParams({"screen_name": name, "trim_user": "true"}, count="18", ext=false) url = photoRail ? ps - result = parsePhotoRail(await fetch(url, Api.timeline)) + result = parsePhotoRail(await fetch(url, Api.photoRail)) proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} = let client = newAsyncHttpClient(maxRedirects=0) diff --git a/src/consts.nim b/src/consts.nim index 8dd1b14..7ba09fb 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -11,6 +11,8 @@ const userSearch* = api / "1.1/users/search.json" tweetSearch* = api / "1.1/search/tweets.json" + oldUserTweets* = api / "2/timeline/profile" + graphql = api / "graphql" graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" graphUserById* = graphql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery" @@ -98,6 +100,18 @@ const "includeHasBirdwatchNotes": false }""" + oldUserTweetsVariables* = """{ + "userId": "$1", $2 + "count": 20, + "includePromotedContent": false, + "withDownvotePerspective": false, + "withReactionsMetadata": false, + "withReactionsPerspective": false, + "withVoice": false, + "withV2Timeline": true +} +""" + userTweetsVariables* = """{ "rest_id": "$1", $2 "count": 20 diff --git a/src/parser.nim b/src/parser.nim index b988cf7..193d77f 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -329,6 +329,16 @@ proc finalizeTweet(global: GlobalObjects; id: string): Tweet = else: result.retweet = some Tweet() +proc parsePin(js: JsonNode; global: GlobalObjects): Tweet = + let pin = js{"pinEntry", "entry", "entryId"}.getStr + if pin.len == 0: return + + let id = pin.getId + if id notin global.tweets: return + + global.tweets[id].pinned = true + return finalizeTweet(global, id) + proc parseGlobalObjects(js: JsonNode): GlobalObjects = result = GlobalObjects() let @@ -339,24 +349,28 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects = result.users[k] = parseUser(v, k) for k, v in tweets: - var tweet = parseTweet(v, v{"tweet_card"}) + var tweet = parseTweet(v, v{"card"}) if tweet.user.id in result.users: tweet.user = result.users[tweet.user.id] result.tweets[k] = tweet -proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) = +proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) = if js.kind != JArray or js.len == 0: return for i in js: + if res.tweets.beginning and i{"pinEntry"}.notNull: + with pin, parsePin(i, global): + res.pinned = some pin + with r, i{"replaceEntry", "entry"}: if "top" in r{"entryId"}.getStr: - res.top = r.getCursor + res.tweets.top = r.getCursor elif "bottom" in r{"entryId"}.getStr: - res.bottom = r.getCursor + res.tweets.bottom = r.getCursor -proc parseTimeline*(js: JsonNode; after=""): Timeline = - result = Timeline(beginning: after.len == 0) +proc parseTimeline*(js: JsonNode; after=""): Profile = + result = Profile(tweets: Timeline(beginning: after.len == 0)) let global = parseGlobalObjects(? js) let instructions = ? js{"timeline", "instructions"} @@ -374,17 +388,17 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline = if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry: let tweet = finalizeTweet(global, e.getEntryId) if not tweet.available: continue - result.content.add tweet + result.tweets.content.add tweet elif "cursor-top" in entry: - result.top = e.getCursor + result.tweets.top = e.getCursor elif "cursor-bottom" in entry: - result.bottom = e.getCursor + result.tweets.bottom = e.getCursor elif entry.startsWith("sq-cursor"): with cursor, e{"content", "operation", "cursor"}: if cursor{"cursorType"}.getStr == "Bottom": - result.bottom = cursor{"value"}.getStr + result.tweets.bottom = cursor{"value"}.getStr else: - result.top = cursor{"value"}.getStr + result.tweets.top = cursor{"value"}.getStr proc parsePhotoRail*(js: JsonNode): PhotoRail = with error, js{"error"}: diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index ef3d012..b574631 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -53,7 +53,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; result = case query.kind - of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after) + of posts: await getTimeline(userId, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after) else: Profile(tweets: await getTweetSearch(query, after)) diff --git a/src/tokens.nim b/src/tokens.nim index 531f557..8a25257 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -41,8 +41,10 @@ proc getPoolJson*(): JsonNode = let maxReqs = case api - of Api.timeline, Api.search: 180 - of Api.userTweets, Api.userTweetsAndReplies, Api.userRestId, + of Api.photoRail, Api.search: 180 + of Api.timeline: 187 + of Api.userTweets: 300 + of Api.userTweetsAndReplies, Api.userRestId, Api.userScreenName, Api.tweetDetail, Api.tweetResult: 500 of Api.list, Api.listTweets, Api.listMembers, Api.listBySlug, Api.userMedia: 500 of Api.userSearch: 900 diff --git a/src/types.nim b/src/types.nim index f7d5f6b..5db9ec3 100644 --- a/src/types.nim +++ b/src/types.nim @@ -18,6 +18,7 @@ type tweetDetail tweetResult timeline + photoRail search userSearch list