blankie
/
nitter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use search instead of old timeline endpoint

This commit is contained in:
Zed 2023-07-22 03:03:45 +02:00
parent cc5841df30
commit 50f821dbd8
7 changed files with 178 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/api.nim
View File

@ -33,12 +33,12 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
js = await fetch(url ? params, apiId) js = await fetch(url ? params, apiId)
result = parseGraphTimeline(js, "user", after) result = parseGraphTimeline(js, "user", after)
proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} = # proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} =
if id.len == 0: return # if id.len == 0: return
let # let
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after) # ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
url = oldUserTweets / (id & ".json") ? ps # url = oldUserTweets / (id & ".json") ? ps
result = parseTimeline(await fetch(url, Api.timeline), after) # result = parseTimeline(await fetch(url, Api.timeline), after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return if id.len == 0: return
@ -123,20 +123,22 @@ proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} =
result.tweets.query = query result.tweets.query = query
proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
let q = genQueryParam(query) var q = genQueryParam(query)
if q.len == 0 or q == emptyQuery: if q.len == 0 or q == emptyQuery:
return Timeline(query: query, beginning: true) return Timeline(query: query, beginning: true)
if after.len > 0:
q &= " max_id:" & after
let url = tweetSearch ? genParams({ let url = tweetSearch ? genParams({
"q": q, "q": q ,
"tweet_search_mode": "live", "modules": "status",
"max_id": after "result_type": "recent",
}) })
result = parseTweetSearch(await fetch(url, Api.search)) result = parseTweetSearch(await fetch(url, Api.search), after)
result.query = query result.query = query
if after.len == 0:
result.beginning = true
proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} = proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
if query.text.len == 0: if query.text.len == 0:

2
src/apiutils.nim
View File

@ -16,7 +16,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
for p in pars: for p in pars:
result &= p result &= p
if ext: if ext:
result &= ("ext", "mediaStats") result &= ("ext", "mediaStats,isBlueVerified,isVerified,blue,blueVerified")
result &= ("include_ext_alt_text", "1") result &= ("include_ext_alt_text", "1")
result &= ("include_ext_media_availability", "1") result &= ("include_ext_media_availability", "1")
if count.len > 0: if count.len > 0:

63
src/consts.nim
View File

@ -9,9 +9,9 @@ const
photoRail* = api / "1.1/statuses/media_timeline.json" photoRail* = api / "1.1/statuses/media_timeline.json"
userSearch* = api / "1.1/users/search.json" userSearch* = api / "1.1/users/search.json"
tweetSearch* = api / "1.1/search/tweets.json" tweetSearch* = api / "1.1/search/universal.json"
oldUserTweets* = api / "2/timeline/profile" # oldUserTweets* = api / "2/timeline/profile"
graphql = api / "graphql" graphql = api / "graphql"
graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
@ -28,27 +28,28 @@ const
graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
timelineParams* = { timelineParams* = {
"include_profile_interstitial_type": "0", "cards_platform": "Web-13",
"include_blocking": "0",
"include_blocked_by": "0",
"include_followed_by": "0",
"include_want_retweets": "0",
"include_mute_edge": "0",
"include_can_dm": "0",
"include_can_media_tag": "1",
"include_ext_is_blue_verified": "1",
"skip_status": "1",
"cards_platform": "Web-12",
"include_cards": "1",
"include_composer_source": "0",
"include_reply_count": "1",
"tweet_mode": "extended", "tweet_mode": "extended",
"include_entities": "1", "ui_lang": "en-US",
"include_user_entities": "1",
"include_ext_media_color": "0",
"send_error_codes": "1", "send_error_codes": "1",
"simple_quoted_tweet": "1", "simple_quoted_tweet": "1",
"include_quote_count": "1" "skip_status": "1",
"include_blocked_by": "0",
"include_blocking": "0",
"include_can_dm": "0",
"include_can_media_tag": "1",
"include_cards": "1",
"include_composer_source": "0",
"include_entities": "1",
"include_ext_is_blue_verified": "1",
"include_ext_media_color": "0",
"include_followed_by": "0",
"include_mute_edge": "0",
"include_profile_interstitial_type": "0",
"include_quote_count": "1",
"include_reply_count": "1",
"include_user_entities": "1",
"include_want_retweets": "0",
}.toSeq }.toSeq
gqlFeatures* = """{ gqlFeatures* = """{
@ -100,17 +101,17 @@ const
"includeHasBirdwatchNotes": false "includeHasBirdwatchNotes": false
}""" }"""
oldUserTweetsVariables* = """{ # oldUserTweetsVariables* = """{
"userId": "$1", $2 # "userId": "$1", $2
"count": 20, # "count": 20,
"includePromotedContent": false, # "includePromotedContent": false,
"withDownvotePerspective": false, # "withDownvotePerspective": false,
"withReactionsMetadata": false, # "withReactionsMetadata": false,
"withReactionsPerspective": false, # "withReactionsPerspective": false,
"withVoice": false, # "withVoice": false,
"withV2Timeline": true # "withV2Timeline": true
} # }
""" # """
userTweetsVariables* = """{ userTweetsVariables* = """{
"rest_id": "$1", $2 "rest_id": "$1", $2

162
src/parser.nim
View File

@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import strutils, options, tables, times, math import strutils, options, times, math
import packedjson, packedjson/deserialiser import packedjson, packedjson/deserialiser
import types, parserutils, utils import types, parserutils, utils
import experimental/parser/unifiedcard import experimental/parser/unifiedcard
@ -295,11 +295,14 @@ proc parseLegacyTweet(js: JsonNode): Tweet =
if result.quote.isSome: if result.quote.isSome:
result.quote = some parseLegacyTweet(js{"quoted_status"}) result.quote = some parseLegacyTweet(js{"quoted_status"})
proc parseTweetSearch*(js: JsonNode): Timeline = proc parseTweetSearch*(js: JsonNode; after=""): Timeline =
if js.kind == JNull or "statuses" notin js: result.beginning = after.len == 0
return Timeline(beginning: true)
for tweet in js{"statuses"}: if js.kind == JNull or "modules" notin js or js{"modules"}.len == 0:
return
for item in js{"modules"}:
with tweet, item{"status", "data"}:
let parsed = parseLegacyTweet(tweet) let parsed = parseLegacyTweet(tweet)
if parsed.retweet.isSome: if parsed.retweet.isSome:
@ -307,98 +310,97 @@ proc parseTweetSearch*(js: JsonNode): Timeline =
result.content.add @[parsed] result.content.add @[parsed]
let cursor = js{"search_metadata", "next_results"}.getStr if result.content.len > 0:
if cursor.len > 0 and "max_id" in cursor: result.bottom = $(result.content[^1][0].id - 1)
result.bottom = cursor[cursor.find("=") + 1 .. cursor.find("&q=")]
proc finalizeTweet(global: GlobalObjects; id: string): Tweet = # proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
let intId = if id.len > 0: parseBiggestInt(id) else: 0 # let intId = if id.len > 0: parseBiggestInt(id) else: 0
result = global.tweets.getOrDefault(id, Tweet(id: intId)) # result = global.tweets.getOrDefault(id, Tweet(id: intId))
if result.quote.isSome: # if result.quote.isSome:
let quote = get(result.quote).id # let quote = get(result.quote).id
if $quote in global.tweets: # if $quote in global.tweets:
result.quote = some global.tweets[$quote] # result.quote = some global.tweets[$quote]
else: # else:
result.quote = some Tweet() # result.quote = some Tweet()
if result.retweet.isSome: # if result.retweet.isSome:
let rt = get(result.retweet).id # let rt = get(result.retweet).id
if $rt in global.tweets: # if $rt in global.tweets:
result.retweet = some finalizeTweet(global, $rt) # result.retweet = some finalizeTweet(global, $rt)
else: # else:
result.retweet = some Tweet() # result.retweet = some Tweet()
proc parsePin(js: JsonNode; global: GlobalObjects): Tweet = # proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
let pin = js{"pinEntry", "entry", "entryId"}.getStr # let pin = js{"pinEntry", "entry", "entryId"}.getStr
if pin.len == 0: return # if pin.len == 0: return
let id = pin.getId # let id = pin.getId
if id notin global.tweets: return # if id notin global.tweets: return
global.tweets[id].pinned = true # global.tweets[id].pinned = true
return finalizeTweet(global, id) # return finalizeTweet(global, id)
proc parseGlobalObjects(js: JsonNode): GlobalObjects = # proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result = GlobalObjects() # result = GlobalObjects()
let # let
tweets = ? js{"globalObjects", "tweets"} # tweets = ? js{"globalObjects", "tweets"}
users = ? js{"globalObjects", "users"} # users = ? js{"globalObjects", "users"}
for k, v in users: # for k, v in users:
result.users[k] = parseUser(v, k) # result.users[k] = parseUser(v, k)
for k, v in tweets: # for k, v in tweets:
var tweet = parseTweet(v, v{"card"}) # var tweet = parseTweet(v, v{"card"})
if tweet.user.id in result.users: # if tweet.user.id in result.users:
tweet.user = result.users[tweet.user.id] # tweet.user = result.users[tweet.user.id]
result.tweets[k] = tweet # result.tweets[k] = tweet
proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) = # proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) =
if js.kind != JArray or js.len == 0: # if js.kind != JArray or js.len == 0:
return # return
for i in js: # for i in js:
if res.tweets.beginning and i{"pinEntry"}.notNull: # if res.tweets.beginning and i{"pinEntry"}.notNull:
with pin, parsePin(i, global): # with pin, parsePin(i, global):
res.pinned = some pin # res.pinned = some pin
with r, i{"replaceEntry", "entry"}: # with r, i{"replaceEntry", "entry"}:
if "top" in r{"entryId"}.getStr: # if "top" in r{"entryId"}.getStr:
res.tweets.top = r.getCursor # res.tweets.top = r.getCursor
elif "bottom" in r{"entryId"}.getStr: # elif "bottom" in r{"entryId"}.getStr:
res.tweets.bottom = r.getCursor # res.tweets.bottom = r.getCursor
proc parseTimeline*(js: JsonNode; after=""): Profile = # proc parseTimeline*(js: JsonNode; after=""): Profile =
result = Profile(tweets: Timeline(beginning: after.len == 0)) # result = Profile(tweets: Timeline(beginning: after.len == 0))
let global = parseGlobalObjects(? js) # let global = parseGlobalObjects(? js)
let instructions = ? js{"timeline", "instructions"} # let instructions = ? js{"timeline", "instructions"}
if instructions.len == 0: return # if instructions.len == 0: return
result.parseInstructions(global, instructions) # result.parseInstructions(global, instructions)
var entries: JsonNode # var entries: JsonNode
for i in instructions: # for i in instructions:
if "addEntries" in i: # if "addEntries" in i:
entries = i{"addEntries", "entries"} # entries = i{"addEntries", "entries"}
for e in ? entries: # for e in ? entries:
let entry = e{"entryId"}.getStr # let entry = e{"entryId"}.getStr
if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry: # if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId) # let tweet = finalizeTweet(global, e.getEntryId)
if not tweet.available: continue # if not tweet.available: continue
result.tweets.content.add tweet # result.tweets.content.add tweet
elif "cursor-top" in entry: # elif "cursor-top" in entry:
result.tweets.top = e.getCursor # result.tweets.top = e.getCursor
elif "cursor-bottom" in entry: # elif "cursor-bottom" in entry:
result.tweets.bottom = e.getCursor # result.tweets.bottom = e.getCursor
elif entry.startsWith("sq-cursor"): # elif entry.startsWith("sq-cursor"):
with cursor, e{"content", "operation", "cursor"}: # with cursor, e{"content", "operation", "cursor"}:
if cursor{"cursorType"}.getStr == "Bottom": # if cursor{"cursorType"}.getStr == "Bottom":
result.tweets.bottom = cursor{"value"}.getStr # result.tweets.bottom = cursor{"value"}.getStr
else: # else:
result.tweets.top = cursor{"value"}.getStr # result.tweets.top = cursor{"value"}.getStr
proc parsePhotoRail*(js: JsonNode): PhotoRail = proc parsePhotoRail*(js: JsonNode): PhotoRail =
with error, js{"error"}: with error, js{"error"}:

12
src/routes/timeline.nim
View File

@ -53,7 +53,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
result = result =
case query.kind case query.kind
of posts: await getTimeline(userId, after) # of posts: await getTimeline(userId, after)
of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
of media: await getGraphUserTweets(userId, TimelineKind.media, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after)
else: Profile(tweets: await getTweetSearch(query, after)) else: Profile(tweets: await getTweetSearch(query, after))
@ -61,10 +61,18 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
result.user = await user result.user = await user
result.photoRail = await rail result.photoRail = await rail
result.tweets.query = query
if result.user.protected or result.user.suspended: if result.user.protected or result.user.suspended:
return return
result.tweets.query = query if not skipPinned and query.kind == posts and
result.user.pinnedTweet > 0 and after.len == 0:
let tweet = await getCachedTweet(result.user.pinnedTweet)
if not tweet.isNil:
tweet.pinned = true
tweet.user = result.user
result.pinned = some tweet
proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
rss, after: string): Future[string] {.async.} = rss, after: string): Future[string] {.async.} =

8
tests/test_timeline.py
View File

@ -1,12 +1,12 @@
from base import BaseTestCase, Timeline from base import BaseTestCase, Timeline
from parameterized import parameterized from parameterized import parameterized
normal = [['mobile_test'], ['mobile_test_2']] normal = [['jack'], ['elonmusk']]
after = [['mobile_test', 'HBaAgJPsqtGNhA0AAA%3D%3D'], after = [['jack', '1681686036294803456'],
['mobile_test_2', 'HBaAgJPsqtGNhA0AAA%3D%3D']] ['elonmusk', '1681686036294803456']]
no_more = [['mobile_test_8?cursor=HBaAwJCsk%2F6%2FtgQAAA%3D%3D']] no_more = [['mobile_test_8?cursor=1000']]
empty = [['emptyuser'], ['mobile_test_10']] empty = [['emptyuser'], ['mobile_test_10']]

36
tests/test_tweet.py
View File

@ -80,16 +80,16 @@ retweet = [
class TweetTest(BaseTestCase): class TweetTest(BaseTestCase):
@parameterized.expand(timeline) # @parameterized.expand(timeline)
def test_timeline(self, index, fullname, username, date, tid, text): # def test_timeline(self, index, fullname, username, date, tid, text):
self.open_nitter(username) # self.open_nitter(username)
tweet = get_timeline_tweet(index) # tweet = get_timeline_tweet(index)
self.assert_exact_text(fullname, tweet.fullname) # self.assert_exact_text(fullname, tweet.fullname)
self.assert_exact_text('@' + username, tweet.username) # self.assert_exact_text('@' + username, tweet.username)
self.assert_exact_text(date, tweet.date) # self.assert_exact_text(date, tweet.date)
self.assert_text(text, tweet.text) # self.assert_text(text, tweet.text)
permalink = self.find_element(tweet.date + ' a') # permalink = self.find_element(tweet.date + ' a')
self.assertIn(tid, permalink.get_attribute('href')) # self.assertIn(tid, permalink.get_attribute('href'))
@parameterized.expand(status) @parameterized.expand(status)
def test_status(self, tid, fullname, username, date, text): def test_status(self, tid, fullname, username, date, text):
@ -123,14 +123,14 @@ class TweetTest(BaseTestCase):
link = self.find_link_text(f'@{un}') link = self.find_link_text(f'@{un}')
self.assertIn(f'/{un}', link.get_property('href')) self.assertIn(f'/{un}', link.get_property('href'))
@parameterized.expand(retweet) # @parameterized.expand(retweet)
def test_retweet(self, index, url, retweet_by, fullname, username, text): # def test_retweet(self, index, url, retweet_by, fullname, username, text):
self.open_nitter(url) # self.open_nitter(url)
tweet = get_timeline_tweet(index) # tweet = get_timeline_tweet(index)
self.assert_text(f'{retweet_by} retweeted', tweet.retweet) # self.assert_text(f'{retweet_by} retweeted', tweet.retweet)
self.assert_text(text, tweet.text) # self.assert_text(text, tweet.text)
self.assert_exact_text(fullname, tweet.fullname) # self.assert_exact_text(fullname, tweet.fullname)
self.assert_exact_text(username, tweet.username) # self.assert_exact_text(username, tweet.username)
@parameterized.expand(invalid) @parameterized.expand(invalid)
def test_invalid_id(self, tweet): def test_invalid_id(self, tweet):