Use old api to optimize photo rails

This commit is contained in:
Zed 2020-06-17 00:20:34 +02:00
parent 7e40af5611
commit 77da2e32c6
7 changed files with 37 additions and 40 deletions

View File

@ -49,9 +49,12 @@ proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
let url = mediaTimeline / (id & ".json") ? genParams(cursor=after) let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
result = parseTimeline(await fetch(url), after) result = parseTimeline(await fetch(url), after)
proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} = proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
let url = mediaTimeline / (id & ".json") ? genParams() let
result = parsePhotoRail(await fetch(url)) ps = genParams({"screen_name": name, "trim_user": "true"},
count="18", ext=false)
url = photoRail ? ps
result = parsePhotoRail(await fetch(url, oldApi=true))
proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
when T is Profile: when T is Profile:

View File

@ -4,13 +4,17 @@ import types, tokens, consts, parserutils
const rl = "x-rate-limit-" const rl = "x-rate-limit-"
proc genParams*(pars: openarray[(string, string)] = @[]; proc genParams*(pars: openarray[(string, string)] = @[]; cursor="";
cursor=""): seq[(string, string)] = count="20"; ext=true): seq[(string, string)] =
result = timelineParams result = timelineParams
for p in pars: for p in pars:
result &= p result &= p
if ext:
result &= ("ext", "mediaStats")
if cursor.len > 0: if cursor.len > 0:
result &= ("cursor", cursor) result &= ("cursor", cursor)
if count.len > 0:
result &= ("count", count)
proc genHeaders*(token: Token = nil): HttpHeaders = proc genHeaders*(token: Token = nil): HttpHeaders =
result = newHttpHeaders({ result = newHttpHeaders({
@ -34,11 +38,11 @@ proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} =
resp = await client.get($url) resp = await client.get($url)
body = await resp.body body = await resp.body
if not body.startsWith('{'): if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
echo resp.status, ": ", body echo resp.status, ": ", body
result = newJNull() result = newJNull()
else:
result = parseJson(body)
if not oldApi and resp.headers.hasKey(rl & "limit"): if not oldApi and resp.headers.hasKey(rl & "limit"):
token.remaining = parseInt(resp.headers[rl & "remaining"]) token.remaining = parseInt(resp.headers[rl & "remaining"])

View File

@ -14,6 +14,7 @@ const
listTimeline* = timelineApi / "list.json" listTimeline* = timelineApi / "list.json"
listMembers* = api / "1.1/lists/members.json" listMembers* = api / "1.1/lists/members.json"
userLookup* = api / "1.1/users/show.json" userLookup* = api / "1.1/users/show.json"
photoRail* = api / "1.1/statuses/media_timeline.json"
tweet* = timelineApi / "conversation" tweet* = timelineApi / "conversation"
search* = api / "2/search/adaptive.json" search* = api / "2/search/adaptive.json"
@ -21,7 +22,7 @@ const
"include_profile_interstitial_type": "0", "include_profile_interstitial_type": "0",
"include_blocking": "0", "include_blocking": "0",
"include_blocked_by": "0", "include_blocked_by": "0",
"include_followed_by": "1", "include_followed_by": "0",
"include_want_retweets": "0", "include_want_retweets": "0",
"include_mute_edge": "0", "include_mute_edge": "0",
"include_can_dm": "0", "include_can_dm": "0",
@ -39,8 +40,7 @@ const
"include_ext_media_availability": "true", "include_ext_media_availability": "true",
"send_error_codes": "true", "send_error_codes": "true",
"simple_quoted_tweet": "true", "simple_quoted_tweet": "true",
"count": "20", "ext": "mediaStats",
"ext": "mediaStats,highlightedLabel,cameraMoment",
"include_quote_count": "true" "include_quote_count": "true"
}.toSeq }.toSeq

View File

@ -412,24 +412,14 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
result.bottom = e.getCursor result.bottom = e.getCursor
proc parsePhotoRail*(js: JsonNode): PhotoRail = proc parsePhotoRail*(js: JsonNode): PhotoRail =
for tweet in js:
let let
tweets = ? js{"globalObjects", "tweets"} t = parseTweet(tweet)
instructions = ? js{"timeline", "instructions"} url = if t.photos.len > 0: t.photos[0]
elif t.video.isSome: get(t.video).thumb
if instructions.len == 0 or tweets.len == 0: return elif t.gif.isSome: get(t.gif).thumb
elif t.card.isSome: get(t.card).image
for e in instructions[0]{"addEntries", "entries"}:
if result.len == 16: break
let entry = e{"entryId"}.getStr
if "tweet" in entry:
let id = entry.getId
if id notin tweets: continue
let tweet = parseTweet(tweets{id})
var url = if tweet.photos.len > 0: tweet.photos[0]
elif tweet.video.isSome: get(tweet.video).thumb
elif tweet.gif.isSome: get(tweet.gif).thumb
elif tweet.card.isSome: get(tweet.card).image
else: "" else: ""
if url.len == 0: continue if url.len == 0: continue
result.add GalleryPhoto(url: url, tweetId: $tweet.id) result.add GalleryPhoto(url: url, tweetId: $t.id)

View File

@ -54,8 +54,8 @@ proc setex(key: string; time: int; data: string) {.async.} =
proc cache*(data: List) {.async.} = proc cache*(data: List) {.async.} =
await setex(data.toKey, listCacheTime, compress(freeze(data))) await setex(data.toKey, listCacheTime, compress(freeze(data)))
proc cache*(data: PhotoRail; id: string) {.async.} = proc cache*(data: PhotoRail; name: string) {.async.} =
await setex("pr:" & id, baseCacheTime, compress(freeze(data))) await setex("pr:" & name, baseCacheTime, compress(freeze(data)))
proc cache*(data: Profile) {.async.} = proc cache*(data: Profile) {.async.} =
if data.username.len == 0: return if data.username.len == 0: return
@ -94,14 +94,14 @@ proc getCachedProfile*(username: string; fetch=true;
result = await getProfile(username) result = await getProfile(username)
if cache: await cache(result) if cache: await cache(result)
proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} = proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if id.len == 0: return if name.len == 0: return
let rail = await get("pr:" & toLower(id)) let rail = await get("pr:" & toLower(name))
if rail != redisNil: if rail != redisNil:
uncompress(rail).thaw(result) uncompress(rail).thaw(result)
else: else:
result = await getPhotoRail(id) result = await getPhotoRail(name)
await cache(result, id) await cache(result, name)
proc getCachedList*(username=""; name=""; id=""): Future[List] {.async.} = proc getCachedList*(username=""; name=""; id=""): Future[List] {.async.} =
let list = if id.len > 0: redisNil let list = if id.len > 0: redisNil

View File

@ -46,7 +46,7 @@ proc fetchSingleTimeline*(after: string; query: Query; skipRail=false):
rail = newFuture[PhotoRail]() rail = newFuture[PhotoRail]()
rail.complete(@[]) rail.complete(@[])
else: else:
rail = getCachedPhotoRail(profileId) rail = getCachedPhotoRail(name)
var timeline = var timeline =
case query.kind case query.kind

View File

@ -87,7 +87,7 @@ proc genDate*(pref, state: string): VNode =
proc genImg*(url: string; class=""): VNode = proc genImg*(url: string; class=""): VNode =
buildHtml(): buildHtml():
img(src=getPicUrl(url), class=class, alt="Image") img(src=getPicUrl(url), class=class, alt="")
proc getTabClass*(query: Query; tab: QueryKind): string = proc getTabClass*(query: Query; tab: QueryKind): string =
result = "tab-item" result = "tab-item"