From 77da2e32c68104d5565e14165ff0cc1a77e2f942 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 17 Jun 2020 00:20:34 +0200 Subject: [PATCH] Use old api to optimize photo rails --- src/api.nim | 9 ++++++--- src/apiutils.nim | 14 +++++++++----- src/consts.nim | 6 +++--- src/parser.nim | 30 ++++++++++-------------------- src/redis_cache.nim | 14 +++++++------- src/routes/timeline.nim | 2 +- src/views/renderutils.nim | 2 +- 7 files changed, 37 insertions(+), 40 deletions(-) diff --git a/src/api.nim b/src/api.nim index 9c033c0..a18e998 100644 --- a/src/api.nim +++ b/src/api.nim @@ -49,9 +49,12 @@ proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} = let url = mediaTimeline / (id & ".json") ? genParams(cursor=after) result = parseTimeline(await fetch(url), after) -proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} = - let url = mediaTimeline / (id & ".json") ? genParams() - result = parsePhotoRail(await fetch(url)) +proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} = + let + ps = genParams({"screen_name": name, "trim_user": "true"}, + count="18", ext=false) + url = photoRail ? ps + result = parsePhotoRail(await fetch(url, oldApi=true)) proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} = when T is Profile: diff --git a/src/apiutils.nim b/src/apiutils.nim index 60145d6..12a915e 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -4,13 +4,17 @@ import types, tokens, consts, parserutils const rl = "x-rate-limit-" -proc genParams*(pars: openarray[(string, string)] = @[]; - cursor=""): seq[(string, string)] = +proc genParams*(pars: openarray[(string, string)] = @[]; cursor=""; + count="20"; ext=true): seq[(string, string)] = result = timelineParams for p in pars: result &= p + if ext: + result &= ("ext", "mediaStats") if cursor.len > 0: result &= ("cursor", cursor) + if count.len > 0: + result &= ("count", count) proc genHeaders*(token: Token = nil): HttpHeaders = result = newHttpHeaders({ @@ -34,11 +38,11 @@ proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} = resp = await client.get($url) body = await resp.body - if not body.startsWith('{'): + if body.startsWith('{') or body.startsWith('['): + result = parseJson(body) + else: echo resp.status, ": ", body result = newJNull() - else: - result = parseJson(body) if not oldApi and resp.headers.hasKey(rl & "limit"): token.remaining = parseInt(resp.headers[rl & "remaining"]) diff --git a/src/consts.nim b/src/consts.nim index 2846d47..b303190 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -14,6 +14,7 @@ const listTimeline* = timelineApi / "list.json" listMembers* = api / "1.1/lists/members.json" userLookup* = api / "1.1/users/show.json" + photoRail* = api / "1.1/statuses/media_timeline.json" tweet* = timelineApi / "conversation" search* = api / "2/search/adaptive.json" @@ -21,7 +22,7 @@ const "include_profile_interstitial_type": "0", "include_blocking": "0", "include_blocked_by": "0", - "include_followed_by": "1", + "include_followed_by": "0", "include_want_retweets": "0", "include_mute_edge": "0", "include_can_dm": "0", @@ -39,8 +40,7 @@ const "include_ext_media_availability": "true", "send_error_codes": "true", "simple_quoted_tweet": "true", - "count": "20", - "ext": "mediaStats,highlightedLabel,cameraMoment", + "ext": "mediaStats", "include_quote_count": "true" }.toSeq diff --git a/src/parser.nim b/src/parser.nim index ee23749..a4f1649 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -412,24 +412,14 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline = result.bottom = e.getCursor proc parsePhotoRail*(js: JsonNode): PhotoRail = - let - tweets = ? js{"globalObjects", "tweets"} - instructions = ? js{"timeline", "instructions"} + for tweet in js: + let + t = parseTweet(tweet) + url = if t.photos.len > 0: t.photos[0] + elif t.video.isSome: get(t.video).thumb + elif t.gif.isSome: get(t.gif).thumb + elif t.card.isSome: get(t.card).image + else: "" - if instructions.len == 0 or tweets.len == 0: return - - for e in instructions[0]{"addEntries", "entries"}: - if result.len == 16: break - let entry = e{"entryId"}.getStr - if "tweet" in entry: - let id = entry.getId - if id notin tweets: continue - let tweet = parseTweet(tweets{id}) - var url = if tweet.photos.len > 0: tweet.photos[0] - elif tweet.video.isSome: get(tweet.video).thumb - elif tweet.gif.isSome: get(tweet.gif).thumb - elif tweet.card.isSome: get(tweet.card).image - else: "" - - if url.len == 0: continue - result.add GalleryPhoto(url: url, tweetId: $tweet.id) + if url.len == 0: continue + result.add GalleryPhoto(url: url, tweetId: $t.id) diff --git a/src/redis_cache.nim b/src/redis_cache.nim index 660b0e3..e523cdd 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -54,8 +54,8 @@ proc setex(key: string; time: int; data: string) {.async.} = proc cache*(data: List) {.async.} = await setex(data.toKey, listCacheTime, compress(freeze(data))) -proc cache*(data: PhotoRail; id: string) {.async.} = - await setex("pr:" & id, baseCacheTime, compress(freeze(data))) +proc cache*(data: PhotoRail; name: string) {.async.} = + await setex("pr:" & name, baseCacheTime, compress(freeze(data))) proc cache*(data: Profile) {.async.} = if data.username.len == 0: return @@ -94,14 +94,14 @@ proc getCachedProfile*(username: string; fetch=true; result = await getProfile(username) if cache: await cache(result) -proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} = - if id.len == 0: return - let rail = await get("pr:" & toLower(id)) +proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} = + if name.len == 0: return + let rail = await get("pr:" & toLower(name)) if rail != redisNil: uncompress(rail).thaw(result) else: - result = await getPhotoRail(id) - await cache(result, id) + result = await getPhotoRail(name) + await cache(result, name) proc getCachedList*(username=""; name=""; id=""): Future[List] {.async.} = let list = if id.len > 0: redisNil diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 5e9b14c..edfaea2 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -46,7 +46,7 @@ proc fetchSingleTimeline*(after: string; query: Query; skipRail=false): rail = newFuture[PhotoRail]() rail.complete(@[]) else: - rail = getCachedPhotoRail(profileId) + rail = getCachedPhotoRail(name) var timeline = case query.kind diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index db76cab..1d428ac 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -87,7 +87,7 @@ proc genDate*(pref, state: string): VNode = proc genImg*(url: string; class=""): VNode = buildHtml(): - img(src=getPicUrl(url), class=class, alt="Image") + img(src=getPicUrl(url), class=class, alt="") proc getTabClass*(query: Query; tab: QueryKind): string = result = "tab-item"