From 6fb039dd79b406e875af440be52b7e7fa5c4013a Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 21 Oct 2019 23:12:40 +0200 Subject: [PATCH] Optimize RSS requests by disabling media fetching --- src/api/list.nim | 4 ++-- src/api/search.nim | 4 ++-- src/api/timeline.nim | 13 +++++++------ src/routes/rss.nim | 8 +++++--- src/routes/timeline.nim | 10 +++++----- 5 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/api/list.nim b/src/api/list.nim index ea495ac..3adf75b 100644 --- a/src/api/list.nim +++ b/src/api/list.nim @@ -4,7 +4,7 @@ import sequtils, strutils, json, uri import ".."/[types, parser, parserutils, query] import utils, consts, timeline, search -proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} = +proc getListTimeline*(username, list, agent, after: string; media=true): Future[Timeline] {.async.} = let url = base / (listUrl % [username, list]) var params = toSeq({ @@ -17,7 +17,7 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {. params.add {"max_position": after} let json = await fetchJson(url ? params, genHeaders(agent, url)) - result = await finishTimeline(json, Query(), after, agent) + result = await finishTimeline(json, Query(), after, agent, media) if result.content.len == 0: return diff --git a/src/api/search.nim b/src/api/search.nim index 2a55c5a..0554d78 100644 --- a/src/api/search.nim +++ b/src/api/search.nim @@ -14,7 +14,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] = beginning: after.len == 0 ) -proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} = +proc getSearch*[T](query: Query; after, agent: string; media=true): Future[Result[T]] {.async.} = let kind = if query.kind == users: "users" else: "tweets" @@ -44,7 +44,7 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn if json == nil or not json.hasKey("items_html"): return when T is Tweet: - result = await finishTimeline(json, query, after, agent) + result = await finishTimeline(json, query, after, agent, media) elif T is Profile: let html = json["items_html"].to(string) result.hasMore = html != "\n" diff --git a/src/api/timeline.nim b/src/api/timeline.nim index bed761a..e62f8ae 100644 --- a/src/api/timeline.nim +++ b/src/api/timeline.nim @@ -9,7 +9,8 @@ proc getMedia(thread: Chain | Timeline; agent: string) {.async.} = getCards(thread, agent), getPolls(thread, agent)) -proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future[Timeline] {.async.} = +proc finishTimeline*(json: JsonNode; query: Query; after, agent: string; + media=true): Future[Timeline] {.async.} = result = getResult[Tweet](json, query, after) if json == nil: return @@ -19,10 +20,10 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future let html = parseHtml(json["items_html"].to(string)) let thread = parseChain(html) - await getMedia(thread, agent) + if media: await getMedia(thread, agent) result.content = thread.content -proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} = +proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} = var params = toSeq({ "include_available_features": "1", "include_entities": "1", @@ -36,9 +37,9 @@ proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} = let headers = genHeaders(agent, base / username, xml=true) let json = await fetchJson(base / (timelineUrl % username) ? params, headers) - result = await finishTimeline(json, Query(), after, agent) + result = await finishTimeline(json, Query(), after, agent, media) -proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} = +proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} = var url = base / username if after.len > 0: url = url ? {"max_position": after} @@ -49,5 +50,5 @@ proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Ti timeline = parseTimeline(html.select("#timeline > .stream-container"), after) profile = parseTimelineProfile(html) - await getMedia(timeline, agent) + if media: await getMedia(timeline, agent) result = (profile, timeline) diff --git a/src/routes/rss.nim b/src/routes/rss.nim index cf5ab89..b46a2b2 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -9,7 +9,9 @@ import ../views/general include "../views/rss.nimf" proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} = - let (profile, timeline, _) = await fetchSingleTimeline(name, "", getAgent(), query) + let (profile, timeline, _) = + await fetchSingleTimeline(name, "", getAgent(), query, media=false) + if timeline != nil: return renderTimelineRss(timeline, profile, hostname) @@ -28,7 +30,7 @@ proc createRssRouter*(cfg: Config) = if query.kind != tweets: resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg) - let tweets = await getSearch[Tweet](query, "", getAgent()) + let tweets = await getSearch[Tweet](query, "", getAgent(), media=false) respRss(renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg.hostname)) get "/@name/rss": @@ -49,5 +51,5 @@ proc createRssRouter*(cfg: Config) = get "/@name/lists/@list/rss": cond '.' notin @"name" - let list = await getListTimeline(@"name", @"list", getAgent(), "") + let list = await getListTimeline(@"name", @"list", getAgent(), "", media=false) respRss(renderListRss(list.content, @"name", @"list", cfg.hostname)) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 08632f7..9a269b2 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -13,8 +13,8 @@ export profile, timeline, status type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto]) -proc fetchSingleTimeline*(name, after, agent: string; - query: Query): Future[ProfileTimeline] {.async.} = +proc fetchSingleTimeline*(name, after, agent: string; query: Query; + media=true): Future[ProfileTimeline] {.async.} = let railFut = getPhotoRail(name, agent) var timeline: Timeline @@ -26,12 +26,12 @@ proc fetchSingleTimeline*(name, after, agent: string; if query.kind == posts: if cachedProfile.isSome: - timeline = await getTimeline(name, after, agent) + timeline = await getTimeline(name, after, agent, media) else: - (profile, timeline) = await getProfileAndTimeline(name, agent, after) + (profile, timeline) = await getProfileAndTimeline(name, agent, after, media) cache(profile) else: - var timelineFut = getSearch[Tweet](query, after, agent) + var timelineFut = getSearch[Tweet](query, after, agent, media) if cachedProfile.isNone: profile = await getCachedProfile(name, agent) timeline = await timelineFut