Optimize RSS requests by disabling media fetching
This commit is contained in:
parent
453beff09d
commit
6fb039dd79
|
@ -4,7 +4,7 @@ import sequtils, strutils, json, uri
|
|||
import ".."/[types, parser, parserutils, query]
|
||||
import utils, consts, timeline, search
|
||||
|
||||
proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} =
|
||||
proc getListTimeline*(username, list, agent, after: string; media=true): Future[Timeline] {.async.} =
|
||||
let url = base / (listUrl % [username, list])
|
||||
|
||||
var params = toSeq({
|
||||
|
@ -17,7 +17,7 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.
|
|||
params.add {"max_position": after}
|
||||
|
||||
let json = await fetchJson(url ? params, genHeaders(agent, url))
|
||||
result = await finishTimeline(json, Query(), after, agent)
|
||||
result = await finishTimeline(json, Query(), after, agent, media)
|
||||
if result.content.len == 0:
|
||||
return
|
||||
|
||||
|
|
|
@ -14,7 +14,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
|
|||
beginning: after.len == 0
|
||||
)
|
||||
|
||||
proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
|
||||
proc getSearch*[T](query: Query; after, agent: string; media=true): Future[Result[T]] {.async.} =
|
||||
let
|
||||
kind = if query.kind == users: "users" else: "tweets"
|
||||
|
||||
|
@ -44,7 +44,7 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn
|
|||
if json == nil or not json.hasKey("items_html"): return
|
||||
|
||||
when T is Tweet:
|
||||
result = await finishTimeline(json, query, after, agent)
|
||||
result = await finishTimeline(json, query, after, agent, media)
|
||||
elif T is Profile:
|
||||
let html = json["items_html"].to(string)
|
||||
result.hasMore = html != "\n"
|
||||
|
|
|
@ -9,7 +9,8 @@ proc getMedia(thread: Chain | Timeline; agent: string) {.async.} =
|
|||
getCards(thread, agent),
|
||||
getPolls(thread, agent))
|
||||
|
||||
proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future[Timeline] {.async.} =
|
||||
proc finishTimeline*(json: JsonNode; query: Query; after, agent: string;
|
||||
media=true): Future[Timeline] {.async.} =
|
||||
result = getResult[Tweet](json, query, after)
|
||||
if json == nil: return
|
||||
|
||||
|
@ -19,10 +20,10 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future
|
|||
let html = parseHtml(json["items_html"].to(string))
|
||||
let thread = parseChain(html)
|
||||
|
||||
await getMedia(thread, agent)
|
||||
if media: await getMedia(thread, agent)
|
||||
result.content = thread.content
|
||||
|
||||
proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
|
||||
proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
|
||||
var params = toSeq({
|
||||
"include_available_features": "1",
|
||||
"include_entities": "1",
|
||||
|
@ -36,9 +37,9 @@ proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
|
|||
let headers = genHeaders(agent, base / username, xml=true)
|
||||
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
|
||||
|
||||
result = await finishTimeline(json, Query(), after, agent)
|
||||
result = await finishTimeline(json, Query(), after, agent, media)
|
||||
|
||||
proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
|
||||
proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} =
|
||||
var url = base / username
|
||||
if after.len > 0:
|
||||
url = url ? {"max_position": after}
|
||||
|
@ -49,5 +50,5 @@ proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Ti
|
|||
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
|
||||
profile = parseTimelineProfile(html)
|
||||
|
||||
await getMedia(timeline, agent)
|
||||
if media: await getMedia(timeline, agent)
|
||||
result = (profile, timeline)
|
||||
|
|
|
@ -9,7 +9,9 @@ import ../views/general
|
|||
include "../views/rss.nimf"
|
||||
|
||||
proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} =
|
||||
let (profile, timeline, _) = await fetchSingleTimeline(name, "", getAgent(), query)
|
||||
let (profile, timeline, _) =
|
||||
await fetchSingleTimeline(name, "", getAgent(), query, media=false)
|
||||
|
||||
if timeline != nil:
|
||||
return renderTimelineRss(timeline, profile, hostname)
|
||||
|
||||
|
@ -28,7 +30,7 @@ proc createRssRouter*(cfg: Config) =
|
|||
if query.kind != tweets:
|
||||
resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg)
|
||||
|
||||
let tweets = await getSearch[Tweet](query, "", getAgent())
|
||||
let tweets = await getSearch[Tweet](query, "", getAgent(), media=false)
|
||||
respRss(renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg.hostname))
|
||||
|
||||
get "/@name/rss":
|
||||
|
@ -49,5 +51,5 @@ proc createRssRouter*(cfg: Config) =
|
|||
|
||||
get "/@name/lists/@list/rss":
|
||||
cond '.' notin @"name"
|
||||
let list = await getListTimeline(@"name", @"list", getAgent(), "")
|
||||
let list = await getListTimeline(@"name", @"list", getAgent(), "", media=false)
|
||||
respRss(renderListRss(list.content, @"name", @"list", cfg.hostname))
|
||||
|
|
|
@ -13,8 +13,8 @@ export profile, timeline, status
|
|||
|
||||
type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
|
||||
|
||||
proc fetchSingleTimeline*(name, after, agent: string;
|
||||
query: Query): Future[ProfileTimeline] {.async.} =
|
||||
proc fetchSingleTimeline*(name, after, agent: string; query: Query;
|
||||
media=true): Future[ProfileTimeline] {.async.} =
|
||||
let railFut = getPhotoRail(name, agent)
|
||||
|
||||
var timeline: Timeline
|
||||
|
@ -26,12 +26,12 @@ proc fetchSingleTimeline*(name, after, agent: string;
|
|||
|
||||
if query.kind == posts:
|
||||
if cachedProfile.isSome:
|
||||
timeline = await getTimeline(name, after, agent)
|
||||
timeline = await getTimeline(name, after, agent, media)
|
||||
else:
|
||||
(profile, timeline) = await getProfileAndTimeline(name, agent, after)
|
||||
(profile, timeline) = await getProfileAndTimeline(name, agent, after, media)
|
||||
cache(profile)
|
||||
else:
|
||||
var timelineFut = getSearch[Tweet](query, after, agent)
|
||||
var timelineFut = getSearch[Tweet](query, after, agent, media)
|
||||
if cachedProfile.isNone:
|
||||
profile = await getCachedProfile(name, agent)
|
||||
timeline = await timelineFut
|
||||
|
|
Loading…
Reference in New Issue