Use media endpoint for profile media tab

This bypasses "search" rate limits. It now includes media beyond
images and videos (eg. YouTube links are "media"), but the old
behaviour can be restored by clicking search, then filtering "Media"
and excluding retweets and replies.
This commit is contained in:
Zed 2019-10-23 08:34:03 +02:00
parent 0e6ac69a3f
commit ffce6e21ab
6 changed files with 55 additions and 28 deletions

View File

@ -21,10 +21,7 @@ proc getListTimeline*(username, list, agent, after: string; media=true): Future[
if result.content.len == 0: if result.content.len == 0:
return return
let last = result.content[^1] result.minId = getLastId(result)
result.minId =
if last.retweet.isNone: $last.id
else: $(get(last.retweet).id)
proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} = proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
let let

View File

@ -123,9 +123,10 @@ proc getCard*(tweet: Tweet; agent: string) {.async.} =
if html == nil: return if html == nil: return
parseCard(get(tweet.card), html) parseCard(get(tweet.card), html)
proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} = proc getPhotoRail*(username, agent: string; skip=false): Future[seq[GalleryPhoto]] {.async.} =
if skip: return
let let
headers = genHeaders({"x-requested-with": "XMLHttpRequest"}, agent, base / username) headers = genHeaders(agent, base / username, xml=true)
params = {"for_photo_rail": "true", "oldest_unread_id": "0"} params = {"for_photo_rail": "true", "oldest_unread_id": "0"}
url = base / (timelineMediaUrl % username) ? params url = base / (timelineMediaUrl % username) ? params
html = await fetchHtml(url, headers, jsonKey="items_html") html = await fetchHtml(url, headers, jsonKey="items_html")

View File

@ -18,10 +18,24 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string;
if not json.hasKey("items_html"): return if not json.hasKey("items_html"): return
let html = parseHtml(json["items_html"].to(string)) let html = parseHtml(json["items_html"].to(string))
let thread = parseChain(html) let timeline = parseChain(html)
if media: await getMedia(thread, agent) if media: await getMedia(timeline, agent)
result.content = thread.content result.content = timeline.content
proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} =
var url = base / username
if after.len > 0:
url = url ? {"max_position": after}
let
headers = genHeaders(agent, base / username, auth=true)
html = await fetchHtml(url, headers)
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
profile = parseTimelineProfile(html)
if media: await getMedia(timeline, agent)
result = (profile, timeline)
proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} = proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
var params = toSeq({ var params = toSeq({
@ -39,16 +53,19 @@ proc getTimeline*(username, after, agent: string; media=true): Future[Timeline]
result = await finishTimeline(json, Query(), after, agent, media) result = await finishTimeline(json, Query(), after, agent, media)
proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} = proc getMediaTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
var url = base / username echo "mediaTimeline"
var params = toSeq({
"include_available_features": "1",
"include_entities": "1",
"reset_error_state": "false"
})
if after.len > 0: if after.len > 0:
url = url ? {"max_position": after} params.add {"max_position": after}
let let headers = genHeaders(agent, base / username, xml=true)
headers = genHeaders(agent, base / username, auth=true) let json = await fetchJson(base / (timelineMediaUrl % username) ? params, headers)
html = await fetchHtml(url, headers)
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
profile = parseTimelineProfile(html)
if media: await getMedia(timeline, agent) result = await finishTimeline(json, Query(kind: QueryKind.media), after, agent, media)
result = (profile, timeline) result.minId = getLastId(result)

View File

@ -1,6 +1,7 @@
import httpclient, asyncdispatch, htmlparser import httpclient, asyncdispatch, htmlparser
import strutils, json, xmltree, uri import strutils, json, xmltree, uri
import ../types
import consts import consts
proc genHeaders*(headers: openArray[tuple[key: string, val: string]]; proc genHeaders*(headers: openArray[tuple[key: string, val: string]];
@ -52,3 +53,11 @@ proc fetchJson*(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
result = parseJson(resp) result = parseJson(resp)
except: except:
return nil return nil
proc getLastId*(tweets: Result[Tweet]): string =
if tweets.content.len == 0: return
let last = tweets.content[^1]
if last.retweet.isNone:
$last.id
else:
$(get(last.retweet).id)

View File

@ -9,7 +9,7 @@ import ../views/general
include "../views/rss.nimf" include "../views/rss.nimf"
proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} = proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} =
let (profile, timeline, _) = let (profile, timeline) =
await fetchSingleTimeline(name, "", getAgent(), query, media=false) await fetchSingleTimeline(name, "", getAgent(), query, media=false)
if timeline != nil: if timeline != nil:

View File

@ -11,12 +11,8 @@ export router_utils
export api, cache, formatters, query, agents export api, cache, formatters, query, agents
export profile, timeline, status export profile, timeline, status
type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
proc fetchSingleTimeline*(name, after, agent: string; query: Query; proc fetchSingleTimeline*(name, after, agent: string; query: Query;
media=true): Future[ProfileTimeline] {.async.} = media=true): Future[(Profile, Timeline)] {.async.} =
let railFut = getPhotoRail(name, agent)
var timeline: Timeline var timeline: Timeline
var profile: Profile var profile: Profile
var cachedProfile = hasCachedProfile(name) var cachedProfile = hasCachedProfile(name)
@ -31,13 +27,17 @@ proc fetchSingleTimeline*(name, after, agent: string; query: Query;
(profile, timeline) = await getProfileAndTimeline(name, agent, after, media) (profile, timeline) = await getProfileAndTimeline(name, agent, after, media)
cache(profile) cache(profile)
else: else:
var timelineFut = getSearch[Tweet](query, after, agent, media) var timelineFut =
if query.kind == QueryKind.media:
getMediaTimeline(name, after, agent, media)
else:
getSearch[Tweet](query, after, agent, media)
if cachedProfile.isNone: if cachedProfile.isNone:
profile = await getCachedProfile(name, agent) profile = await getCachedProfile(name, agent)
timeline = await timelineFut timeline = await timelineFut
if profile.username.len == 0: return if profile.username.len == 0: return
return (profile, timeline, await railFut) return (profile, timeline)
proc fetchMultiTimeline*(names: seq[string]; after, agent: string; proc fetchMultiTimeline*(names: seq[string]; after, agent: string;
query: Query): Future[Timeline] {.async.} = query: Query): Future[Timeline] {.async.} =
@ -60,7 +60,10 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; rss: string): Fu
names = name.strip(chars={'/'}).split(",").filterIt(it.len > 0) names = name.strip(chars={'/'}).split(",").filterIt(it.len > 0)
if names.len == 1: if names.len == 1:
let (p, t, r) = await fetchSingleTimeline(names[0], after, agent, query) let
rail = getPhotoRail(names[0], agent, skip=(query.kind == media))
(p, t) = await fetchSingleTimeline(names[0], after, agent, query)
r = await rail
if p.username.len == 0: return if p.username.len == 0: return
let pHtml = renderProfile(p, t, r, prefs, getPath()) let pHtml = renderProfile(p, t, r, prefs, getPath())
return renderMain(pHtml, request, cfg, pageTitle(p), pageDesc(p), return renderMain(pHtml, request, cfg, pageTitle(p), pageDesc(p),