Use media endpoint for profile media tab

This bypasses "search" rate limits. It now includes media beyond
images and videos (eg. YouTube links are "media"), but the old
behaviour can be restored by clicking search, then filtering "Media"
and excluding retweets and replies.
This commit is contained in:
Zed 2019-10-23 08:34:03 +02:00
parent 0e6ac69a3f
commit ffce6e21ab
6 changed files with 55 additions and 28 deletions

View File

@ -21,10 +21,7 @@ proc getListTimeline*(username, list, agent, after: string; media=true): Future[
if result.content.len == 0:
return
let last = result.content[^1]
result.minId =
if last.retweet.isNone: $last.id
else: $(get(last.retweet).id)
result.minId = getLastId(result)
proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
let

View File

@ -123,9 +123,10 @@ proc getCard*(tweet: Tweet; agent: string) {.async.} =
if html == nil: return
parseCard(get(tweet.card), html)
proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} =
proc getPhotoRail*(username, agent: string; skip=false): Future[seq[GalleryPhoto]] {.async.} =
if skip: return
let
headers = genHeaders({"x-requested-with": "XMLHttpRequest"}, agent, base / username)
headers = genHeaders(agent, base / username, xml=true)
params = {"for_photo_rail": "true", "oldest_unread_id": "0"}
url = base / (timelineMediaUrl % username) ? params
html = await fetchHtml(url, headers, jsonKey="items_html")

View File

@ -18,10 +18,24 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string;
if not json.hasKey("items_html"): return
let html = parseHtml(json["items_html"].to(string))
let thread = parseChain(html)
let timeline = parseChain(html)
if media: await getMedia(thread, agent)
result.content = thread.content
if media: await getMedia(timeline, agent)
result.content = timeline.content
proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} =
var url = base / username
if after.len > 0:
url = url ? {"max_position": after}
let
headers = genHeaders(agent, base / username, auth=true)
html = await fetchHtml(url, headers)
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
profile = parseTimelineProfile(html)
if media: await getMedia(timeline, agent)
result = (profile, timeline)
proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
var params = toSeq({
@ -39,16 +53,19 @@ proc getTimeline*(username, after, agent: string; media=true): Future[Timeline]
result = await finishTimeline(json, Query(), after, agent, media)
proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} =
var url = base / username
proc getMediaTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
echo "mediaTimeline"
var params = toSeq({
"include_available_features": "1",
"include_entities": "1",
"reset_error_state": "false"
})
if after.len > 0:
url = url ? {"max_position": after}
params.add {"max_position": after}
let
headers = genHeaders(agent, base / username, auth=true)
html = await fetchHtml(url, headers)
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
profile = parseTimelineProfile(html)
let headers = genHeaders(agent, base / username, xml=true)
let json = await fetchJson(base / (timelineMediaUrl % username) ? params, headers)
if media: await getMedia(timeline, agent)
result = (profile, timeline)
result = await finishTimeline(json, Query(kind: QueryKind.media), after, agent, media)
result.minId = getLastId(result)

View File

@ -1,6 +1,7 @@
import httpclient, asyncdispatch, htmlparser
import strutils, json, xmltree, uri
import ../types
import consts
proc genHeaders*(headers: openArray[tuple[key: string, val: string]];
@ -52,3 +53,11 @@ proc fetchJson*(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
result = parseJson(resp)
except:
return nil
proc getLastId*(tweets: Result[Tweet]): string =
if tweets.content.len == 0: return
let last = tweets.content[^1]
if last.retweet.isNone:
$last.id
else:
$(get(last.retweet).id)

View File

@ -9,7 +9,7 @@ import ../views/general
include "../views/rss.nimf"
proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} =
let (profile, timeline, _) =
let (profile, timeline) =
await fetchSingleTimeline(name, "", getAgent(), query, media=false)
if timeline != nil:

View File

@ -11,12 +11,8 @@ export router_utils
export api, cache, formatters, query, agents
export profile, timeline, status
type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
proc fetchSingleTimeline*(name, after, agent: string; query: Query;
media=true): Future[ProfileTimeline] {.async.} =
let railFut = getPhotoRail(name, agent)
media=true): Future[(Profile, Timeline)] {.async.} =
var timeline: Timeline
var profile: Profile
var cachedProfile = hasCachedProfile(name)
@ -31,13 +27,17 @@ proc fetchSingleTimeline*(name, after, agent: string; query: Query;
(profile, timeline) = await getProfileAndTimeline(name, agent, after, media)
cache(profile)
else:
var timelineFut = getSearch[Tweet](query, after, agent, media)
var timelineFut =
if query.kind == QueryKind.media:
getMediaTimeline(name, after, agent, media)
else:
getSearch[Tweet](query, after, agent, media)
if cachedProfile.isNone:
profile = await getCachedProfile(name, agent)
timeline = await timelineFut
if profile.username.len == 0: return
return (profile, timeline, await railFut)
return (profile, timeline)
proc fetchMultiTimeline*(names: seq[string]; after, agent: string;
query: Query): Future[Timeline] {.async.} =
@ -60,7 +60,10 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; rss: string): Fu
names = name.strip(chars={'/'}).split(",").filterIt(it.len > 0)
if names.len == 1:
let (p, t, r) = await fetchSingleTimeline(names[0], after, agent, query)
let
rail = getPhotoRail(names[0], agent, skip=(query.kind == media))
(p, t) = await fetchSingleTimeline(names[0], after, agent, query)
r = await rail
if p.username.len == 0: return
let pHtml = renderProfile(p, t, r, prefs, getPath())
return renderMain(pHtml, request, cfg, pageTitle(p), pageDesc(p),