Out with the old
This commit is contained in:
Zed
parent
4bb92a5201
commit
4167ce458b
|
|
@ -1,24 +0,0 @@
|
||||||
import uri
|
|
||||||
|
|
||||||
const
|
|
||||||
lang* = "en-US,en;q=0.9"
|
|
||||||
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
|
|
||||||
htmlAccept* = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
|
|
||||||
jsonAccept* = "application/json, text/javascript, */*; q=0.01"
|
|
||||||
|
|
||||||
base* = parseUri("https://twitter.com/")
|
|
||||||
apiBase* = parseUri("https://api.twitter.com/1.1/")
|
|
||||||
|
|
||||||
timelineUrl* = "i/profiles/show/$1/timeline/tweets"
|
|
||||||
timelineMediaUrl* = "i/profiles/show/$1/media_timeline"
|
|
||||||
listUrl* = "$1/lists/$2/timeline"
|
|
||||||
listMembersUrl* = "$1/lists/$2/members"
|
|
||||||
profilePopupUrl* = "i/profiles/popup"
|
|
||||||
profileIntentUrl* = "intent/user"
|
|
||||||
searchUrl* = "i/search/timeline"
|
|
||||||
tweetUrl* = "status"
|
|
||||||
repliesUrl* = "i/$1/conversation/$2"
|
|
||||||
videoUrl* = "videos/tweet/config/$1.json"
|
|
||||||
tokenUrl* = "guest/activate.json"
|
|
||||||
cardUrl* = "i/cards/tfw/v1/$1"
|
|
||||||
pollUrl* = cardUrl & "?cardname=poll2choice_text_only&lang=en"
|
|
||||||
|
|
@ -1,65 +0,0 @@
|
||||||
import httpclient, asyncdispatch, htmlparser
|
|
||||||
import sequtils, strutils, json, uri
|
|
||||||
|
|
||||||
import ".."/[types, parser, parserutils, query]
|
|
||||||
import utils, consts, timeline, search
|
|
||||||
|
|
||||||
proc getListTimeline*(username, list, after, agent: string; media=true): Future[Timeline] {.async.} =
|
|
||||||
let url = base / (listUrl % [username, list])
|
|
||||||
|
|
||||||
var params = toSeq({
|
|
||||||
"include_available_features": "1",
|
|
||||||
"include_entities": "1",
|
|
||||||
"reset_error_state": "false"
|
|
||||||
})
|
|
||||||
|
|
||||||
if after.len > 0:
|
|
||||||
params.add {"max_position": after}
|
|
||||||
|
|
||||||
let json = await fetchJson(url ? params, genHeaders(agent, url))
|
|
||||||
result = await finishTimeline(json, Query(), after, agent, media)
|
|
||||||
if result.content.len == 0:
|
|
||||||
return
|
|
||||||
|
|
||||||
result.minId = getLastId(result)
|
|
||||||
|
|
||||||
proc getListMembersSearch(username, list, after, agent: string): Future[Result[Profile]] {.async.} =
|
|
||||||
let
|
|
||||||
referer = base / (listMembersUrl % [username, list])
|
|
||||||
url = referer / "timeline"
|
|
||||||
headers = genHeaders({"x-push-with": "XMLHttpRequest"}, agent, referer, xml=true)
|
|
||||||
|
|
||||||
var params = toSeq({
|
|
||||||
"include_available_features": "1",
|
|
||||||
"include_entities": "1",
|
|
||||||
"reset_error_state": "false"
|
|
||||||
})
|
|
||||||
|
|
||||||
if after.len > 0:
|
|
||||||
params.add {"max_position": after}
|
|
||||||
|
|
||||||
let json = await fetchJson(url ? params, headers)
|
|
||||||
|
|
||||||
result = getResult[Profile](json, Query(kind: userList), after)
|
|
||||||
if json == nil or not json.hasKey("items_html"): return
|
|
||||||
|
|
||||||
let html = json["items_html"].to(string)
|
|
||||||
result.hasMore = html != "\n"
|
|
||||||
for p in parseHtml(html).selectAll(".account"):
|
|
||||||
result.content.add parseListProfile(p)
|
|
||||||
|
|
||||||
proc getListMembers*(username, list, after, agent: string): Future[Result[Profile]] {.async.} =
|
|
||||||
if after.len > 0:
|
|
||||||
return await getListMembersSearch(username, list, after, agent)
|
|
||||||
|
|
||||||
let
|
|
||||||
url = base / (listMembersUrl % [username, list])
|
|
||||||
html = await fetchHtml(url, genHeaders(agent, url))
|
|
||||||
|
|
||||||
result = Result[Profile](
|
|
||||||
minId: html.selectAttr(".stream-container", "data-min-position"),
|
|
||||||
hasMore: html.select(".has-more-items") != nil,
|
|
||||||
beginning: true,
|
|
||||||
query: Query(kind: userList),
|
|
||||||
content: html.selectAll(".account").map(parseListProfile)
|
|
||||||
)
|
|
||||||
|
|
@ -1,159 +0,0 @@
|
||||||
import httpclient, asyncdispatch, times, sequtils, strutils, json, uri
|
|
||||||
import macros, options
|
|
||||||
|
|
||||||
import ".."/[types, parser, formatters, cache]
|
|
||||||
import utils, consts
|
|
||||||
|
|
||||||
var
|
|
||||||
guestToken = ""
|
|
||||||
tokenUses = 0
|
|
||||||
tokenMaxUses = 230
|
|
||||||
tokenUpdated: Time
|
|
||||||
tokenLifetime = initDuration(minutes=20)
|
|
||||||
|
|
||||||
macro genMediaGet(media: untyped; token=false) =
|
|
||||||
let
|
|
||||||
mediaName = capitalizeAscii($media)
|
|
||||||
multi = ident("get" & mediaName & "s")
|
|
||||||
convo = ident("getConversation" & mediaName & "s")
|
|
||||||
replies = ident("getReplies" & mediaName & "s")
|
|
||||||
single = ident("get" & mediaName)
|
|
||||||
|
|
||||||
quote do:
|
|
||||||
proc `multi`*(thread: Chain | Timeline; agent: string; token="") {.async.} =
|
|
||||||
if thread == nil: return
|
|
||||||
var `media` = thread.content.filterIt(it.`media`.isSome)
|
|
||||||
when `token`:
|
|
||||||
var gToken = token
|
|
||||||
if gToken.len == 0: gToken = await getGuestToken(agent)
|
|
||||||
await all(`media`.mapIt(`single`(it, token, agent)))
|
|
||||||
else:
|
|
||||||
await all(`media`.mapIt(`single`(it, agent)))
|
|
||||||
|
|
||||||
proc `replies`*(replies: Result[Chain]; agent: string; token="") {.async.} =
|
|
||||||
when `token`:
|
|
||||||
var gToken = token
|
|
||||||
if gToken.len == 0: gToken = await getGuestToken(agent)
|
|
||||||
await all(replies.content.mapIt(`multi`(it, agent, token=gToken)))
|
|
||||||
else:
|
|
||||||
await all(replies.content.mapIt(`multi`(it, agent)))
|
|
||||||
|
|
||||||
proc `convo`*(convo: Conversation; agent: string) {.async.} =
|
|
||||||
var futs: seq[Future[void]]
|
|
||||||
when `token`:
|
|
||||||
var token = await getGuestToken(agent)
|
|
||||||
futs.add `single`(convo.tweet, agent, token)
|
|
||||||
futs.add `multi`(convo.before, agent, token=token)
|
|
||||||
futs.add `multi`(convo.after, agent, token=token)
|
|
||||||
if convo.replies != nil:
|
|
||||||
futs.add `replies`(convo.replies, agent, token=token)
|
|
||||||
else:
|
|
||||||
futs.add `single`(convo.tweet, agent)
|
|
||||||
futs.add `multi`(convo.before, agent)
|
|
||||||
futs.add `multi`(convo.after, agent)
|
|
||||||
if convo.replies != nil:
|
|
||||||
futs.add `replies`(convo.replies, agent)
|
|
||||||
await all(futs)
|
|
||||||
|
|
||||||
proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
|
|
||||||
if getTime() - tokenUpdated < tokenLifetime and
|
|
||||||
not force and tokenUses < tokenMaxUses:
|
|
||||||
return guestToken
|
|
||||||
|
|
||||||
tokenUpdated = getTime()
|
|
||||||
tokenUses = 0
|
|
||||||
|
|
||||||
let headers = genHeaders({"authorization": auth}, agent, base, lang=false)
|
|
||||||
newClient()
|
|
||||||
|
|
||||||
var res: string
|
|
||||||
try: res = await client.postContent($(apiBase / tokenUrl))
|
|
||||||
except: return
|
|
||||||
|
|
||||||
let json = parseJson(res)
|
|
||||||
|
|
||||||
if json != nil:
|
|
||||||
result = json["guest_token"].to(string)
|
|
||||||
guestToken = result
|
|
||||||
|
|
||||||
proc getVideoVar(tweet: Tweet): var Option[Video] =
|
|
||||||
if tweet.card.isSome():
|
|
||||||
return get(tweet.card).video
|
|
||||||
else:
|
|
||||||
return tweet.video
|
|
||||||
|
|
||||||
proc getVideoFetch(tweet: Tweet; agent, token: string; retry=true): Future[Option[Video]] {.async.} =
|
|
||||||
if tweet.video.isNone(): return
|
|
||||||
|
|
||||||
let
|
|
||||||
headers = genHeaders({"authorization": auth, "x-guest-token": token},
|
|
||||||
agent, base / getLink(tweet, focus=false), lang=false)
|
|
||||||
url = apiBase / (videoUrl % $tweet.id)
|
|
||||||
json = await fetchJson(url, headers)
|
|
||||||
|
|
||||||
if json == nil:
|
|
||||||
if not retry: return
|
|
||||||
if getTime() - tokenUpdated > initDuration(seconds=1):
|
|
||||||
tokenUpdated = getTime()
|
|
||||||
discard await getGuestToken(agent, force=true)
|
|
||||||
return await getVideoFetch(tweet, agent, guestToken, retry=false)
|
|
||||||
|
|
||||||
var video = parseVideo(json, tweet.id)
|
|
||||||
video.title = get(tweet.video).title
|
|
||||||
video.description = get(tweet.video).description
|
|
||||||
cache(video)
|
|
||||||
|
|
||||||
result = some video
|
|
||||||
tokenUses.inc
|
|
||||||
|
|
||||||
proc videoIsInvalid(video: Video): bool =
|
|
||||||
not video.available and video.url.len == 0
|
|
||||||
|
|
||||||
proc getVideo*(tweet: Tweet; agent, token: string; force=false) {.async.} =
|
|
||||||
let token = if token.len == 0: guestToken else: token
|
|
||||||
var video = getCachedVideo(tweet.id)
|
|
||||||
if video.isNone:
|
|
||||||
video = await getVideoFetch(tweet, agent, token)
|
|
||||||
elif videoIsInvalid(get(video)) and tweet.gif.isSome:
|
|
||||||
# gif was mistakenly parsed as a gif
|
|
||||||
uncache(tweet.id)
|
|
||||||
return
|
|
||||||
|
|
||||||
getVideoVar(tweet) = video
|
|
||||||
if tweet.card.isSome: tweet.video = none Video
|
|
||||||
|
|
||||||
proc getPoll*(tweet: Tweet; agent: string) {.async.} =
|
|
||||||
if tweet.poll.isNone(): return
|
|
||||||
|
|
||||||
let
|
|
||||||
headers = genHeaders(agent, base / getLink(tweet, focus=false), auth=true)
|
|
||||||
url = base / (pollUrl % $tweet.id)
|
|
||||||
html = await fetchHtml(url, headers)
|
|
||||||
|
|
||||||
if html == nil: return
|
|
||||||
tweet.poll = some parsePoll(html)
|
|
||||||
|
|
||||||
proc getCard*(tweet: Tweet; agent: string) {.async.} =
|
|
||||||
if tweet.card.isNone(): return
|
|
||||||
|
|
||||||
let
|
|
||||||
headers = genHeaders(agent, base / getLink(tweet, focus=false), auth=true)
|
|
||||||
query = get(tweet.card).query.replace("sensitive=true", "sensitive=false")
|
|
||||||
html = await fetchHtml(base / query, headers)
|
|
||||||
|
|
||||||
if html == nil: return
|
|
||||||
parseCard(get(tweet.card), html)
|
|
||||||
|
|
||||||
proc getPhotoRail*(username, agent: string; skip=false): Future[seq[GalleryPhoto]] {.async.} =
|
|
||||||
if skip: return
|
|
||||||
let
|
|
||||||
headers = genHeaders(agent, base / username, xml=true)
|
|
||||||
params = {"for_photo_rail": "true", "oldest_unread_id": "0"}
|
|
||||||
url = base / (timelineMediaUrl % username) ? params
|
|
||||||
html = await fetchHtml(url, headers, jsonKey="items_html")
|
|
||||||
|
|
||||||
result = parsePhotoRail(html)
|
|
||||||
|
|
||||||
genMediaGet(video, token=true)
|
|
||||||
genMediaGet(poll)
|
|
||||||
genMediaGet(card)
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
import httpclient, asyncdispatch, times, strutils, uri
|
|
||||||
|
|
||||||
import ".."/[types, parser, parserutils]
|
|
||||||
|
|
||||||
import utils, consts
|
|
||||||
|
|
||||||
proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} =
|
|
||||||
let url = base / profileIntentUrl ? {"screen_name": username}
|
|
||||||
let html = await fetchHtml(url, headers)
|
|
||||||
if html == nil: return Profile()
|
|
||||||
|
|
||||||
result = parseIntentProfile(html)
|
|
||||||
|
|
||||||
proc getProfile*(username, agent: string): Future[Profile] {.async.} =
|
|
||||||
let
|
|
||||||
headers = genHeaders(agent, base / username, xml=true)
|
|
||||||
|
|
||||||
params = {
|
|
||||||
"screen_name": username,
|
|
||||||
"wants_hovercard": "true",
|
|
||||||
"_": $(epochTime().int)
|
|
||||||
}
|
|
||||||
|
|
||||||
url = base / profilePopupUrl ? params
|
|
||||||
html = await fetchHtml(url, headers, jsonKey="html")
|
|
||||||
|
|
||||||
if html == nil: return Profile()
|
|
||||||
|
|
||||||
if html.select(".ProfileCard-sensitiveWarningContainer") != nil:
|
|
||||||
return await getProfileFallback(username, headers)
|
|
||||||
|
|
||||||
result = parsePopupProfile(html)
|
|
||||||
|
|
||||||
proc getProfileFull*(username, agent: string): Future[Profile] {.async.} =
|
|
||||||
let
|
|
||||||
url = base / username
|
|
||||||
headers = genHeaders(agent, url, auth=true)
|
|
||||||
html = await fetchHtml(url, headers)
|
|
||||||
|
|
||||||
if html == nil: return
|
|
||||||
result = parseTimelineProfile(html)
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
import asyncdispatch, httpclient
|
|
||||||
|
|
||||||
import ".."/[formatters, types]
|
|
||||||
|
|
||||||
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
|
|
||||||
let client = newAsyncHttpClient(maxRedirects=0)
|
|
||||||
try:
|
|
||||||
let resp = await client.request(url, $HttpHead)
|
|
||||||
result = resp.headers["location"].replaceUrl(prefs)
|
|
||||||
except:
|
|
||||||
discard
|
|
||||||
finally:
|
|
||||||
client.close()
|
|
||||||
|
|
@ -1,53 +0,0 @@
|
||||||
import httpclient, asyncdispatch, htmlparser
|
|
||||||
import strutils, json, xmltree, uri
|
|
||||||
|
|
||||||
import ".."/[types, parser, parserutils, query]
|
|
||||||
import utils, consts, timeline
|
|
||||||
|
|
||||||
proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
|
|
||||||
if json == nil: return Result[T](beginning: true, query: query)
|
|
||||||
Result[T](
|
|
||||||
hasMore: json{"has_more_items"}.getBool(false),
|
|
||||||
maxId: json{"max_position"}.getStr,
|
|
||||||
minId: json{"min_position"}.getStr,
|
|
||||||
query: query,
|
|
||||||
beginning: after.len == 0
|
|
||||||
)
|
|
||||||
|
|
||||||
proc getSearch*[T](query: Query; after, agent: string;
|
|
||||||
media=true): Future[Result[T]] {.async.} =
|
|
||||||
let
|
|
||||||
kind = if query.kind == users: "users" else: "tweets"
|
|
||||||
|
|
||||||
param = genQueryParam(query)
|
|
||||||
encoded = encodeUrl(param, usePlus=false)
|
|
||||||
|
|
||||||
referer = base / ("search?f=$1&q=$2&src=typd" % [kind, encoded])
|
|
||||||
headers = genHeaders(agent, referer, auth=true, xml=true)
|
|
||||||
|
|
||||||
params = {
|
|
||||||
"f": kind,
|
|
||||||
"vertical": "default",
|
|
||||||
"q": param,
|
|
||||||
"src": "typd",
|
|
||||||
"include_available_features": "1",
|
|
||||||
"include_entities": "1",
|
|
||||||
"max_position": if after.len > 0: after else: "0",
|
|
||||||
"reset_error_state": "false"
|
|
||||||
}
|
|
||||||
|
|
||||||
if param in ["include:nativeretweets", "-filter:nativeretweets", ""]:
|
|
||||||
return Result[T](query: query, beginning: true)
|
|
||||||
|
|
||||||
let json = await fetchJson(base / searchUrl ? params, headers)
|
|
||||||
|
|
||||||
result = getResult[T](json, query, after)
|
|
||||||
if json == nil or not json.hasKey("items_html"): return
|
|
||||||
|
|
||||||
when T is Tweet:
|
|
||||||
result = await finishTimeline(json, query, after, agent, media)
|
|
||||||
elif T is Profile:
|
|
||||||
let html = json["items_html"].to(string)
|
|
||||||
result.hasMore = html != "\n"
|
|
||||||
for p in parseHtml(html).selectAll(".js-stream-item"):
|
|
||||||
result.content.add parsePopupProfile(p, ".ProfileCard")
|
|
||||||
|
|
@ -1,75 +0,0 @@
|
||||||
import httpclient, asyncdispatch, htmlparser, strformat
|
|
||||||
import sequtils, strutils, json, uri
|
|
||||||
|
|
||||||
import ".."/[types, parser, parserutils, formatters, query]
|
|
||||||
import utils, consts, media, search
|
|
||||||
|
|
||||||
proc getMedia(thread: Chain | Timeline; agent: string) {.async.} =
|
|
||||||
await all(getVideos(thread, agent),
|
|
||||||
getCards(thread, agent),
|
|
||||||
getPolls(thread, agent))
|
|
||||||
|
|
||||||
proc finishTimeline*(json: JsonNode; query: Query; after, agent: string;
|
|
||||||
media=true): Future[Timeline] {.async.} =
|
|
||||||
result = getResult[Tweet](json, query, after)
|
|
||||||
if json == nil: return
|
|
||||||
|
|
||||||
if json["new_latent_count"].to(int) == 0: return
|
|
||||||
if not json.hasKey("items_html"): return
|
|
||||||
|
|
||||||
let html = parseHtml(json["items_html"].to(string))
|
|
||||||
let timeline = parseChain(html)
|
|
||||||
|
|
||||||
if media: await getMedia(timeline, agent)
|
|
||||||
result.content = timeline.content
|
|
||||||
|
|
||||||
proc getProfileAndTimeline*(username, after, agent: string;
|
|
||||||
media=true): Future[(Profile, Timeline)] {.async.} =
|
|
||||||
var url = base / username
|
|
||||||
if after.len > 0:
|
|
||||||
url = url ? {"max_position": after}
|
|
||||||
|
|
||||||
let
|
|
||||||
headers = genHeaders(agent, base / username, auth=true)
|
|
||||||
html = await fetchHtml(url, headers)
|
|
||||||
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
|
|
||||||
profile = parseTimelineProfile(html)
|
|
||||||
|
|
||||||
if media and profile.username.len > 0: await getMedia(timeline, agent)
|
|
||||||
result = (profile, timeline)
|
|
||||||
|
|
||||||
proc getTimeline*(username, after, agent: string;
|
|
||||||
media=true): Future[Timeline] {.async.} =
|
|
||||||
var params = toSeq({
|
|
||||||
"include_available_features": "1",
|
|
||||||
"include_entities": "1",
|
|
||||||
"include_new_items_bar": "false",
|
|
||||||
"reset_error_state": "false"
|
|
||||||
})
|
|
||||||
|
|
||||||
if after.len > 0:
|
|
||||||
params.add {"max_position": after}
|
|
||||||
|
|
||||||
let headers = genHeaders(agent, base / username, xml=true)
|
|
||||||
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
|
|
||||||
|
|
||||||
result = await finishTimeline(json, Query(), after, agent, media)
|
|
||||||
|
|
||||||
proc getMediaTimeline*(username, after, agent: string;
|
|
||||||
media=true): Future[Timeline] {.async.} =
|
|
||||||
var params = toSeq({
|
|
||||||
"include_available_features": "1",
|
|
||||||
"include_entities": "1",
|
|
||||||
"reset_error_state": "false"
|
|
||||||
})
|
|
||||||
|
|
||||||
if after.len > 0:
|
|
||||||
params.add {"max_position": after}
|
|
||||||
|
|
||||||
let
|
|
||||||
headers = genHeaders(agent, base / username, xml=true)
|
|
||||||
json = await fetchJson(base / (timelineMediaUrl % username) ? params, headers)
|
|
||||||
query = Query(kind: QueryKind.media)
|
|
||||||
|
|
||||||
result = await finishTimeline(json, query, after, agent, media)
|
|
||||||
result.minId = getLastId(result)
|
|
||||||
|
|
@ -1,61 +0,0 @@
|
||||||
import asyncdispatch, strutils, uri, httpclient, json, xmltree, htmlparser
|
|
||||||
|
|
||||||
import ".."/[types, parser]
|
|
||||||
import utils, consts, media
|
|
||||||
|
|
||||||
proc getTweet*(username, id, after, agent: string): Future[Conversation] {.async.} =
|
|
||||||
let
|
|
||||||
headers = genHeaders({
|
|
||||||
"pragma": "no-cache",
|
|
||||||
"x-previous-page-name": "profile",
|
|
||||||
"accept": htmlAccept
|
|
||||||
}, agent, base, xml=true)
|
|
||||||
|
|
||||||
url = base / username / tweetUrl / id ? {"max_position": after}
|
|
||||||
|
|
||||||
newClient()
|
|
||||||
var html: XmlNode
|
|
||||||
try:
|
|
||||||
let resp = await client.get($url)
|
|
||||||
if resp.code == Http403 and "suspended" in (await resp.body):
|
|
||||||
return Conversation(tweet: Tweet(tombstone: "User has been suspended"))
|
|
||||||
html = parseHtml(await resp.body)
|
|
||||||
except:
|
|
||||||
discard
|
|
||||||
|
|
||||||
if html == nil: return
|
|
||||||
|
|
||||||
result = parseConversation(html, after)
|
|
||||||
|
|
||||||
await all(getConversationVideos(result, agent),
|
|
||||||
getConversationCards(result, agent),
|
|
||||||
getConversationPolls(result, agent))
|
|
||||||
|
|
||||||
proc getReplies*(username, id, after, agent: string): Future[Result[Chain]] {.async.} =
|
|
||||||
let
|
|
||||||
headers = genHeaders({
|
|
||||||
"pragma": "no-cache",
|
|
||||||
"x-previous-page-name": "permalink",
|
|
||||||
"accept": htmlAccept
|
|
||||||
}, agent, base, xml=true)
|
|
||||||
|
|
||||||
params = {
|
|
||||||
"include_available_features": "1",
|
|
||||||
"include_entities": "1",
|
|
||||||
"max_position": after,
|
|
||||||
}
|
|
||||||
|
|
||||||
url = base / (repliesUrl % [username, id]) ? params
|
|
||||||
|
|
||||||
let json = await fetchJson(url, headers)
|
|
||||||
if json == nil or not json.hasKey("items_html"): return
|
|
||||||
let html = parseHtml(json{"items_html"}.getStr)
|
|
||||||
|
|
||||||
result = parseReplies(html)
|
|
||||||
result.minId = json{"min_position"}.getStr(result.minId)
|
|
||||||
if result.minId.len > 0:
|
|
||||||
result.hasMore = true
|
|
||||||
|
|
||||||
await all(getRepliesVideos(result, agent),
|
|
||||||
getRepliesCards(result, agent),
|
|
||||||
getRepliesPolls(result, agent))
|
|
||||||
|
|
@ -1,64 +0,0 @@
|
||||||
import httpclient, asyncdispatch, htmlparser, options
|
|
||||||
import strutils, json, xmltree, uri
|
|
||||||
|
|
||||||
import ../types
|
|
||||||
import consts
|
|
||||||
|
|
||||||
proc genHeaders*(headers: openArray[tuple[key: string, val: string]];
|
|
||||||
agent: string; referer: Uri; lang=true;
|
|
||||||
auth=false; xml=false): HttpHeaders =
|
|
||||||
result = newHttpHeaders({
|
|
||||||
"referer": $referer,
|
|
||||||
"user-agent": agent,
|
|
||||||
"x-twitter-active-user": "yes",
|
|
||||||
})
|
|
||||||
|
|
||||||
if auth: result["authority"] = "twitter.com"
|
|
||||||
if lang: result["accept-language"] = consts.lang
|
|
||||||
if xml: result["x-requested-with"] = "XMLHttpRequest"
|
|
||||||
|
|
||||||
for (key, val) in headers:
|
|
||||||
result[key] = val
|
|
||||||
|
|
||||||
proc genHeaders*(agent: string; referer: Uri; lang=true;
|
|
||||||
auth=false; xml=false): HttpHeaders =
|
|
||||||
genHeaders([], agent, referer, lang, auth, xml)
|
|
||||||
|
|
||||||
template newClient*() {.dirty.} =
|
|
||||||
var client = newAsyncHttpClient()
|
|
||||||
defer:
|
|
||||||
try: client.close()
|
|
||||||
except: discard
|
|
||||||
client.headers = headers
|
|
||||||
|
|
||||||
proc fetchHtml*(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} =
|
|
||||||
headers["accept"] = htmlAccept
|
|
||||||
newClient()
|
|
||||||
|
|
||||||
var resp = ""
|
|
||||||
try:
|
|
||||||
resp = await client.getContent($url)
|
|
||||||
except:
|
|
||||||
return nil
|
|
||||||
|
|
||||||
if jsonKey.len > 0:
|
|
||||||
resp = parseJson(resp)[jsonKey].str
|
|
||||||
return parseHtml(resp)
|
|
||||||
|
|
||||||
proc fetchJson*(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
|
|
||||||
headers["accept"] = jsonAccept
|
|
||||||
newClient()
|
|
||||||
|
|
||||||
try:
|
|
||||||
let resp = await client.getContent($url)
|
|
||||||
result = parseJson(resp)
|
|
||||||
except:
|
|
||||||
return nil
|
|
||||||
|
|
||||||
proc getLastId*(tweets: Result[Tweet]): string =
|
|
||||||
if tweets.content.len == 0: return
|
|
||||||
let last = tweets.content[^1]
|
|
||||||
if last.retweet.isNone:
|
|
||||||
$last.id
|
|
||||||
else:
|
|
||||||
$(get(last.retweet).id)
|
|
||||||
|
|
@ -1,91 +0,0 @@
|
||||||
import asyncdispatch, times, strutils
|
|
||||||
import norm/sqlite
|
|
||||||
|
|
||||||
import types, api/profile
|
|
||||||
|
|
||||||
template safeAddColumn(field: typedesc): untyped =
|
|
||||||
try: field.addColumn
|
|
||||||
except DbError: discard
|
|
||||||
|
|
||||||
dbFromTypes("cache.db", "", "", "", [Profile, Video])
|
|
||||||
|
|
||||||
withDb:
|
|
||||||
Video.createTable(force=true)
|
|
||||||
try: Profile.createTable()
|
|
||||||
except DbError: discard
|
|
||||||
|
|
||||||
safeAddColumn Profile.lowername
|
|
||||||
safeAddColumn Profile.suspended
|
|
||||||
|
|
||||||
var profileCacheTime = initDuration(minutes=10)
|
|
||||||
|
|
||||||
proc isOutdated*(profile: Profile): bool =
|
|
||||||
getTime() - profile.updated > profileCacheTime
|
|
||||||
|
|
||||||
proc cache*(profile: var Profile) =
|
|
||||||
withDb:
|
|
||||||
try:
|
|
||||||
let p = Profile.getOne("lowername = ?", profile.lowername)
|
|
||||||
profile.id = p.id
|
|
||||||
profile.update()
|
|
||||||
except KeyError:
|
|
||||||
if profile.username.len > 0:
|
|
||||||
profile.insert()
|
|
||||||
|
|
||||||
proc hasCachedProfile*(username: string): Option[Profile] =
|
|
||||||
withDb:
|
|
||||||
try:
|
|
||||||
let p = Profile.getOne("lowername = ?", toLower(username))
|
|
||||||
doAssert not p.isOutdated
|
|
||||||
result = some p
|
|
||||||
except AssertionError, KeyError:
|
|
||||||
result = none Profile
|
|
||||||
|
|
||||||
proc getCachedProfile*(username, agent: string;
|
|
||||||
force=false): Future[Profile] {.async.} =
|
|
||||||
withDb:
|
|
||||||
try:
|
|
||||||
result.getOne("lowername = ?", toLower(username))
|
|
||||||
doAssert not result.isOutdated
|
|
||||||
except AssertionError, KeyError:
|
|
||||||
result = await getProfileFull(username, agent)
|
|
||||||
cache(result)
|
|
||||||
|
|
||||||
proc setProfileCacheTime*(minutes: int) =
|
|
||||||
profileCacheTime = initDuration(minutes=minutes)
|
|
||||||
|
|
||||||
proc cache*(video: var Video) =
|
|
||||||
withDb:
|
|
||||||
try:
|
|
||||||
let v = Video.getOne("videoId = ?", video.videoId)
|
|
||||||
video.id = v.id
|
|
||||||
video.update()
|
|
||||||
except KeyError:
|
|
||||||
if video.videoId.len > 0:
|
|
||||||
video.insert()
|
|
||||||
|
|
||||||
proc uncache*(id: int64) =
|
|
||||||
withDb:
|
|
||||||
try:
|
|
||||||
var video = Video.getOne("videoId = ?", $id)
|
|
||||||
video.delete()
|
|
||||||
except:
|
|
||||||
discard
|
|
||||||
|
|
||||||
proc getCachedVideo*(id: int64): Option[Video] =
|
|
||||||
withDb:
|
|
||||||
try:
|
|
||||||
return some Video.getOne("videoId = ?", $id)
|
|
||||||
except KeyError:
|
|
||||||
return none Video
|
|
||||||
|
|
||||||
proc cacheCleaner*() {.async.} =
|
|
||||||
while true:
|
|
||||||
await sleepAsync(profileCacheTime.inMilliseconds.int)
|
|
||||||
withDb:
|
|
||||||
let up = "updated<" & $toUnix(getTime() - profileCacheTime)
|
|
||||||
var profiles = Profile.getMany(10000, cond=up)
|
|
||||||
var videos = Video.getMany(10000, cond=up)
|
|
||||||
transaction:
|
|
||||||
for p in profiles.mitems: delete(p)
|
|
||||||
for v in videos.mitems: delete(v)
|
|
||||||
290
src/parser.nim
290
src/parser.nim
|
|
@ -1,290 +0,0 @@
|
||||||
import xmltree, sequtils, strutils, json, options
|
|
||||||
|
|
||||||
import types, parserutils, formatters
|
|
||||||
|
|
||||||
proc parseJsonData*(node: XmlNode): JsonNode =
|
|
||||||
let jsonData = node.selectAttr("input.json-data", "value")
|
|
||||||
if jsonData.len > 0:
|
|
||||||
return parseJson(jsonData)
|
|
||||||
|
|
||||||
proc parseTimelineProfile*(node: XmlNode): Profile =
|
|
||||||
let profile = node.select(".ProfileHeaderCard")
|
|
||||||
if profile == nil:
|
|
||||||
let data = parseJsonData(node)
|
|
||||||
if data != nil and data{"sectionName"}.getStr == "suspended":
|
|
||||||
let username = data{"internalReferer"}.getStr.strip(chars={'/'})
|
|
||||||
return Profile(username: username, suspended: true)
|
|
||||||
return
|
|
||||||
|
|
||||||
let pre = ".ProfileHeaderCard-"
|
|
||||||
let username = profile.getUsername(pre & "screenname")
|
|
||||||
result = Profile(
|
|
||||||
fullname: profile.getName(pre & "nameLink"),
|
|
||||||
username: username,
|
|
||||||
lowername: toLower(username),
|
|
||||||
joinDate: profile.getDate(pre & "joinDateText"),
|
|
||||||
website: profile.selectAttr(pre & "urlText a", "title"),
|
|
||||||
bio: profile.getBio(pre & "bio"),
|
|
||||||
location: getLocation(profile),
|
|
||||||
userpic: node.getAvatar(".profile-picture img"),
|
|
||||||
verified: isVerified(profile),
|
|
||||||
protected: isProtected(profile),
|
|
||||||
banner: getTimelineBanner(node),
|
|
||||||
media: getMediaCount(node)
|
|
||||||
)
|
|
||||||
|
|
||||||
result.getProfileStats(node.select(".ProfileNav-list"))
|
|
||||||
|
|
||||||
proc parsePopupProfile*(node: XmlNode; selector=".profile-card"): Profile =
|
|
||||||
let profile = node.select(selector)
|
|
||||||
if profile == nil: return
|
|
||||||
|
|
||||||
let username = profile.getUsername(".username")
|
|
||||||
result = Profile(
|
|
||||||
fullname: profile.getName(".fullname"),
|
|
||||||
username: username,
|
|
||||||
lowername: toLower(username),
|
|
||||||
bio: profile.getBio(".bio", fallback=".ProfileCard-bio"),
|
|
||||||
userpic: profile.getAvatar(".ProfileCard-avatarImage"),
|
|
||||||
verified: isVerified(profile),
|
|
||||||
protected: isProtected(profile),
|
|
||||||
banner: getBanner(profile)
|
|
||||||
)
|
|
||||||
|
|
||||||
result.getPopupStats(profile)
|
|
||||||
|
|
||||||
proc parseListProfile*(profile: XmlNode): Profile =
|
|
||||||
result = Profile(
|
|
||||||
fullname: profile.getName(".fullname"),
|
|
||||||
username: profile.getUsername(".username"),
|
|
||||||
bio: profile.getBio(".bio").stripText(),
|
|
||||||
userpic: profile.getAvatar(".avatar"),
|
|
||||||
verified: isVerified(profile),
|
|
||||||
protected: isProtected(profile),
|
|
||||||
)
|
|
||||||
|
|
||||||
proc parseIntentProfile*(profile: XmlNode): Profile =
|
|
||||||
result = Profile(
|
|
||||||
fullname: profile.getName("a.fn.url.alternate-context"),
|
|
||||||
username: profile.getUsername(".nickname"),
|
|
||||||
bio: profile.getBio("p.note"),
|
|
||||||
userpic: profile.select(".profile.summary").getAvatar("img.photo"),
|
|
||||||
verified: profile.select("li.verified") != nil,
|
|
||||||
protected: profile.select("li.protected") != nil,
|
|
||||||
banner: getBanner(profile)
|
|
||||||
)
|
|
||||||
|
|
||||||
result.getIntentStats(profile)
|
|
||||||
|
|
||||||
proc parseTweetProfile*(profile: XmlNode): Profile =
|
|
||||||
result = Profile(
|
|
||||||
fullname: profile.attr("data-name").stripText(),
|
|
||||||
username: profile.attr("data-screen-name"),
|
|
||||||
userpic: profile.getAvatar(".avatar"),
|
|
||||||
verified: isVerified(profile)
|
|
||||||
)
|
|
||||||
|
|
||||||
proc parseQuote*(quote: XmlNode): Quote =
|
|
||||||
result = Quote(
|
|
||||||
id: parseBiggestInt(quote.attr("data-item-id")),
|
|
||||||
text: getQuoteText(quote),
|
|
||||||
reply: parseTweetReply(quote),
|
|
||||||
hasThread: quote.select(".self-thread-context") != nil,
|
|
||||||
available: true
|
|
||||||
)
|
|
||||||
|
|
||||||
result.profile = Profile(
|
|
||||||
fullname: quote.selectText(".QuoteTweet-fullname").stripText(),
|
|
||||||
username: quote.attr("data-screen-name"),
|
|
||||||
verified: isVerified(quote)
|
|
||||||
)
|
|
||||||
|
|
||||||
result.getQuoteMedia(quote)
|
|
||||||
|
|
||||||
proc parseTweet*(node: XmlNode): Tweet =
|
|
||||||
if node == nil:
|
|
||||||
return Tweet()
|
|
||||||
|
|
||||||
if "withheld" in node.attr("class"):
|
|
||||||
return Tweet(tombstone: getTombstone(node.selectText(".Tombstone-label")))
|
|
||||||
|
|
||||||
let tweet = node.select(".tweet")
|
|
||||||
if tweet == nil:
|
|
||||||
return Tweet()
|
|
||||||
|
|
||||||
result = Tweet(
|
|
||||||
id: parseBiggestInt(tweet.attr("data-item-id")),
|
|
||||||
threadId: parseBiggestInt(tweet.attr("data-conversation-id")),
|
|
||||||
text: getTweetText(tweet),
|
|
||||||
time: getTimestamp(tweet),
|
|
||||||
shortTime: getShortTime(tweet),
|
|
||||||
profile: parseTweetProfile(tweet),
|
|
||||||
stats: parseTweetStats(tweet),
|
|
||||||
reply: parseTweetReply(tweet),
|
|
||||||
mediaTags: getMediaTags(tweet),
|
|
||||||
location: getTweetLocation(tweet),
|
|
||||||
hasThread: tweet.select(".content > .self-thread-context") != nil,
|
|
||||||
pinned: "pinned" in tweet.attr("class"),
|
|
||||||
available: true
|
|
||||||
)
|
|
||||||
|
|
||||||
result.getTweetMedia(tweet)
|
|
||||||
result.getTweetCard(tweet)
|
|
||||||
|
|
||||||
let by = tweet.selectText(".js-retweet-text > a > b")
|
|
||||||
if by.len > 0:
|
|
||||||
result.retweet = some Retweet(
|
|
||||||
by: stripText(by),
|
|
||||||
id: parseBiggestInt(tweet.attr("data-retweet-id"))
|
|
||||||
)
|
|
||||||
|
|
||||||
let quote = tweet.select(".QuoteTweet-innerContainer")
|
|
||||||
if quote != nil:
|
|
||||||
result.quote = some parseQuote(quote)
|
|
||||||
|
|
||||||
let tombstone = tweet.select(".Tombstone")
|
|
||||||
if tombstone != nil:
|
|
||||||
if "unavailable" in tombstone.innerText():
|
|
||||||
let quote = Quote(tombstone: getTombstone(node.selectText(".Tombstone-label")))
|
|
||||||
result.quote = some quote
|
|
||||||
|
|
||||||
proc parseChain*(nodes: XmlNode): Chain =
|
|
||||||
if nodes == nil: return
|
|
||||||
result = Chain()
|
|
||||||
for n in nodes.filterIt(it.kind != xnText):
|
|
||||||
let class = n.attr("class").toLower()
|
|
||||||
if "tombstone" in class or "unavailable" in class or "withheld" in class:
|
|
||||||
result.content.add Tweet()
|
|
||||||
elif "morereplies" in class:
|
|
||||||
result.more = getMoreReplies(n)
|
|
||||||
else:
|
|
||||||
result.content.add parseTweet(n)
|
|
||||||
|
|
||||||
proc parseReplies*(replies: XmlNode; skipFirst=false): Result[Chain] =
|
|
||||||
new(result)
|
|
||||||
for i, reply in replies.filterIt(it.kind != xnText):
|
|
||||||
if skipFirst and i == 0: continue
|
|
||||||
let class = reply.attr("class").toLower()
|
|
||||||
if "lone" in class:
|
|
||||||
result.content.add parseChain(reply)
|
|
||||||
elif "showmore" in class:
|
|
||||||
result.minId = reply.selectAttr("button", "data-cursor")
|
|
||||||
result.hasMore = true
|
|
||||||
else:
|
|
||||||
result.content.add parseChain(reply.select(".stream-items"))
|
|
||||||
|
|
||||||
proc parseConversation*(node: XmlNode; after: string): Conversation =
|
|
||||||
let tweet = node.select(".permalink-tweet-container")
|
|
||||||
|
|
||||||
if tweet == nil:
|
|
||||||
return Conversation(tweet: parseTweet(node.select(".permalink-tweet-withheld")))
|
|
||||||
|
|
||||||
result = Conversation(
|
|
||||||
tweet: parseTweet(tweet),
|
|
||||||
before: parseChain(node.select(".in-reply-to .stream-items")),
|
|
||||||
)
|
|
||||||
|
|
||||||
if result.before != nil:
|
|
||||||
let maxId = node.selectAttr(".in-reply-to .stream-container", "data-max-position")
|
|
||||||
if maxId.len > 0:
|
|
||||||
result.before.more = -1
|
|
||||||
|
|
||||||
let replies = node.select(".replies-to .stream-items")
|
|
||||||
if replies == nil: return
|
|
||||||
|
|
||||||
let nodes = replies.filterIt(it.kind != xnText and "self" in it.attr("class"))
|
|
||||||
if nodes.len > 0 and "self" in nodes[0].attr("class"):
|
|
||||||
result.after = parseChain(nodes[0].select(".stream-items"))
|
|
||||||
|
|
||||||
result.replies = parseReplies(replies, result.after != nil)
|
|
||||||
|
|
||||||
result.replies.beginning = after.len == 0
|
|
||||||
if result.replies.minId.len == 0:
|
|
||||||
result.replies.minId = node.selectAttr(".replies-to .stream-container", "data-min-position")
|
|
||||||
result.replies.hasMore = node.select(".stream-footer .has-more-items") != nil
|
|
||||||
|
|
||||||
proc parseTimeline*(node: XmlNode; after: string): Timeline =
|
|
||||||
if node == nil: return Timeline()
|
|
||||||
result = Timeline(
|
|
||||||
content: parseChain(node.select(".stream > .stream-items")).content,
|
|
||||||
minId: node.attr("data-min-position"),
|
|
||||||
maxId: node.attr("data-max-position"),
|
|
||||||
hasMore: node.select(".has-more-items") != nil,
|
|
||||||
beginning: after.len == 0
|
|
||||||
)
|
|
||||||
|
|
||||||
proc parseVideo*(node: JsonNode; tweetId: int64): Video =
|
|
||||||
let
|
|
||||||
track = node{"track"}
|
|
||||||
cType = track["contentType"].to(string)
|
|
||||||
pType = track["playbackType"].to(string)
|
|
||||||
|
|
||||||
case cType
|
|
||||||
of "media_entity":
|
|
||||||
result = Video(
|
|
||||||
playbackType: if "mp4" in pType: mp4 else: m3u8,
|
|
||||||
contentId: track["contentId"].to(string),
|
|
||||||
durationMs: track["durationMs"].to(int),
|
|
||||||
views: track["viewCount"].to(string),
|
|
||||||
url: track["playbackUrl"].to(string),
|
|
||||||
available: track{"mediaAvailability"}["status"].to(string) == "available",
|
|
||||||
reason: track{"mediaAvailability"}["reason"].to(string))
|
|
||||||
of "vmap":
|
|
||||||
result = Video(
|
|
||||||
playbackType: vmap,
|
|
||||||
durationMs: track.getOrDefault("durationMs").getInt(0),
|
|
||||||
url: track["vmapUrl"].to(string),
|
|
||||||
available: true)
|
|
||||||
else:
|
|
||||||
echo "Can't parse video of type ", cType, " ", tweetId
|
|
||||||
|
|
||||||
result.videoId = $tweetId
|
|
||||||
result.thumb = node["posterImage"].to(string)
|
|
||||||
|
|
||||||
proc parsePoll*(node: XmlNode): Poll =
|
|
||||||
let
|
|
||||||
choices = node.selectAll(".PollXChoice-choice")
|
|
||||||
votes = node.selectText(".PollXChoice-footer--total")
|
|
||||||
|
|
||||||
result.votes = votes.strip().split(" ")[0]
|
|
||||||
result.status = node.selectText(".PollXChoice-footer--time")
|
|
||||||
|
|
||||||
for choice in choices:
|
|
||||||
for span in choice.select(".PollXChoice-choice--text").filterIt(it.kind != xnText):
|
|
||||||
if span.attr("class").len == 0:
|
|
||||||
result.options.add span.innerText()
|
|
||||||
elif "progress" in span.attr("class"):
|
|
||||||
result.values.add parseInt(span.innerText()[0 .. ^2])
|
|
||||||
|
|
||||||
var highest = 0
|
|
||||||
for i, n in result.values:
|
|
||||||
if n > highest:
|
|
||||||
highest = n
|
|
||||||
result.leader = i
|
|
||||||
|
|
||||||
proc parsePhotoRail*(node: XmlNode): seq[GalleryPhoto] =
|
|
||||||
for img in node.selectAll(".tweet-media-img-placeholder"):
|
|
||||||
result.add GalleryPhoto(
|
|
||||||
url: img.attr("data-image-url"),
|
|
||||||
tweetId: img.attr("data-tweet-id"),
|
|
||||||
color: img.attr("background-color").replace("style: ", "")
|
|
||||||
)
|
|
||||||
|
|
||||||
proc parseCard*(card: var Card; node: XmlNode) =
|
|
||||||
card.title = node.selectText("h2.TwitterCard-title")
|
|
||||||
card.text = node.selectText("p.tcu-resetMargin")
|
|
||||||
card.dest = node.selectText("span.SummaryCard-destination")
|
|
||||||
|
|
||||||
if card.url.len == 0:
|
|
||||||
card.url = node.selectAttr("a", "href")
|
|
||||||
if card.url.len == 0:
|
|
||||||
card.url = node.selectAttr(".ConvoCard-thankYouContent", "data-thank-you-url")
|
|
||||||
|
|
||||||
let image = node.select(".tcu-imageWrapper img")
|
|
||||||
if image != nil:
|
|
||||||
# workaround for issue 11713
|
|
||||||
card.image = some image.attr("data-src").replace("gname", "g&name")
|
|
||||||
|
|
||||||
if card.kind == liveEvent:
|
|
||||||
card.text = card.title
|
|
||||||
card.title = node.selectText(".TwitterCard-attribution--category")
|
|
||||||
|
|
@ -1,294 +0,0 @@
|
||||||
import xmltree, times, uri, options, json
|
|
||||||
import strtabs, strformat, strutils, sequtils
|
|
||||||
import regex
|
|
||||||
|
|
||||||
import types, formatters
|
|
||||||
|
|
||||||
from q import nil
|
|
||||||
from htmlgen import a
|
|
||||||
|
|
||||||
const
|
|
||||||
thumbRegex = re".+:url\('([^']+)'\)"
|
|
||||||
gifRegex = re".+thumb/([^\.']+)\.[jpng].*"
|
|
||||||
reColor = re"a:active \{\n +color: (#[A-Z0-9]+)"
|
|
||||||
|
|
||||||
proc selectAll*(node: XmlNode; selector: string): seq[XmlNode] =
|
|
||||||
if node == nil: return
|
|
||||||
q.select(node, selector)
|
|
||||||
|
|
||||||
proc select*(node: XmlNode; selector: string): XmlNode =
|
|
||||||
if node == nil: return
|
|
||||||
let nodes = node.selectAll(selector)
|
|
||||||
if nodes.len > 0: nodes[0] else: nil
|
|
||||||
|
|
||||||
proc selectAttr*(node: XmlNode; selector: string; attr: string): string =
|
|
||||||
let res = node.select(selector)
|
|
||||||
if res == nil: "" else: res.attr(attr)
|
|
||||||
|
|
||||||
proc selectText*(node: XmlNode; selector: string): string =
|
|
||||||
let res = node.select(selector)
|
|
||||||
result = if res == nil: "" else: res.innerText()
|
|
||||||
|
|
||||||
proc getHeader(profile: XmlNode): XmlNode =
|
|
||||||
result = profile.select(".permalink-header")
|
|
||||||
if result == nil:
|
|
||||||
result = profile.select(".stream-item-header")
|
|
||||||
if result == nil:
|
|
||||||
result = profile.select(".ProfileCard-userFields")
|
|
||||||
if result == nil:
|
|
||||||
result = profile
|
|
||||||
|
|
||||||
proc isVerified*(profile: XmlNode): bool =
|
|
||||||
getHeader(profile).select(".Icon.Icon--verified") != nil
|
|
||||||
|
|
||||||
proc isProtected*(profile: XmlNode): bool =
|
|
||||||
getHeader(profile).select(".Icon.Icon--protected") != nil
|
|
||||||
|
|
||||||
proc parseText*(text: XmlNode; skipLink=""): string =
|
|
||||||
if text == nil: return
|
|
||||||
for el in text:
|
|
||||||
case el.kind
|
|
||||||
of xnText:
|
|
||||||
result.add el
|
|
||||||
of xnElement:
|
|
||||||
if el.attrs == nil:
|
|
||||||
if el.tag == "strong":
|
|
||||||
result.add $el
|
|
||||||
continue
|
|
||||||
|
|
||||||
let class = el.attr("class")
|
|
||||||
if "data-expanded-url" in el.attrs:
|
|
||||||
let url = el.attr("data-expanded-url")
|
|
||||||
if url == skipLink: continue
|
|
||||||
if "u-hidden" in class and result.len > 0:
|
|
||||||
result.add "\n"
|
|
||||||
result.add a(shortLink(url), href=url)
|
|
||||||
elif "ashtag" in class or "hashflag" in class:
|
|
||||||
let hash = el.innerText()
|
|
||||||
result.add a(hash, href=("/search?q=" & encodeUrl(hash)))
|
|
||||||
elif "atreply" in class:
|
|
||||||
result.add a(el.innerText(), href=el.attr("href"))
|
|
||||||
elif "Emoji" in class:
|
|
||||||
result.add el.attr("alt")
|
|
||||||
else: discard
|
|
||||||
|
|
||||||
proc getQuoteText*(tweet: XmlNode): string =
|
|
||||||
parseText(tweet.select(".QuoteTweet-text"))
|
|
||||||
|
|
||||||
proc getTweetText*(tweet: XmlNode): string =
|
|
||||||
let
|
|
||||||
quote = tweet.select(".QuoteTweet")
|
|
||||||
text = tweet.select(".tweet-text")
|
|
||||||
link = text.selectAttr("a.twitter-timeline-link.u-hidden", "data-expanded-url")
|
|
||||||
parseText(text, if quote != nil: link else: "")
|
|
||||||
|
|
||||||
proc getTimestamp*(tweet: XmlNode): Time =
|
|
||||||
let time = tweet.selectAttr(".js-short-timestamp", "data-time")
|
|
||||||
fromUnix(if time.len > 0: parseBiggestInt(time) else: 0)
|
|
||||||
|
|
||||||
proc getShortTime*(tweet: XmlNode): string =
|
|
||||||
tweet.selectText(".js-short-timestamp")
|
|
||||||
|
|
||||||
proc getDate*(node: XmlNode; selector: string): Time =
|
|
||||||
let date = node.select(selector)
|
|
||||||
if date == nil: return
|
|
||||||
parseTime(date.attr("title"), "h:mm tt - d MMM YYYY", utc())
|
|
||||||
|
|
||||||
proc getName*(profile: XmlNode; selector: string): string =
|
|
||||||
profile.selectText(selector).stripText()
|
|
||||||
|
|
||||||
proc getUsername*(profile: XmlNode; selector: string): string =
|
|
||||||
profile.selectText(selector).strip(chars={'@', ' ', '\n'})
|
|
||||||
|
|
||||||
proc getBio*(profile: XmlNode; selector: string; fallback=""): string =
|
|
||||||
var bio = profile.select(selector)
|
|
||||||
if bio == nil and fallback.len > 0:
|
|
||||||
bio = profile.select(fallback)
|
|
||||||
parseText(bio)
|
|
||||||
|
|
||||||
proc getLocation*(profile: XmlNode): string =
|
|
||||||
let sel = ".ProfileHeaderCard-locationText"
|
|
||||||
result = profile.selectText(sel).stripText()
|
|
||||||
|
|
||||||
let link = profile.selectAttr(sel & " a", "data-place-id")
|
|
||||||
if link.len > 0:
|
|
||||||
result &= ":" & link
|
|
||||||
|
|
||||||
proc getAvatar*(profile: XmlNode; selector: string): string =
|
|
||||||
profile.selectAttr(selector, "src").getUserpic()
|
|
||||||
|
|
||||||
proc getBanner*(node: XmlNode): string =
|
|
||||||
let url = node.selectAttr("svg > image", "xlink:href")
|
|
||||||
if url.len > 0:
|
|
||||||
result = url.replace("600x200", "1500x500")
|
|
||||||
else:
|
|
||||||
result = node.selectAttr(".ProfileCard-bg", "style")
|
|
||||||
result = result.replace("background-color: ", "")
|
|
||||||
|
|
||||||
if result.len == 0:
|
|
||||||
result = "#161616"
|
|
||||||
|
|
||||||
proc getTimelineBanner*(node: XmlNode): string =
|
|
||||||
let banner = node.select(".ProfileCanopy-headerBg img")
|
|
||||||
let img = banner.attr("src")
|
|
||||||
if img.len > 0:
|
|
||||||
return img
|
|
||||||
|
|
||||||
let style = node.select("style").innerText()
|
|
||||||
var m: RegexMatch
|
|
||||||
if style.find(reColor, m):
|
|
||||||
return style[m.group(0)[0]]
|
|
||||||
|
|
||||||
proc getMediaCount*(node: XmlNode): string =
|
|
||||||
let text = node.selectText(".PhotoRail-headingWithCount")
|
|
||||||
return text.stripText().split(" ")[0]
|
|
||||||
|
|
||||||
proc getProfileStats*(profile: var Profile; node: XmlNode) =
|
|
||||||
for s in node.selectAll( ".ProfileNav-stat"):
|
|
||||||
let text = s.attr("title").split(" ")[0]
|
|
||||||
case s.attr("data-nav")
|
|
||||||
of "followers": profile.followers = text
|
|
||||||
of "following": profile.following = text
|
|
||||||
of "favorites": profile.likes = text
|
|
||||||
of "tweets": profile.tweets = text
|
|
||||||
|
|
||||||
proc getPopupStats*(profile: var Profile; node: XmlNode) =
|
|
||||||
for s in node.selectAll( ".ProfileCardStats-statLink"):
|
|
||||||
let text = s.attr("title").split(" ")[0]
|
|
||||||
case s.attr("href").split("/")[^1]
|
|
||||||
of "followers": profile.followers = text
|
|
||||||
of "following": profile.following = text
|
|
||||||
else: profile.tweets = text
|
|
||||||
|
|
||||||
proc getIntentStats*(profile: var Profile; node: XmlNode) =
|
|
||||||
profile.tweets = "?"
|
|
||||||
for s in node.selectAll( "dd.count > a"):
|
|
||||||
let text = s.innerText()
|
|
||||||
case s.attr("href").split("/")[^1]
|
|
||||||
of "followers": profile.followers = text
|
|
||||||
of "following": profile.following = text
|
|
||||||
|
|
||||||
proc parseTweetStats*(node: XmlNode): TweetStats =
|
|
||||||
result = TweetStats()
|
|
||||||
for action in node.selectAll(".ProfileTweet-actionCountForAria"):
|
|
||||||
let text = action.innerText.split()
|
|
||||||
case text[1][0 .. 2]
|
|
||||||
of "ret": result.retweets = text[0].parseInt
|
|
||||||
of "rep": result.replies = text[0].parseInt
|
|
||||||
of "lik": result.likes = text[0].parseInt
|
|
||||||
|
|
||||||
proc parseTweetReply*(node: XmlNode): seq[string] =
|
|
||||||
let reply = node.select(".ReplyingToContextBelowAuthor")
|
|
||||||
if reply == nil: return
|
|
||||||
|
|
||||||
let selector = if "Quote" in node.attr("class"): "b"
|
|
||||||
else: "a b"
|
|
||||||
|
|
||||||
result = reply.selectAll(selector).map(innerText)
|
|
||||||
|
|
||||||
proc getGif(player: XmlNode): Gif =
|
|
||||||
let
|
|
||||||
thumb = player.attr("style").replace(thumbRegex, "$1")
|
|
||||||
id = thumb.replace(gifRegex, "$1")
|
|
||||||
url = &"https://video.twimg.com/tweet_video/{id}.mp4"
|
|
||||||
Gif(url: url, thumb: thumb)
|
|
||||||
|
|
||||||
proc getTweetMedia*(tweet: Tweet; node: XmlNode) =
|
|
||||||
for photo in node.selectAll(".AdaptiveMedia-photoContainer"):
|
|
||||||
tweet.photos.add photo.attrs["data-image-url"]
|
|
||||||
|
|
||||||
let player = node.select(".PlayableMedia")
|
|
||||||
if player == nil: return
|
|
||||||
|
|
||||||
let attrib = player.select(".PlayableMedia-attribution")
|
|
||||||
if attrib != nil:
|
|
||||||
tweet.attribution = some Profile(
|
|
||||||
username: attrib.attr("href").strip(chars={'/'}),
|
|
||||||
fullname: attrib.selectText(".fullname"),
|
|
||||||
userpic: attrib.selectAttr(".avatar", "src")
|
|
||||||
)
|
|
||||||
|
|
||||||
if "gif" in player.attr("class"):
|
|
||||||
tweet.gif = some getGif(player.select(".PlayableMedia-player"))
|
|
||||||
elif "video" in player.attr("class"):
|
|
||||||
let
|
|
||||||
thumb = player.selectAttr(".PlayableMedia-player", "style").split("'")
|
|
||||||
desc = player.selectText(".PlayableMedia-description")
|
|
||||||
title = player.selectText(".PlayableMedia-title")
|
|
||||||
var video = Video(title: title, description: desc)
|
|
||||||
if thumb.len > 1:
|
|
||||||
video.thumb = thumb[^2]
|
|
||||||
tweet.video = some video
|
|
||||||
|
|
||||||
proc getQuoteMedia*(quote: var Quote; node: XmlNode) =
|
|
||||||
if node.select(".QuoteTweet--sensitive") != nil:
|
|
||||||
quote.sensitive = true
|
|
||||||
return
|
|
||||||
|
|
||||||
let media = node.select(".QuoteMedia")
|
|
||||||
if media != nil:
|
|
||||||
quote.thumb = media.selectAttr("img", "src")
|
|
||||||
|
|
||||||
let badge = node.select(".AdaptiveMedia-badgeText")
|
|
||||||
let gifBadge = node.select(".Icon--gifBadge")
|
|
||||||
|
|
||||||
if badge != nil:
|
|
||||||
quote.badge = badge.innerText()
|
|
||||||
elif gifBadge != nil:
|
|
||||||
quote.badge = "GIF"
|
|
||||||
|
|
||||||
proc getTweetCard*(tweet: Tweet; node: XmlNode) =
|
|
||||||
if node.attr("data-has-cards") == "false": return
|
|
||||||
var cardType = node.attr("data-card2-type")
|
|
||||||
|
|
||||||
if ":" in cardType:
|
|
||||||
cardType = cardType.split(":")[^1]
|
|
||||||
|
|
||||||
if "poll" in cardType:
|
|
||||||
tweet.poll = some Poll()
|
|
||||||
return
|
|
||||||
|
|
||||||
if "message_me" in cardType:
|
|
||||||
return
|
|
||||||
|
|
||||||
let cardDiv = node.select(".card2 > .js-macaw-cards-iframe-container")
|
|
||||||
if cardDiv == nil: return
|
|
||||||
|
|
||||||
var card = Card(
|
|
||||||
id: $tweet.id,
|
|
||||||
query: cardDiv.attr("data-src")
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
card.kind = parseEnum[CardKind](cardType)
|
|
||||||
except ValueError:
|
|
||||||
card.kind = summary
|
|
||||||
|
|
||||||
let cardUrl = cardDiv.attr("data-card-url")
|
|
||||||
for n in node.selectAll(".tweet-text a"):
|
|
||||||
if n.attr("href") == cardUrl:
|
|
||||||
card.url = n.attr("data-expanded-url")
|
|
||||||
|
|
||||||
tweet.card = some card
|
|
||||||
|
|
||||||
proc getMoreReplies*(node: XmlNode): int64 =
|
|
||||||
let text = node.innerText().strip()
|
|
||||||
try:
|
|
||||||
result = parseBiggestInt(text.split(" ")[0])
|
|
||||||
except:
|
|
||||||
result = -1
|
|
||||||
|
|
||||||
proc getMediaTags*(node: XmlNode): seq[Profile] =
|
|
||||||
let usernames = node.attr("data-tagged")
|
|
||||||
if usernames.len == 0: return
|
|
||||||
let users = parseJson(node.attr("data-reply-to-users-json"))
|
|
||||||
for user in users:
|
|
||||||
let un = user["screen_name"].getStr
|
|
||||||
if un notin usernames: continue
|
|
||||||
result.add Profile(username: un, fullname: user["name"].getStr)
|
|
||||||
|
|
||||||
proc getTweetLocation*(node: XmlNode): string =
|
|
||||||
let geo = node.select(".js-geo-pivot-link")
|
|
||||||
if geo == nil: return
|
|
||||||
result = geo.innerText().stripText()
|
|
||||||
result &= ":" & geo.attr("data-place-id")
|
|
||||||
Loading…
Reference in New Issue