Revamp profile api to display more metadata

This commit is contained in:
Zed 2019-08-11 21:26:55 +02:00
parent 3f1d9777b6
commit 7171486f03
9 changed files with 192 additions and 61 deletions

View File

@ -445,14 +445,6 @@ video {
display: flex; display: flex;
} }
.profile-card-tabs {
display: flex;
justify-content: space-between;
align-items: center;
flex: 1 1 auto;
max-width: 100%;
}
.profile-card-tabs-name { .profile-card-tabs-name {
max-width: 100%; max-width: 100%;
} }
@ -496,20 +488,26 @@ video {
.profile-card-extra { .profile-card-extra {
display: contents; display: contents;
flex: 100%; flex: 100%;
margin-top: 4px; margin-top: 6px;
} }
.profile-bio { .profile-bio {
overflow: hidden; overflow: hidden;
overflow-wrap: break-word; overflow-wrap: break-word;
width: 100%; width: 100%;
margin: 10px -6px 0px 0px; margin: 4px -6px 6px 0px;
} }
.profile-bio p { .profile-bio p {
margin: 0; margin: 0;
} }
.profile-location, .profile-website, .profile-joindate {
color: #f8f8f2cf;
margin: 2px 0px;
width: 100%;
}
.profile-description { .profile-description {
font-size: 14px; font-size: 14px;
font-weight: 400; font-weight: 400;
@ -735,6 +733,7 @@ video {
margin: 0; margin: 0;
padding: 0; padding: 0;
width: 100%; width: 100%;
justify-content: space-between;
} }
.profile-statlist > li { .profile-statlist > li {
@ -742,19 +741,6 @@ video {
text-align: center; text-align: center;
} }
.profile-statlist .posts {
flex: 0.4 1 0;
}
.profile-statlist .followers {
flex: 1 1 0;
padding: 0 3px;
}
.profile-statlist .following {
flex: 0.5 1 0;
}
.profile-stat-header { .profile-stat-header {
font-weight: bold; font-weight: bold;
} }

View File

@ -6,7 +6,7 @@ import types, parser, parserutils, formatters, search
const const
lang = "en-US,en;q=0.9" lang = "en-US,en;q=0.9"
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3" accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
jsonAccept = "application/json, text/javascript, */*; q=0.01" jsonAccept = "application/json, text/javascript, */*; q=0.01"
base = parseUri("https://twitter.com/") base = parseUri("https://twitter.com/")
@ -38,7 +38,7 @@ macro genMediaGet(media: untyped; token=false) =
single = ident("get" & mediaName) single = ident("get" & mediaName)
quote do: quote do:
proc `multi`(thread: Thread; agent: string; token="") {.async.} = proc `multi`(thread: Thread | Timeline; agent: string; token="") {.async.} =
if thread == nil: return if thread == nil: return
var `media` = thread.tweets.filterIt(it.`media`.isSome) var `media` = thread.tweets.filterIt(it.`media`.isSome)
when `token`: when `token`:
@ -165,7 +165,7 @@ proc getPoll*(tweet: Tweet; agent: string) {.async.} =
if tweet.poll.isNone(): return if tweet.poll.isNone(): return
let headers = newHttpHeaders({ let headers = newHttpHeaders({
"Accept": cardAccept, "Accept": accept,
"Referer": $(base / getLink(tweet)), "Referer": $(base / getLink(tweet)),
"User-Agent": agent, "User-Agent": agent,
"Authority": "twitter.com", "Authority": "twitter.com",
@ -182,7 +182,7 @@ proc getCard*(tweet: Tweet; agent: string) {.async.} =
if tweet.card.isNone(): return if tweet.card.isNone(): return
let headers = newHttpHeaders({ let headers = newHttpHeaders({
"Accept": cardAccept, "Accept": accept,
"Referer": $(base / getLink(tweet)), "Referer": $(base / getLink(tweet)),
"User-Agent": agent, "User-Agent": agent,
"Authority": "twitter.com", "Authority": "twitter.com",
@ -350,3 +350,39 @@ proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.
let json = await fetchJson(base / timelineSearchUrl ? params, headers) let json = await fetchJson(base / timelineSearchUrl ? params, headers)
result = await finishTimeline(json, some(query), after, agent) result = await finishTimeline(json, some(query), after, agent)
proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
let headers = newHttpHeaders({
"authority": "twitter.com",
"accept": accept,
"referer": "https://twitter.com/" & username,
"accept-language": lang
})
var url = base / username
if after.len > 0:
url = url ? {"max_position": after}
let
html = await fetchHtml(url, headers)
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
profile = parseTimelineProfile(html)
vidsFut = getVideos(timeline, agent)
pollFut = getPolls(timeline, agent)
cardFut = getCards(timeline, agent)
await all(vidsFut, pollFut, cardFut)
result = (profile, timeline)
proc getProfileFull*(username: string): Future[Profile] {.async.} =
let headers = newHttpHeaders({
"authority": "twitter.com",
"accept": accept,
"referer": "https://twitter.com/" & username,
"accept-language": lang
})
let html = await fetchHtml(base / username, headers)
if html == nil: return
result = parseTimelineProfile(html)

View File

@ -9,23 +9,36 @@ withDb:
var profileCacheTime = initDuration(minutes=10) var profileCacheTime = initDuration(minutes=10)
proc outdated(profile: Profile): bool = proc isOutdated*(profile: Profile): bool =
getTime() - profile.updated > profileCacheTime getTime() - profile.updated > profileCacheTime
proc cache*(profile: var Profile) =
withDb:
try:
let p = Profile.getOne("lower(username) = ?", toLower(profile.username))
profile.id = p.id
profile.update()
except KeyError:
if profile.username.len > 0:
profile.insert()
proc hasCachedProfile*(username: string): Option[Profile] =
withDb:
try:
let p = Profile.getOne("lower(username) = ?", toLower(username))
doAssert not p.isOutdated
result = some(p)
except AssertionError, KeyError:
result = none(Profile)
proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} = proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} =
withDb: withDb:
try: try:
result.getOne("username = ?", username) result.getOne("lower(username) = ?", toLower(username))
doAssert not result.outdated() doAssert not result.isOutdated
except AssertionError: except AssertionError, KeyError:
var profile = await getProfile(username, agent) result = await getProfileFull(username)
profile.id = result.id cache(result)
result = profile
result.update()
except KeyError:
result = await getProfile(username, agent)
if result.username.len > 0:
result.insert()
proc setProfileCacheTime*(minutes: int) = proc setProfileCacheTime*(minutes: int) =
profileCacheTime = initDuration(minutes=minutes) profileCacheTime = initDuration(minutes=minutes)

View File

@ -62,7 +62,7 @@ proc stripTwitterUrls*(text: string): string =
result = result.replace(ellipsisRegex, "") result = result.replace(ellipsisRegex, "")
proc getUserpic*(userpic: string; style=""): string = proc getUserpic*(userpic: string; style=""): string =
let pic = userpic.replace(re"_(normal|bigger|mini|200x200)(\.[A-z]+)$", "$2") let pic = userpic.replace(re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$", "$2")
pic.replace(re"(\.[A-z]+)$", style & "$1") pic.replace(re"(\.[A-z]+)$", style & "$1")
proc getUserpic*(profile: Profile; style=""): string = proc getUserpic*(profile: Profile; style=""): string =
@ -77,6 +77,12 @@ proc pageTitle*(profile: Profile): string =
proc pageDesc*(profile: Profile): string = proc pageDesc*(profile: Profile): string =
"The latest tweets from " & profile.fullname "The latest tweets from " & profile.fullname
proc getJoinDate*(profile: Profile): string =
profile.joinDate.format("'Joined' MMMM YYYY")
proc getJoinDateFull*(profile: Profile): string =
profile.joinDate.format("h:mm tt - d MMM YYYY")
proc getTime*(tweet: Tweet): string = proc getTime*(tweet: Tweet): string =
tweet.time.format("d/M/yyyy', ' HH:mm:ss") tweet.time.format("d/M/yyyy', ' HH:mm:ss")

View File

@ -10,20 +10,31 @@ const configPath {.strdefine.} = "./nitter.conf"
let cfg = getConfig(configPath) let cfg = getConfig(configPath)
proc showSingleTimeline(name, after, agent: string; query: Option[Query]): Future[string] {.async.} = proc showSingleTimeline(name, after, agent: string; query: Option[Query]): Future[string] {.async.} =
let profileFut = getCachedProfile(name, agent)
let railFut = getPhotoRail(name, agent) let railFut = getPhotoRail(name, agent)
var timelineFut: Future[Timeline] var timeline: Timeline
if query.isNone: var profile: Profile
timelineFut = getTimeline(name, after, agent) var cachedProfile = hasCachedProfile(name)
else:
timelineFut = getTimelineSearch(get(query), after, agent) if cachedProfile.isSome:
profile = get(cachedProfile)
if query.isNone:
if cachedProfile.isSome:
timeline = await getTimeline(name, after, agent)
else:
(profile, timeline) = await getProfileAndTimeline(name, agent, after)
cache(profile)
else:
var timelineFut = getTimelineSearch(get(query), after, agent)
if cachedProfile.isNone:
profile = await getCachedProfile(name, agent)
timeline = await timelineFut
let profile = await profileFut
if profile.username.len == 0: if profile.username.len == 0:
return "" return ""
let profileHtml = renderProfile(profile, await timelineFut, await railFut) let profileHtml = renderProfile(profile, timeline, await railFut)
return renderMain(profileHtml, title=cfg.title, titleText=pageTitle(profile), desc=pageDesc(profile)) return renderMain(profileHtml, title=cfg.title, titleText=pageTitle(profile), desc=pageDesc(profile))
proc showMultiTimeline(names: seq[string]; after, agent: string; query: Option[Query]): Future[string] {.async.} = proc showMultiTimeline(names: seq[string]; after, agent: string; query: Option[Query]): Future[string] {.async.} =

View File

@ -2,6 +2,26 @@ import xmltree, sequtils, strutils, json
import types, parserutils, formatters import types, parserutils, formatters
proc parseTimelineProfile*(node: XmlNode): Profile =
let profile = node.select(".ProfileHeaderCard")
if profile == nil: return
let pre = ".ProfileHeaderCard-"
result = Profile(
fullname: profile.getName(pre & "nameLink"),
username: profile.getUsername(pre & "screenname"),
joinDate: profile.getDate(pre & "joinDateText"),
location: profile.selectText(pre & "locationText").stripText(),
website: profile.selectText(pre & "url").stripText(),
bio: profile.getBio(pre & "bio"),
userpic: node.getAvatar(".profile-picture img"),
verified: isVerified(profile),
protected: isProtected(profile),
banner: getTimelineBanner(node)
)
result.getProfileStats(node.select(".ProfileNav-list"))
proc parsePopupProfile*(node: XmlNode): Profile = proc parsePopupProfile*(node: XmlNode): Profile =
let profile = node.select(".profile-card") let profile = node.select(".profile-card")
if profile == nil: return if profile == nil: return
@ -125,6 +145,16 @@ proc parseConversation*(node: XmlNode): Conversation =
else: else:
result.replies.add parseThread(thread) result.replies.add parseThread(thread)
proc parseTimeline*(node: XmlNode; after: string): Timeline =
if node == nil: return
result = Timeline(
tweets: parseThread(node.select(".stream > .stream-items")).tweets,
minId: node.attr("data-min-position"),
maxId: node.attr("data-max-position"),
hasMore: node.select(".has-more-items") != nil,
beginning: after.len == 0
)
proc parseVideo*(node: JsonNode; tweetId: string): Video = proc parseVideo*(node: JsonNode; tweetId: string): Video =
let let
track = node{"track"} track = node{"track"}

View File

@ -32,6 +32,8 @@ proc getHeader(profile: XmlNode): XmlNode =
result = profile.select(".stream-item-header") result = profile.select(".stream-item-header")
if result == nil: if result == nil:
result = profile.select(".ProfileCard-userFields") result = profile.select(".ProfileCard-userFields")
if result == nil:
result = profile
proc isVerified*(profile: XmlNode): bool = proc isVerified*(profile: XmlNode): bool =
getHeader(profile).select(".Icon.Icon--verified") != nil getHeader(profile).select(".Icon.Icon--verified") != nil
@ -39,12 +41,6 @@ proc isVerified*(profile: XmlNode): bool =
proc isProtected*(profile: XmlNode): bool = proc isProtected*(profile: XmlNode): bool =
getHeader(profile).select(".Icon.Icon--protected") != nil getHeader(profile).select(".Icon.Icon--protected") != nil
proc getName*(profile: XmlNode; selector: string): string =
profile.selectText(selector).stripText()
proc getUsername*(profile: XmlNode; selector: string): string =
profile.selectText(selector).strip(chars={'@', ' '})
proc emojify*(node: XmlNode) = proc emojify*(node: XmlNode) =
for i in node.selectAll(".Emoji"): for i in node.selectAll(".Emoji"):
i.add newText(i.attr("alt")) i.add newText(i.attr("alt"))
@ -79,23 +75,54 @@ proc getTimestamp*(tweet: XmlNode): Time =
proc getShortTime*(tweet: XmlNode): string = proc getShortTime*(tweet: XmlNode): string =
getTime(tweet).innerText() getTime(tweet).innerText()
proc getDate*(node: XmlNode; selector: string): Time =
let date = node.select(selector)
if date == nil: return
parseTime(date.attr("title"), "h:mm tt - d MMM YYYY", utc())
proc getName*(profile: XmlNode; selector: string): string =
profile.selectText(selector).stripText()
proc getUsername*(profile: XmlNode; selector: string): string =
profile.selectText(selector).strip(chars={'@', ' ', '\n'})
proc getBio*(profile: XmlNode; selector: string): string = proc getBio*(profile: XmlNode; selector: string): string =
profile.selectText(selector).stripText() profile.selectText(selector).stripText()
proc getAvatar*(profile: XmlNode; selector: string): string = proc getAvatar*(profile: XmlNode; selector: string): string =
profile.selectAttr(selector, "src").getUserpic() profile.selectAttr(selector, "src").getUserpic()
proc getBanner*(tweet: XmlNode): string = proc getBanner*(node: XmlNode): string =
let url = tweet.selectAttr("svg > image", "xlink:href") let url = node.selectAttr("svg > image", "xlink:href")
if url.len > 0: if url.len > 0:
result = url.replace("600x200", "1500x500") result = url.replace("600x200", "1500x500")
else: else:
result = tweet.selectAttr(".ProfileCard-bg", "style") result = node.selectAttr(".ProfileCard-bg", "style")
result = result.replace("background-color: ", "") result = result.replace("background-color: ", "")
if result.len == 0: if result.len == 0:
result = "#161616" result = "#161616"
proc getTimelineBanner*(node: XmlNode): string =
let banner = node.select(".ProfileCanopy-headerBg img")
let img = banner.attr("src")
if img.len > 0:
return img
let style = node.select("style").innerText()
var m: RegexMatch
if style.find(re"a:active \{\n +color: (#[A-Z0-9]+)", m):
return style[m.group(0)[0]]
proc getProfileStats*(profile: var Profile; node: XmlNode) =
for s in node.selectAll( ".ProfileNav-stat"):
let text = s.attr("title").split(" ")[0]
case s.attr("data-nav")
of "followers": profile.followers = text
of "following": profile.following = text
of "favorites": profile.likes = text
of "tweets": profile.tweets = text
proc getPopupStats*(profile: var Profile; node: XmlNode) = proc getPopupStats*(profile: var Profile; node: XmlNode) =
for s in node.selectAll( ".ProfileCardStats-statLink"): for s in node.selectAll( ".ProfileCardStats-statLink"):
let text = s.attr("title").split(" ")[0] let text = s.attr("title").split(" ")[0]

View File

@ -12,12 +12,15 @@ db("cache.db", "", "", ""):
Profile* = object Profile* = object
username*: string username*: string
fullname*: string fullname*: string
location*: string
website*: string
bio*: string bio*: string
userpic*: string userpic*: string
banner*: string banner*: string
following*: string following*: string
followers*: string followers*: string
tweets*: string tweets*: string
likes*: string
verified* {. verified* {.
dbType: "STRING", dbType: "STRING",
parseIt: parseBool(it.s) parseIt: parseBool(it.s)
@ -28,6 +31,11 @@ db("cache.db", "", "", ""):
parseIt: parseBool(it.s) parseIt: parseBool(it.s)
formatIt: $it formatIt: $it
.}: bool .}: bool
joinDate* {.
dbType: "INTEGER",
parseIt: it.i.fromUnix(),
formatIt: it.toUnix()
.}: Time
updated* {. updated* {.
dbType: "INTEGER", dbType: "INTEGER",
parseIt: it.i.fromUnix(), parseIt: it.i.fromUnix(),

View File

@ -15,7 +15,6 @@ proc renderProfileCard*(profile: Profile): VNode =
a(class="profile-card-avatar", href=profile.getUserPic().getSigUrl("pic")): a(class="profile-card-avatar", href=profile.getUserPic().getSigUrl("pic")):
genImg(profile.getUserpic("_200x200")) genImg(profile.getUserpic("_200x200"))
tdiv(class="profile-card-tabs"):
tdiv(class="profile-card-tabs-name"): tdiv(class="profile-card-tabs-name"):
linkUser(profile, class="profile-card-fullname") linkUser(profile, class="profile-card-fullname")
linkUser(profile, class="profile-card-username") linkUser(profile, class="profile-card-username")
@ -25,11 +24,26 @@ proc renderProfileCard*(profile: Profile): VNode =
tdiv(class="profile-bio"): tdiv(class="profile-bio"):
p: verbatim linkifyText(profile.bio) p: verbatim linkifyText(profile.bio)
if profile.location.len > 0:
tdiv(class="profile-location"):
span: text "📍 " & profile.location
if profile.website.len > 0:
tdiv(class="profile-website"):
span:
text "🔗 "
a(href=profile.website): text profile.website
tdiv(class="profile-joindate"):
span(title=getJoinDateFull(profile)):
text "📅 " & getJoinDate(profile)
tdiv(class="profile-card-extra-links"): tdiv(class="profile-card-extra-links"):
ul(class="profile-statlist"): ul(class="profile-statlist"):
renderStat(profile.tweets, "posts", text="Tweets") renderStat(profile.tweets, "posts", text="Tweets")
renderStat(profile.followers, "followers") renderStat(profile.followers, "followers")
renderStat(profile.following, "following") renderStat(profile.following, "following")
renderStat(profile.likes, "likes")
proc renderPhotoRail(username: string; photoRail: seq[GalleryPhoto]): VNode = proc renderPhotoRail(username: string; photoRail: seq[GalleryPhoto]): VNode =
buildHtml(tdiv(class="photo-rail-card")): buildHtml(tdiv(class="photo-rail-card")):