From 7171486f032944f9156fafbfce27406241241d26 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 11 Aug 2019 21:26:55 +0200 Subject: [PATCH] Revamp profile api to display more metadata --- public/style.css | 32 +++++++++--------------------- src/api.nim | 44 ++++++++++++++++++++++++++++++++++++++---- src/cache.nim | 37 +++++++++++++++++++++++------------ src/formatters.nim | 8 +++++++- src/nitter.nim | 27 ++++++++++++++++++-------- src/parser.nim | 30 +++++++++++++++++++++++++++++ src/parserutils.nim | 45 ++++++++++++++++++++++++++++++++++--------- src/types.nim | 8 ++++++++ src/views/profile.nim | 22 +++++++++++++++++---- 9 files changed, 192 insertions(+), 61 deletions(-) diff --git a/public/style.css b/public/style.css index d4b2dc6..e6125b3 100644 --- a/public/style.css +++ b/public/style.css @@ -445,14 +445,6 @@ video { display: flex; } -.profile-card-tabs { - display: flex; - justify-content: space-between; - align-items: center; - flex: 1 1 auto; - max-width: 100%; -} - .profile-card-tabs-name { max-width: 100%; } @@ -496,20 +488,26 @@ video { .profile-card-extra { display: contents; flex: 100%; - margin-top: 4px; + margin-top: 6px; } .profile-bio { overflow: hidden; overflow-wrap: break-word; width: 100%; - margin: 10px -6px 0px 0px; + margin: 4px -6px 6px 0px; } .profile-bio p { margin: 0; } +.profile-location, .profile-website, .profile-joindate { + color: #f8f8f2cf; + margin: 2px 0px; + width: 100%; +} + .profile-description { font-size: 14px; font-weight: 400; @@ -735,6 +733,7 @@ video { margin: 0; padding: 0; width: 100%; + justify-content: space-between; } .profile-statlist > li { @@ -742,19 +741,6 @@ video { text-align: center; } -.profile-statlist .posts { - flex: 0.4 1 0; -} - -.profile-statlist .followers { - flex: 1 1 0; - padding: 0 3px; -} - -.profile-statlist .following { - flex: 0.5 1 0; -} - .profile-stat-header { font-weight: bold; } diff --git a/src/api.nim b/src/api.nim index 2d52569..a4c0c25 100644 --- a/src/api.nim +++ b/src/api.nim @@ -6,7 +6,7 @@ import types, parser, parserutils, formatters, search const lang = "en-US,en;q=0.9" auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" - cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3" + accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3" jsonAccept = "application/json, text/javascript, */*; q=0.01" base = parseUri("https://twitter.com/") @@ -38,7 +38,7 @@ macro genMediaGet(media: untyped; token=false) = single = ident("get" & mediaName) quote do: - proc `multi`(thread: Thread; agent: string; token="") {.async.} = + proc `multi`(thread: Thread | Timeline; agent: string; token="") {.async.} = if thread == nil: return var `media` = thread.tweets.filterIt(it.`media`.isSome) when `token`: @@ -165,7 +165,7 @@ proc getPoll*(tweet: Tweet; agent: string) {.async.} = if tweet.poll.isNone(): return let headers = newHttpHeaders({ - "Accept": cardAccept, + "Accept": accept, "Referer": $(base / getLink(tweet)), "User-Agent": agent, "Authority": "twitter.com", @@ -182,7 +182,7 @@ proc getCard*(tweet: Tweet; agent: string) {.async.} = if tweet.card.isNone(): return let headers = newHttpHeaders({ - "Accept": cardAccept, + "Accept": accept, "Referer": $(base / getLink(tweet)), "User-Agent": agent, "Authority": "twitter.com", @@ -350,3 +350,39 @@ proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {. let json = await fetchJson(base / timelineSearchUrl ? params, headers) result = await finishTimeline(json, some(query), after, agent) + +proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} = + let headers = newHttpHeaders({ + "authority": "twitter.com", + "accept": accept, + "referer": "https://twitter.com/" & username, + "accept-language": lang + }) + + var url = base / username + if after.len > 0: + url = url ? {"max_position": after} + + let + html = await fetchHtml(url, headers) + timeline = parseTimeline(html.select("#timeline > .stream-container"), after) + profile = parseTimelineProfile(html) + + vidsFut = getVideos(timeline, agent) + pollFut = getPolls(timeline, agent) + cardFut = getCards(timeline, agent) + + await all(vidsFut, pollFut, cardFut) + result = (profile, timeline) + +proc getProfileFull*(username: string): Future[Profile] {.async.} = + let headers = newHttpHeaders({ + "authority": "twitter.com", + "accept": accept, + "referer": "https://twitter.com/" & username, + "accept-language": lang + }) + + let html = await fetchHtml(base / username, headers) + if html == nil: return + result = parseTimelineProfile(html) diff --git a/src/cache.nim b/src/cache.nim index a9a31be..f010973 100644 --- a/src/cache.nim +++ b/src/cache.nim @@ -9,23 +9,36 @@ withDb: var profileCacheTime = initDuration(minutes=10) -proc outdated(profile: Profile): bool = +proc isOutdated*(profile: Profile): bool = getTime() - profile.updated > profileCacheTime +proc cache*(profile: var Profile) = + withDb: + try: + let p = Profile.getOne("lower(username) = ?", toLower(profile.username)) + profile.id = p.id + profile.update() + except KeyError: + if profile.username.len > 0: + profile.insert() + +proc hasCachedProfile*(username: string): Option[Profile] = + withDb: + try: + let p = Profile.getOne("lower(username) = ?", toLower(username)) + doAssert not p.isOutdated + result = some(p) + except AssertionError, KeyError: + result = none(Profile) + proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} = withDb: try: - result.getOne("username = ?", username) - doAssert not result.outdated() - except AssertionError: - var profile = await getProfile(username, agent) - profile.id = result.id - result = profile - result.update() - except KeyError: - result = await getProfile(username, agent) - if result.username.len > 0: - result.insert() + result.getOne("lower(username) = ?", toLower(username)) + doAssert not result.isOutdated + except AssertionError, KeyError: + result = await getProfileFull(username) + cache(result) proc setProfileCacheTime*(minutes: int) = profileCacheTime = initDuration(minutes=minutes) diff --git a/src/formatters.nim b/src/formatters.nim index 7f37c00..9e27fe5 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -62,7 +62,7 @@ proc stripTwitterUrls*(text: string): string = result = result.replace(ellipsisRegex, "") proc getUserpic*(userpic: string; style=""): string = - let pic = userpic.replace(re"_(normal|bigger|mini|200x200)(\.[A-z]+)$", "$2") + let pic = userpic.replace(re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$", "$2") pic.replace(re"(\.[A-z]+)$", style & "$1") proc getUserpic*(profile: Profile; style=""): string = @@ -77,6 +77,12 @@ proc pageTitle*(profile: Profile): string = proc pageDesc*(profile: Profile): string = "The latest tweets from " & profile.fullname +proc getJoinDate*(profile: Profile): string = + profile.joinDate.format("'Joined' MMMM YYYY") + +proc getJoinDateFull*(profile: Profile): string = + profile.joinDate.format("h:mm tt - d MMM YYYY") + proc getTime*(tweet: Tweet): string = tweet.time.format("d/M/yyyy', ' HH:mm:ss") diff --git a/src/nitter.nim b/src/nitter.nim index d225434..a2dea9a 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -10,20 +10,31 @@ const configPath {.strdefine.} = "./nitter.conf" let cfg = getConfig(configPath) proc showSingleTimeline(name, after, agent: string; query: Option[Query]): Future[string] {.async.} = - let profileFut = getCachedProfile(name, agent) let railFut = getPhotoRail(name, agent) - var timelineFut: Future[Timeline] - if query.isNone: - timelineFut = getTimeline(name, after, agent) - else: - timelineFut = getTimelineSearch(get(query), after, agent) + var timeline: Timeline + var profile: Profile + var cachedProfile = hasCachedProfile(name) + + if cachedProfile.isSome: + profile = get(cachedProfile) + + if query.isNone: + if cachedProfile.isSome: + timeline = await getTimeline(name, after, agent) + else: + (profile, timeline) = await getProfileAndTimeline(name, agent, after) + cache(profile) + else: + var timelineFut = getTimelineSearch(get(query), after, agent) + if cachedProfile.isNone: + profile = await getCachedProfile(name, agent) + timeline = await timelineFut - let profile = await profileFut if profile.username.len == 0: return "" - let profileHtml = renderProfile(profile, await timelineFut, await railFut) + let profileHtml = renderProfile(profile, timeline, await railFut) return renderMain(profileHtml, title=cfg.title, titleText=pageTitle(profile), desc=pageDesc(profile)) proc showMultiTimeline(names: seq[string]; after, agent: string; query: Option[Query]): Future[string] {.async.} = diff --git a/src/parser.nim b/src/parser.nim index 168547f..443b2e8 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -2,6 +2,26 @@ import xmltree, sequtils, strutils, json import types, parserutils, formatters +proc parseTimelineProfile*(node: XmlNode): Profile = + let profile = node.select(".ProfileHeaderCard") + if profile == nil: return + + let pre = ".ProfileHeaderCard-" + result = Profile( + fullname: profile.getName(pre & "nameLink"), + username: profile.getUsername(pre & "screenname"), + joinDate: profile.getDate(pre & "joinDateText"), + location: profile.selectText(pre & "locationText").stripText(), + website: profile.selectText(pre & "url").stripText(), + bio: profile.getBio(pre & "bio"), + userpic: node.getAvatar(".profile-picture img"), + verified: isVerified(profile), + protected: isProtected(profile), + banner: getTimelineBanner(node) + ) + + result.getProfileStats(node.select(".ProfileNav-list")) + proc parsePopupProfile*(node: XmlNode): Profile = let profile = node.select(".profile-card") if profile == nil: return @@ -125,6 +145,16 @@ proc parseConversation*(node: XmlNode): Conversation = else: result.replies.add parseThread(thread) +proc parseTimeline*(node: XmlNode; after: string): Timeline = + if node == nil: return + result = Timeline( + tweets: parseThread(node.select(".stream > .stream-items")).tweets, + minId: node.attr("data-min-position"), + maxId: node.attr("data-max-position"), + hasMore: node.select(".has-more-items") != nil, + beginning: after.len == 0 + ) + proc parseVideo*(node: JsonNode; tweetId: string): Video = let track = node{"track"} diff --git a/src/parserutils.nim b/src/parserutils.nim index 6bb58bf..4052984 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -32,6 +32,8 @@ proc getHeader(profile: XmlNode): XmlNode = result = profile.select(".stream-item-header") if result == nil: result = profile.select(".ProfileCard-userFields") + if result == nil: + result = profile proc isVerified*(profile: XmlNode): bool = getHeader(profile).select(".Icon.Icon--verified") != nil @@ -39,12 +41,6 @@ proc isVerified*(profile: XmlNode): bool = proc isProtected*(profile: XmlNode): bool = getHeader(profile).select(".Icon.Icon--protected") != nil -proc getName*(profile: XmlNode; selector: string): string = - profile.selectText(selector).stripText() - -proc getUsername*(profile: XmlNode; selector: string): string = - profile.selectText(selector).strip(chars={'@', ' '}) - proc emojify*(node: XmlNode) = for i in node.selectAll(".Emoji"): i.add newText(i.attr("alt")) @@ -79,23 +75,54 @@ proc getTimestamp*(tweet: XmlNode): Time = proc getShortTime*(tweet: XmlNode): string = getTime(tweet).innerText() +proc getDate*(node: XmlNode; selector: string): Time = + let date = node.select(selector) + if date == nil: return + parseTime(date.attr("title"), "h:mm tt - d MMM YYYY", utc()) + +proc getName*(profile: XmlNode; selector: string): string = + profile.selectText(selector).stripText() + +proc getUsername*(profile: XmlNode; selector: string): string = + profile.selectText(selector).strip(chars={'@', ' ', '\n'}) + proc getBio*(profile: XmlNode; selector: string): string = profile.selectText(selector).stripText() proc getAvatar*(profile: XmlNode; selector: string): string = profile.selectAttr(selector, "src").getUserpic() -proc getBanner*(tweet: XmlNode): string = - let url = tweet.selectAttr("svg > image", "xlink:href") +proc getBanner*(node: XmlNode): string = + let url = node.selectAttr("svg > image", "xlink:href") if url.len > 0: result = url.replace("600x200", "1500x500") else: - result = tweet.selectAttr(".ProfileCard-bg", "style") + result = node.selectAttr(".ProfileCard-bg", "style") result = result.replace("background-color: ", "") if result.len == 0: result = "#161616" +proc getTimelineBanner*(node: XmlNode): string = + let banner = node.select(".ProfileCanopy-headerBg img") + let img = banner.attr("src") + if img.len > 0: + return img + + let style = node.select("style").innerText() + var m: RegexMatch + if style.find(re"a:active \{\n +color: (#[A-Z0-9]+)", m): + return style[m.group(0)[0]] + +proc getProfileStats*(profile: var Profile; node: XmlNode) = + for s in node.selectAll( ".ProfileNav-stat"): + let text = s.attr("title").split(" ")[0] + case s.attr("data-nav") + of "followers": profile.followers = text + of "following": profile.following = text + of "favorites": profile.likes = text + of "tweets": profile.tweets = text + proc getPopupStats*(profile: var Profile; node: XmlNode) = for s in node.selectAll( ".ProfileCardStats-statLink"): let text = s.attr("title").split(" ")[0] diff --git a/src/types.nim b/src/types.nim index 762f443..ba68dcf 100644 --- a/src/types.nim +++ b/src/types.nim @@ -12,12 +12,15 @@ db("cache.db", "", "", ""): Profile* = object username*: string fullname*: string + location*: string + website*: string bio*: string userpic*: string banner*: string following*: string followers*: string tweets*: string + likes*: string verified* {. dbType: "STRING", parseIt: parseBool(it.s) @@ -28,6 +31,11 @@ db("cache.db", "", "", ""): parseIt: parseBool(it.s) formatIt: $it .}: bool + joinDate* {. + dbType: "INTEGER", + parseIt: it.i.fromUnix(), + formatIt: it.toUnix() + .}: Time updated* {. dbType: "INTEGER", parseIt: it.i.fromUnix(), diff --git a/src/views/profile.nim b/src/views/profile.nim index 9ab9f08..eabc0fb 100644 --- a/src/views/profile.nim +++ b/src/views/profile.nim @@ -15,21 +15,35 @@ proc renderProfileCard*(profile: Profile): VNode = a(class="profile-card-avatar", href=profile.getUserPic().getSigUrl("pic")): genImg(profile.getUserpic("_200x200")) - tdiv(class="profile-card-tabs"): - tdiv(class="profile-card-tabs-name"): - linkUser(profile, class="profile-card-fullname") - linkUser(profile, class="profile-card-username") + tdiv(class="profile-card-tabs-name"): + linkUser(profile, class="profile-card-fullname") + linkUser(profile, class="profile-card-username") tdiv(class="profile-card-extra"): if profile.bio.len > 0: tdiv(class="profile-bio"): p: verbatim linkifyText(profile.bio) + if profile.location.len > 0: + tdiv(class="profile-location"): + span: text "📍 " & profile.location + + if profile.website.len > 0: + tdiv(class="profile-website"): + span: + text "🔗 " + a(href=profile.website): text profile.website + + tdiv(class="profile-joindate"): + span(title=getJoinDateFull(profile)): + text "📅 " & getJoinDate(profile) + tdiv(class="profile-card-extra-links"): ul(class="profile-statlist"): renderStat(profile.tweets, "posts", text="Tweets") renderStat(profile.followers, "followers") renderStat(profile.following, "following") + renderStat(profile.likes, "likes") proc renderPhotoRail(username: string; photoRail: seq[GalleryPhoto]): VNode = buildHtml(tdiv(class="photo-rail-card")):