Revamp profile api to display more metadata

public/style.css

@@ -445,14 +445,6 @@ video {
     display: flex;
 }
 
-.profile-card-tabs {
-    display: flex;
-    justify-content: space-between;
-    align-items: center;
-    flex: 1 1 auto;
-    max-width: 100%;
-}
-
 .profile-card-tabs-name {
     max-width: 100%;
 }
@@ -496,20 +488,26 @@ video {
 .profile-card-extra {
     display: contents;
     flex: 100%;
-    margin-top: 4px;
+    margin-top: 6px;
 }
 
 .profile-bio {
     overflow: hidden;
     overflow-wrap: break-word;
     width: 100%;
-    margin: 10px -6px 0px 0px;
+    margin: 4px -6px 6px 0px;
 }
 
 .profile-bio p {
     margin: 0;
 }
 
+.profile-location, .profile-website, .profile-joindate {
+    color: #f8f8f2cf;
+    margin: 2px 0px;
+    width: 100%;
+}
+
 .profile-description {
     font-size: 14px;
     font-weight: 400;
@@ -735,6 +733,7 @@ video {
     margin: 0;
     padding: 0;
     width: 100%;
+    justify-content: space-between;
 }
 
 .profile-statlist > li {
@@ -742,19 +741,6 @@ video {
     text-align: center;
 }
 
-.profile-statlist .posts {
-    flex: 0.4 1 0;
-}
-
-.profile-statlist .followers {
-    flex: 1 1 0;
-    padding: 0 3px;
-}
-
-.profile-statlist .following {
-    flex: 0.5 1 0;
-}
-
 .profile-stat-header {
     font-weight: bold;
 }

src/api.nim

@@ -6,7 +6,7 @@ import types, parser, parserutils, formatters, search
 const
   lang = "en-US,en;q=0.9"
   auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
-  cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
+  accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
   jsonAccept = "application/json, text/javascript, */*; q=0.01"
 
   base = parseUri("https://twitter.com/")
@@ -38,7 +38,7 @@ macro genMediaGet(media: untyped; token=false) =
     single = ident("get" & mediaName)
 
   quote do:
-    proc `multi`(thread: Thread; agent: string; token="") {.async.} =
+    proc `multi`(thread: Thread | Timeline; agent: string; token="") {.async.} =
       if thread == nil: return
       var `media` = thread.tweets.filterIt(it.`media`.isSome)
       when `token`:
@@ -165,7 +165,7 @@ proc getPoll*(tweet: Tweet; agent: string) {.async.} =
   if tweet.poll.isNone(): return
 
   let headers = newHttpHeaders({
-    "Accept": cardAccept,
+    "Accept": accept,
     "Referer": $(base / getLink(tweet)),
     "User-Agent": agent,
     "Authority": "twitter.com",
@@ -182,7 +182,7 @@ proc getCard*(tweet: Tweet; agent: string) {.async.} =
   if tweet.card.isNone(): return
 
   let headers = newHttpHeaders({
-    "Accept": cardAccept,
+    "Accept": accept,
     "Referer": $(base / getLink(tweet)),
     "User-Agent": agent,
     "Authority": "twitter.com",
@@ -350,3 +350,39 @@ proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.
 
   let json = await fetchJson(base / timelineSearchUrl ? params, headers)
   result = await finishTimeline(json, some(query), after, agent)
+
+proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
+  let headers = newHttpHeaders({
+    "authority": "twitter.com",
+    "accept": accept,
+    "referer": "https://twitter.com/" & username,
+    "accept-language": lang
+  })
+
+  var url = base / username
+  if after.len > 0:
+    url = url ? {"max_position": after}
+
+  let
+    html = await fetchHtml(url, headers)
+    timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
+    profile = parseTimelineProfile(html)
+
+    vidsFut = getVideos(timeline, agent)
+    pollFut = getPolls(timeline, agent)
+    cardFut = getCards(timeline, agent)
+
+  await all(vidsFut, pollFut, cardFut)
+  result = (profile, timeline)
+
+proc getProfileFull*(username: string): Future[Profile] {.async.} =
+  let headers = newHttpHeaders({
+    "authority": "twitter.com",
+    "accept": accept,
+    "referer": "https://twitter.com/" & username,
+    "accept-language": lang
+  })
+
+  let html = await fetchHtml(base / username, headers)
+  if html == nil: return
+  result = parseTimelineProfile(html)

src/cache.nim

@@ -9,23 +9,36 @@ withDb:
 
 var profileCacheTime = initDuration(minutes=10)
 
-proc outdated(profile: Profile): bool =
+proc isOutdated*(profile: Profile): bool =
   getTime() - profile.updated > profileCacheTime
 
+proc cache*(profile: var Profile) =
+  withDb:
+    try:
+      let p = Profile.getOne("lower(username) = ?", toLower(profile.username))
+      profile.id = p.id
+      profile.update()
+    except KeyError:
+      if profile.username.len > 0:
+        profile.insert()
+
+proc hasCachedProfile*(username: string): Option[Profile] =
+  withDb:
+    try:
+      let p = Profile.getOne("lower(username) = ?", toLower(username))
+      doAssert not p.isOutdated
+      result = some(p)
+    except AssertionError, KeyError:
+      result = none(Profile)
+
 proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} =
   withDb:
     try:
-      result.getOne("username = ?", username)
+      result.getOne("lower(username) = ?", toLower(username))
-      doAssert not result.outdated()
+      doAssert not result.isOutdated
-    except AssertionError:
+    except AssertionError, KeyError:
-      var profile = await getProfile(username, agent)
+      result = await getProfileFull(username)
-      profile.id = result.id
+      cache(result)
-      result = profile
-      result.update()
-    except KeyError:
-      result = await getProfile(username, agent)
-      if result.username.len > 0:
-        result.insert()
 
 proc setProfileCacheTime*(minutes: int) =
   profileCacheTime = initDuration(minutes=minutes)

src/formatters.nim

@@ -62,7 +62,7 @@ proc stripTwitterUrls*(text: string): string =
   result = result.replace(ellipsisRegex, "")
 
 proc getUserpic*(userpic: string; style=""): string =
-  let pic = userpic.replace(re"_(normal|bigger|mini|200x200)(\.[A-z]+)$", "$2")
+  let pic = userpic.replace(re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$", "$2")
   pic.replace(re"(\.[A-z]+)$", style & "$1")
 
 proc getUserpic*(profile: Profile; style=""): string =
@@ -77,6 +77,12 @@ proc pageTitle*(profile: Profile): string =
 proc pageDesc*(profile: Profile): string =
   "The latest tweets from " & profile.fullname
 
+proc getJoinDate*(profile: Profile): string =
+  profile.joinDate.format("'Joined' MMMM YYYY")
+
+proc getJoinDateFull*(profile: Profile): string =
+  profile.joinDate.format("h:mm tt - d MMM YYYY")
+
 proc getTime*(tweet: Tweet): string =
   tweet.time.format("d/M/yyyy', ' HH:mm:ss")
 

src/nitter.nim

@@ -10,20 +10,31 @@ const configPath {.strdefine.} = "./nitter.conf"
 let cfg = getConfig(configPath)
 
 proc showSingleTimeline(name, after, agent: string; query: Option[Query]): Future[string] {.async.} =
-  let profileFut = getCachedProfile(name, agent)
   let railFut = getPhotoRail(name, agent)
 
-  var timelineFut: Future[Timeline]
+  var timeline: Timeline
-  if query.isNone:
+  var profile: Profile
-    timelineFut = getTimeline(name, after, agent)
+  var cachedProfile = hasCachedProfile(name)
-  else:
+
-    timelineFut = getTimelineSearch(get(query), after, agent)
+  if cachedProfile.isSome:
+    profile = get(cachedProfile)
+
+  if query.isNone:
+    if cachedProfile.isSome:
+      timeline = await getTimeline(name, after, agent)
+    else:
+      (profile, timeline) = await getProfileAndTimeline(name, agent, after)
+      cache(profile)
+  else:
+    var timelineFut = getTimelineSearch(get(query), after, agent)
+    if cachedProfile.isNone:
+      profile = await getCachedProfile(name, agent)
+    timeline = await timelineFut
 
-  let profile = await profileFut
   if profile.username.len == 0:
     return ""
 
-  let profileHtml = renderProfile(profile, await timelineFut, await railFut)
+  let profileHtml = renderProfile(profile, timeline, await railFut)
   return renderMain(profileHtml, title=cfg.title, titleText=pageTitle(profile), desc=pageDesc(profile))
 
 proc showMultiTimeline(names: seq[string]; after, agent: string; query: Option[Query]): Future[string] {.async.} =

src/parser.nim

@@ -2,6 +2,26 @@ import xmltree, sequtils, strutils, json
 
 import types, parserutils, formatters
 
+proc parseTimelineProfile*(node: XmlNode): Profile =
+  let profile = node.select(".ProfileHeaderCard")
+  if profile == nil: return
+
+  let pre = ".ProfileHeaderCard-"
+  result = Profile(
+    fullname:  profile.getName(pre & "nameLink"),
+    username:  profile.getUsername(pre & "screenname"),
+    joinDate:  profile.getDate(pre & "joinDateText"),
+    location:  profile.selectText(pre & "locationText").stripText(),
+    website:   profile.selectText(pre & "url").stripText(),
+    bio:       profile.getBio(pre & "bio"),
+    userpic:   node.getAvatar(".profile-picture img"),
+    verified:  isVerified(profile),
+    protected: isProtected(profile),
+    banner:    getTimelineBanner(node)
+  )
+
+  result.getProfileStats(node.select(".ProfileNav-list"))
+
 proc parsePopupProfile*(node: XmlNode): Profile =
   let profile = node.select(".profile-card")
   if profile == nil: return
@@ -125,6 +145,16 @@ proc parseConversation*(node: XmlNode): Conversation =
     else:
       result.replies.add parseThread(thread)
 
+proc parseTimeline*(node: XmlNode; after: string): Timeline =
+  if node == nil: return
+  result = Timeline(
+    tweets: parseThread(node.select(".stream > .stream-items")).tweets,
+    minId: node.attr("data-min-position"),
+    maxId: node.attr("data-max-position"),
+    hasMore: node.select(".has-more-items") != nil,
+    beginning: after.len == 0
+  )
+
 proc parseVideo*(node: JsonNode; tweetId: string): Video =
   let
     track = node{"track"}

src/parserutils.nim

@@ -32,6 +32,8 @@ proc getHeader(profile: XmlNode): XmlNode =
     result = profile.select(".stream-item-header")
   if result == nil:
     result = profile.select(".ProfileCard-userFields")
+  if result == nil:
+    result = profile
 
 proc isVerified*(profile: XmlNode): bool =
   getHeader(profile).select(".Icon.Icon--verified") != nil
@@ -39,12 +41,6 @@ proc isVerified*(profile: XmlNode): bool =
 proc isProtected*(profile: XmlNode): bool =
   getHeader(profile).select(".Icon.Icon--protected") != nil
 
-proc getName*(profile: XmlNode; selector: string): string =
-  profile.selectText(selector).stripText()
-
-proc getUsername*(profile: XmlNode; selector: string): string =
-  profile.selectText(selector).strip(chars={'@', ' '})
-
 proc emojify*(node: XmlNode) =
   for i in node.selectAll(".Emoji"):
     i.add newText(i.attr("alt"))
@@ -79,23 +75,54 @@ proc getTimestamp*(tweet: XmlNode): Time =
 proc getShortTime*(tweet: XmlNode): string =
   getTime(tweet).innerText()
 
+proc getDate*(node: XmlNode; selector: string): Time =
+  let date = node.select(selector)
+  if date == nil: return
+  parseTime(date.attr("title"), "h:mm tt - d MMM YYYY", utc())
+
+proc getName*(profile: XmlNode; selector: string): string =
+  profile.selectText(selector).stripText()
+
+proc getUsername*(profile: XmlNode; selector: string): string =
+  profile.selectText(selector).strip(chars={'@', ' ', '\n'})
+
 proc getBio*(profile: XmlNode; selector: string): string =
   profile.selectText(selector).stripText()
 
 proc getAvatar*(profile: XmlNode; selector: string): string =
   profile.selectAttr(selector, "src").getUserpic()
 
-proc getBanner*(tweet: XmlNode): string =
+proc getBanner*(node: XmlNode): string =
-  let url = tweet.selectAttr("svg > image", "xlink:href")
+  let url = node.selectAttr("svg > image", "xlink:href")
   if url.len > 0:
     result = url.replace("600x200", "1500x500")
   else:
-    result = tweet.selectAttr(".ProfileCard-bg", "style")
+    result = node.selectAttr(".ProfileCard-bg", "style")
     result = result.replace("background-color: ", "")
 
   if result.len == 0:
     result = "#161616"
 
+proc getTimelineBanner*(node: XmlNode): string =
+  let banner = node.select(".ProfileCanopy-headerBg img")
+  let img = banner.attr("src")
+  if img.len > 0:
+    return img
+
+  let style = node.select("style").innerText()
+  var m: RegexMatch
+  if style.find(re"a:active \{\n +color: (#[A-Z0-9]+)", m):
+    return style[m.group(0)[0]]
+
+proc getProfileStats*(profile: var Profile; node: XmlNode) =
+  for s in node.selectAll( ".ProfileNav-stat"):
+    let text = s.attr("title").split(" ")[0]
+    case s.attr("data-nav")
+    of "followers": profile.followers = text
+    of "following": profile.following = text
+    of "favorites": profile.likes = text
+    of "tweets": profile.tweets = text
+
 proc getPopupStats*(profile: var Profile; node: XmlNode) =
   for s in node.selectAll( ".ProfileCardStats-statLink"):
     let text = s.attr("title").split(" ")[0]

src/types.nim

@@ -12,12 +12,15 @@ db("cache.db", "", "", ""):
     Profile* = object
       username*: string
       fullname*: string
+      location*: string
+      website*: string
       bio*: string
       userpic*: string
       banner*: string
       following*: string
       followers*: string
       tweets*: string
+      likes*: string
       verified* {.
           dbType: "STRING",
           parseIt: parseBool(it.s)
@@ -28,6 +31,11 @@ db("cache.db", "", "", ""):
           parseIt: parseBool(it.s)
           formatIt: $it
         .}: bool
+      joinDate* {.
+        dbType: "INTEGER",
+        parseIt: it.i.fromUnix(),
+        formatIt: it.toUnix()
+        .}: Time
       updated* {.
           dbType: "INTEGER",
           parseIt: it.i.fromUnix(),

src/views/profile.nim

@@ -15,7 +15,6 @@ proc renderProfileCard*(profile: Profile): VNode =
     a(class="profile-card-avatar", href=profile.getUserPic().getSigUrl("pic")):
       genImg(profile.getUserpic("_200x200"))
 
-    tdiv(class="profile-card-tabs"):
     tdiv(class="profile-card-tabs-name"):
       linkUser(profile, class="profile-card-fullname")
       linkUser(profile, class="profile-card-username")
@@ -25,11 +24,26 @@ proc renderProfileCard*(profile: Profile): VNode =
         tdiv(class="profile-bio"):
           p: verbatim linkifyText(profile.bio)
 
+      if profile.location.len > 0:
+        tdiv(class="profile-location"):
+          span: text "📍 " & profile.location
+
+      if profile.website.len > 0:
+        tdiv(class="profile-website"):
+          span:
+            text "🔗 "
+            a(href=profile.website): text profile.website
+
+      tdiv(class="profile-joindate"):
+        span(title=getJoinDateFull(profile)):
+          text "📅 " & getJoinDate(profile)
+
       tdiv(class="profile-card-extra-links"):
         ul(class="profile-statlist"):
           renderStat(profile.tweets, "posts", text="Tweets")
           renderStat(profile.followers, "followers")
           renderStat(profile.following, "following")
+          renderStat(profile.likes, "likes")
 
 proc renderPhotoRail(username: string; photoRail: seq[GalleryPhoto]): VNode =
   buildHtml(tdiv(class="photo-rail-card")):