blankie
/
nitter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Revamp profile api to display more metadata

This commit is contained in:
Zed 2019-08-11 21:26:55 +02:00
parent 3f1d9777b6
commit 7171486f03
9 changed files with 192 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
public/style.css
View File

@ -445,14 +445,6 @@ video {
display: flex;
}
.profile-card-tabs {
display: flex;
justify-content: space-between;
align-items: center;
flex: 1 1 auto;
max-width: 100%;
}
.profile-card-tabs-name {
max-width: 100%;
}
@ -496,20 +488,26 @@ video {
.profile-card-extra {
display: contents;
flex: 100%;
margin-top: 4px;
margin-top: 6px;
}
.profile-bio {
overflow: hidden;
overflow-wrap: break-word;
width: 100%;
margin: 10px -6px 0px 0px;
margin: 4px -6px 6px 0px;
}
.profile-bio p {
margin: 0;
}
.profile-location, .profile-website, .profile-joindate {
color: #f8f8f2cf;
margin: 2px 0px;
width: 100%;
}
.profile-description {
font-size: 14px;
font-weight: 400;
@ -735,6 +733,7 @@ video {
margin: 0;
padding: 0;
width: 100%;
justify-content: space-between;
}
.profile-statlist > li {
@ -742,19 +741,6 @@ video {
text-align: center;
}
.profile-statlist .posts {
flex: 0.4 1 0;
}
.profile-statlist .followers {
flex: 1 1 0;
padding: 0 3px;
}
.profile-statlist .following {
flex: 0.5 1 0;
}
.profile-stat-header {
font-weight: bold;
}

44
src/api.nim
View File

@ -6,7 +6,7 @@ import types, parser, parserutils, formatters, search
const
lang = "en-US,en;q=0.9"
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
jsonAccept = "application/json, text/javascript, */*; q=0.01"
base = parseUri("https://twitter.com/")
@ -38,7 +38,7 @@ macro genMediaGet(media: untyped; token=false) =
single = ident("get" & mediaName)
quote do:
proc `multi`(thread: Thread; agent: string; token="") {.async.} =
proc `multi`(thread: Thread | Timeline; agent: string; token="") {.async.} =
if thread == nil: return
var `media` = thread.tweets.filterIt(it.`media`.isSome)
when `token`:
@ -165,7 +165,7 @@ proc getPoll*(tweet: Tweet; agent: string) {.async.} =
if tweet.poll.isNone(): return
let headers = newHttpHeaders({
"Accept": cardAccept,
"Accept": accept,
"Referer": $(base / getLink(tweet)),
"User-Agent": agent,
"Authority": "twitter.com",
@ -182,7 +182,7 @@ proc getCard*(tweet: Tweet; agent: string) {.async.} =
if tweet.card.isNone(): return
let headers = newHttpHeaders({
"Accept": cardAccept,
"Accept": accept,
"Referer": $(base / getLink(tweet)),
"User-Agent": agent,
"Authority": "twitter.com",
@ -350,3 +350,39 @@ proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.
let json = await fetchJson(base / timelineSearchUrl ? params, headers)
result = await finishTimeline(json, some(query), after, agent)
proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
let headers = newHttpHeaders({
"authority": "twitter.com",
"accept": accept,
"referer": "https://twitter.com/" & username,
"accept-language": lang
})
var url = base / username
if after.len > 0:
url = url ? {"max_position": after}
let
html = await fetchHtml(url, headers)
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
profile = parseTimelineProfile(html)
vidsFut = getVideos(timeline, agent)
pollFut = getPolls(timeline, agent)
cardFut = getCards(timeline, agent)
await all(vidsFut, pollFut, cardFut)
result = (profile, timeline)
proc getProfileFull*(username: string): Future[Profile] {.async.} =
let headers = newHttpHeaders({
"authority": "twitter.com",
"accept": accept,
"referer": "https://twitter.com/" & username,
"accept-language": lang
})
let html = await fetchHtml(base / username, headers)
if html == nil: return
result = parseTimelineProfile(html)

37
src/cache.nim
View File

@ -9,23 +9,36 @@ withDb:
var profileCacheTime = initDuration(minutes=10)
proc outdated(profile: Profile): bool =
proc isOutdated*(profile: Profile): bool =
getTime() - profile.updated > profileCacheTime
proc cache*(profile: var Profile) =
withDb:
try:
let p = Profile.getOne("lower(username) = ?", toLower(profile.username))
profile.id = p.id
profile.update()
except KeyError:
if profile.username.len > 0:
profile.insert()
proc hasCachedProfile*(username: string): Option[Profile] =
withDb:
try:
let p = Profile.getOne("lower(username) = ?", toLower(username))
doAssert not p.isOutdated
result = some(p)
except AssertionError, KeyError:
result = none(Profile)
proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} =
withDb:
try:
result.getOne("username = ?", username)
doAssert not result.outdated()
except AssertionError:
var profile = await getProfile(username, agent)
profile.id = result.id
result = profile
result.update()
except KeyError:
result = await getProfile(username, agent)
if result.username.len > 0:
result.insert()
result.getOne("lower(username) = ?", toLower(username))
doAssert not result.isOutdated
except AssertionError, KeyError:
result = await getProfileFull(username)
cache(result)
proc setProfileCacheTime*(minutes: int) =
profileCacheTime = initDuration(minutes=minutes)

8
src/formatters.nim
View File

@ -62,7 +62,7 @@ proc stripTwitterUrls*(text: string): string =
result = result.replace(ellipsisRegex, "")
proc getUserpic*(userpic: string; style=""): string =
let pic = userpic.replace(re"_(normal|bigger|mini|200x200)(\.[A-z]+)$", "$2")
let pic = userpic.replace(re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$", "$2")
pic.replace(re"(\.[A-z]+)$", style & "$1")
proc getUserpic*(profile: Profile; style=""): string =
@ -77,6 +77,12 @@ proc pageTitle*(profile: Profile): string =
proc pageDesc*(profile: Profile): string =
"The latest tweets from " & profile.fullname
proc getJoinDate*(profile: Profile): string =
profile.joinDate.format("'Joined' MMMM YYYY")
proc getJoinDateFull*(profile: Profile): string =
profile.joinDate.format("h:mm tt - d MMM YYYY")
proc getTime*(tweet: Tweet): string =
tweet.time.format("d/M/yyyy', ' HH:mm:ss")

27
src/nitter.nim
View File

@ -10,20 +10,31 @@ const configPath {.strdefine.} = "./nitter.conf"
let cfg = getConfig(configPath)
proc showSingleTimeline(name, after, agent: string; query: Option[Query]): Future[string] {.async.} =
let profileFut = getCachedProfile(name, agent)
let railFut = getPhotoRail(name, agent)
var timelineFut: Future[Timeline]
if query.isNone:
timelineFut = getTimeline(name, after, agent)
else:
timelineFut = getTimelineSearch(get(query), after, agent)
var timeline: Timeline
var profile: Profile
var cachedProfile = hasCachedProfile(name)
if cachedProfile.isSome:
profile = get(cachedProfile)
if query.isNone:
if cachedProfile.isSome:
timeline = await getTimeline(name, after, agent)
else:
(profile, timeline) = await getProfileAndTimeline(name, agent, after)
cache(profile)
else:
var timelineFut = getTimelineSearch(get(query), after, agent)
if cachedProfile.isNone:
profile = await getCachedProfile(name, agent)
timeline = await timelineFut
let profile = await profileFut
if profile.username.len == 0:
return ""
let profileHtml = renderProfile(profile, await timelineFut, await railFut)
let profileHtml = renderProfile(profile, timeline, await railFut)
return renderMain(profileHtml, title=cfg.title, titleText=pageTitle(profile), desc=pageDesc(profile))
proc showMultiTimeline(names: seq[string]; after, agent: string; query: Option[Query]): Future[string] {.async.} =

30
src/parser.nim
View File

@ -2,6 +2,26 @@ import xmltree, sequtils, strutils, json
import types, parserutils, formatters
proc parseTimelineProfile*(node: XmlNode): Profile =
let profile = node.select(".ProfileHeaderCard")
if profile == nil: return
let pre = ".ProfileHeaderCard-"
result = Profile(
fullname: profile.getName(pre & "nameLink"),
username: profile.getUsername(pre & "screenname"),
joinDate: profile.getDate(pre & "joinDateText"),
location: profile.selectText(pre & "locationText").stripText(),
website: profile.selectText(pre & "url").stripText(),
bio: profile.getBio(pre & "bio"),
userpic: node.getAvatar(".profile-picture img"),
verified: isVerified(profile),
protected: isProtected(profile),
banner: getTimelineBanner(node)
)
result.getProfileStats(node.select(".ProfileNav-list"))
proc parsePopupProfile*(node: XmlNode): Profile =
let profile = node.select(".profile-card")
if profile == nil: return
@ -125,6 +145,16 @@ proc parseConversation*(node: XmlNode): Conversation =
else:
result.replies.add parseThread(thread)
proc parseTimeline*(node: XmlNode; after: string): Timeline =
if node == nil: return
result = Timeline(
tweets: parseThread(node.select(".stream > .stream-items")).tweets,
minId: node.attr("data-min-position"),
maxId: node.attr("data-max-position"),
hasMore: node.select(".has-more-items") != nil,
beginning: after.len == 0
)
proc parseVideo*(node: JsonNode; tweetId: string): Video =
let
track = node{"track"}

45
src/parserutils.nim
View File

@ -32,6 +32,8 @@ proc getHeader(profile: XmlNode): XmlNode =
result = profile.select(".stream-item-header")
if result == nil:
result = profile.select(".ProfileCard-userFields")
if result == nil:
result = profile
proc isVerified*(profile: XmlNode): bool =
getHeader(profile).select(".Icon.Icon--verified") != nil
@ -39,12 +41,6 @@ proc isVerified*(profile: XmlNode): bool =
proc isProtected*(profile: XmlNode): bool =
getHeader(profile).select(".Icon.Icon--protected") != nil
proc getName*(profile: XmlNode; selector: string): string =
profile.selectText(selector).stripText()
proc getUsername*(profile: XmlNode; selector: string): string =
profile.selectText(selector).strip(chars={'@', ' '})
proc emojify*(node: XmlNode) =
for i in node.selectAll(".Emoji"):
i.add newText(i.attr("alt"))
@ -79,23 +75,54 @@ proc getTimestamp*(tweet: XmlNode): Time =
proc getShortTime*(tweet: XmlNode): string =
getTime(tweet).innerText()
proc getDate*(node: XmlNode; selector: string): Time =
let date = node.select(selector)
if date == nil: return
parseTime(date.attr("title"), "h:mm tt - d MMM YYYY", utc())
proc getName*(profile: XmlNode; selector: string): string =
profile.selectText(selector).stripText()
proc getUsername*(profile: XmlNode; selector: string): string =
profile.selectText(selector).strip(chars={'@', ' ', '\n'})
proc getBio*(profile: XmlNode; selector: string): string =
profile.selectText(selector).stripText()
proc getAvatar*(profile: XmlNode; selector: string): string =
profile.selectAttr(selector, "src").getUserpic()
proc getBanner*(tweet: XmlNode): string =
let url = tweet.selectAttr("svg > image", "xlink:href")
proc getBanner*(node: XmlNode): string =
let url = node.selectAttr("svg > image", "xlink:href")
if url.len > 0:
result = url.replace("600x200", "1500x500")
else:
result = tweet.selectAttr(".ProfileCard-bg", "style")
result = node.selectAttr(".ProfileCard-bg", "style")
result = result.replace("background-color: ", "")
if result.len == 0:
result = "#161616"
proc getTimelineBanner*(node: XmlNode): string =
let banner = node.select(".ProfileCanopy-headerBg img")
let img = banner.attr("src")
if img.len > 0:
return img
let style = node.select("style").innerText()
var m: RegexMatch
if style.find(re"a:active \{\n +color: (#[A-Z0-9]+)", m):
return style[m.group(0)[0]]
proc getProfileStats*(profile: var Profile; node: XmlNode) =
for s in node.selectAll( ".ProfileNav-stat"):
let text = s.attr("title").split(" ")[0]
case s.attr("data-nav")
of "followers": profile.followers = text
of "following": profile.following = text
of "favorites": profile.likes = text
of "tweets": profile.tweets = text
proc getPopupStats*(profile: var Profile; node: XmlNode) =
for s in node.selectAll( ".ProfileCardStats-statLink"):
let text = s.attr("title").split(" ")[0]

8
src/types.nim
View File

@ -12,12 +12,15 @@ db("cache.db", "", "", ""):
Profile* = object
username*: string
fullname*: string
location*: string
website*: string
bio*: string
userpic*: string
banner*: string
following*: string
followers*: string
tweets*: string
likes*: string
verified* {.
dbType: "STRING",
parseIt: parseBool(it.s)
@ -28,6 +31,11 @@ db("cache.db", "", "", ""):
parseIt: parseBool(it.s)
formatIt: $it
.}: bool
joinDate* {.
dbType: "INTEGER",
parseIt: it.i.fromUnix(),
formatIt: it.toUnix()
.}: Time
updated* {.
dbType: "INTEGER",
parseIt: it.i.fromUnix(),

16
src/views/profile.nim
View File

@ -15,7 +15,6 @@ proc renderProfileCard*(profile: Profile): VNode =
a(class="profile-card-avatar", href=profile.getUserPic().getSigUrl("pic")):
genImg(profile.getUserpic("_200x200"))
tdiv(class="profile-card-tabs"):
tdiv(class="profile-card-tabs-name"):
linkUser(profile, class="profile-card-fullname")
linkUser(profile, class="profile-card-username")
@ -25,11 +24,26 @@ proc renderProfileCard*(profile: Profile): VNode =
tdiv(class="profile-bio"):
p: verbatim linkifyText(profile.bio)
if profile.location.len > 0:
tdiv(class="profile-location"):
span: text "📍 " & profile.location
if profile.website.len > 0:
tdiv(class="profile-website"):
span:
text "🔗 "
a(href=profile.website): text profile.website
tdiv(class="profile-joindate"):
span(title=getJoinDateFull(profile)):
text "📅 " & getJoinDate(profile)
tdiv(class="profile-card-extra-links"):
ul(class="profile-statlist"):
renderStat(profile.tweets, "posts", text="Tweets")
renderStat(profile.followers, "followers")
renderStat(profile.following, "following")
renderStat(profile.likes, "likes")
proc renderPhotoRail(username: string; photoRail: seq[GalleryPhoto]): VNode =
buildHtml(tdiv(class="photo-rail-card")):