nitter/src/parser.nim

107 lines
3.9 KiB
Nim
Raw Normal View History

2019-06-20 14:16:20 +00:00
import xmltree, sequtils, strtabs, strutils, strformat, json, times
import nimquery, regex
import ./types, ./formatters
proc getAttr(node: XmlNode; attr: string; default=""): string =
if node.isNIl or node.attrs.isNil: return default
return node.attrs.getOrDefault(attr)
proc selectAttr(node: XmlNode; selector: string; attr: string; default=""): string =
let res = node.querySelector(selector)
2019-06-20 15:20:32 +00:00
if res == nil: "" else: res.getAttr(attr, default)
2019-06-20 14:16:20 +00:00
proc selectText(node: XmlNode; selector: string): string =
let res = node.querySelector(selector)
result = if res == nil: "" else: res.innerText()
proc parseProfile*(node: XmlNode): Profile =
let profile = node.querySelector(".profile-card")
2019-06-20 15:57:31 +00:00
result.fullname = profile.selectText(".fullname").strip()
2019-06-20 14:16:20 +00:00
result.username = profile.selectText(".username").strip(chars={'@', ' '})
result.description = profile.selectText(".bio")
2019-06-20 15:57:31 +00:00
result.verified = profile.selectText(".Icon.Icon--verified").len > 0
2019-06-20 14:16:20 +00:00
result.protected = profile.selectText(".Icon.Icon--protected").len > 0
result.userpic = profile.selectAttr(".ProfileCard-avatarImage", "src").getUserpic()
result.banner = profile.selectAttr("svg > image", "xlink:href").replace("600x200", "1500x500")
if result.banner == "":
result.banner = profile.selectAttr(".ProfileCard-bg", "style")
let stats = profile.querySelectorAll(".ProfileCardStats-statLink")
for s in stats:
let text = s.getAttr("title").split(" ")[0]
case s.getAttr("href").split("/")[^1]
of "followers": result.followers = text
of "following": result.following = text
else: result.tweets = text
proc parseTweetProfile*(tweet: XmlNode): Profile =
result = Profile(
fullname: tweet.getAttr("data-name"),
username: tweet.getAttr("data-screen-name"),
userpic: tweet.selectAttr(".avatar", "src").getUserpic(),
verified: tweet.selectText(".Icon.Icon--verified").len > 0
)
proc parseTweet*(tweet: XmlNode): Tweet =
result.id = tweet.getAttr("data-item-id")
result.link = tweet.getAttr("data-permalink-path")
result.text = tweet.selectText(".tweet-text").stripTwitterUrls()
result.pinned = "pinned" in tweet.getAttr("class")
result.profile = parseTweetProfile(tweet)
let time = tweet.querySelector(".js-short-timestamp")
result.time = fromUnix(parseInt(time.getAttr("data-time", "0")))
result.shortTime = time.innerText()
result.replies = "0"
result.likes = "0"
result.retweets = "0"
for action in tweet.querySelectorAll(".ProfileTweet-actionCountForAria"):
let
text = action.innerText.split()
num = text[0]
2019-06-20 15:20:32 +00:00
case text[1]
2019-06-20 14:16:20 +00:00
of "replies": result.replies = num
of "likes": result.likes = num
of "retweets": result.retweets = num
else: discard
2019-06-20 18:04:18 +00:00
let by = tweet.selectText(".js-retweet-text > a > b")
if by.len > 0:
result.retweetBy = some(by)
2019-06-20 14:16:20 +00:00
for photo in tweet.querySelectorAll(".AdaptiveMedia-photoContainer"):
result.photos.add photo.attrs["data-image-url"]
2019-06-20 15:20:32 +00:00
let player = tweet.selectAttr(".PlayableMedia-player", "style")
if player.len > 0:
let thumb = player.replace(re".+:url\('([^']+)'\)", "$1")
if "tweet_video" in thumb:
2019-06-20 20:46:15 +00:00
result.gif = some(thumb.replace(re".+thumb/([^\.']+)\.jpg.*", "$1"))
2019-06-20 15:20:32 +00:00
else:
2019-06-20 18:04:18 +00:00
result.videoThumb = some(thumb)
2019-06-20 14:16:20 +00:00
proc parseTweets*(node: XmlNode): Tweets =
if node.isNil: return
node.querySelectorAll(".tweet").map(parseTweet)
template selectTweets*(node: XmlNode; class: string): untyped =
parseTweets(node.querySelector(class))
proc parseConversation*(node: XmlNode): Conversation =
result.tweet = parseTweet(node.querySelector(".permalink-tweet-container > .tweet"))
result.before = node.selectTweets(".in-reply-to")
let replies = node.querySelector(".replies-to")
if replies.isNil: return
result.after = replies.selectTweets(".ThreadedConversation--selfThread")
for reply in replies.querySelectorAll("li > .stream-items"):
let thread = parseTweets(reply)
if not thread.anyIt(it in result.after):
result.replies.add thread