From abe21e3ebf1b984adaa42084a1e6f9c3a470c28c Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 21 Jun 2019 02:15:46 +0200 Subject: [PATCH] Add fallback for sensitive profiles --- src/api.nim | 78 ++++++++++++++++++++++++-------------------------- src/parser.nim | 52 +++++++++++++++++++++++---------- 2 files changed, 75 insertions(+), 55 deletions(-) diff --git a/src/api.nim b/src/api.nim index 4aa3851..ba0e277 100644 --- a/src/api.nim +++ b/src/api.nim @@ -8,9 +8,31 @@ const base = parseUri("https://twitter.com/") const agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" const timelineUrl = "i/profiles/show/$1/timeline/tweets?include_available_features=1&include_entities=1&include_new_items_bar=true" -const profileUrl = "i/profiles/popup" +const profilePopupUrl = "i/profiles/popup" +const profileIntentUrl = "intent/user" const tweetUrl = "i/status/" +proc fetchHtml(client: AsyncHttpClient; url: Uri; jsonKey = ""): Future[XmlNode] {.async.} = + var resp = "" + try: + resp = await client.getContent($url) + except: + return nil + + if jsonKey.len > 0: + let json = parseJson(resp)[jsonKey].str + return parseHtml(json) + else: + return parseHtml(resp) + +proc getProfileFallback(username: string; client: AsyncHttpClient): Future[Profile] {.async.} = + let + params = {"screen_name": username} + url = base / profileIntentUrl ? params + html = await client.fetchHtml(url) + + result = parseIntentProfile(html) + proc getProfile*(username: string): Future[Profile] {.async.} = let client = newAsyncHttpClient() defer: client.close() @@ -24,25 +46,19 @@ proc getProfile*(username: string): Future[Profile] {.async.} = "Accept-Language": "en-US,en;q=0.9" }) - let params = { - "screen_name": username, - "wants_hovercard": "true", - "_": $(epochTime().int) - } - - let url = base / profileUrl ? params - var resp = "" - - try: - resp = await client.getContent($url) - except: - return Profile() - let - json = parseJson(resp)["html"].str - html = parseHtml(json) + params = { + "screen_name": username, + "wants_hovercard": "true", + "_": $(epochTime().int) + } + url = base / profilePopupUrl ? params + html = await client.fetchHtml(url, jsonKey="html") - result = parseProfile(html) + if not html.querySelector(".ProfileCard-sensitiveWarningContainer").isNil: + return await getProfileFallback(username, client) + + result = parsePopupProfile(html) proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} = let client = newAsyncHttpClient() @@ -61,18 +77,7 @@ proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} = if after != "": url &= "&max_position=" & after - var resp = "" - try: - resp = await client.getContent($(base / url)) - except: - return - - var json: string = "" - var html: XmlNode - json = parseJson(resp)["items_html"].str - html = parseHtml(json) - - writeFile("epic.html", $html) + let html = await client.fetchHtml(base / url, jsonKey="items_html") result = parseTweets(html) @@ -91,15 +96,8 @@ proc getTweet*(id: string): Future[Conversation] {.async.} = "x-previous-page-name": "profile" }) - let url = base / tweetUrl / id - - var resp: string = "" - try: - resp = await client.getContent($url) - except: - return Conversation() - - var html: XmlNode - html = parseHtml(resp) + let + url = base / tweetUrl / id + html = await client.fetchHtml(url) result = parseConversation(html) diff --git a/src/parser.nim b/src/parser.nim index e08e9dc..49a42cc 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -4,7 +4,7 @@ import nimquery, regex import ./types, ./formatters proc getAttr(node: XmlNode; attr: string; default=""): string = - if node.isNIl or node.attrs.isNil: return default + if node.isNil or node.attrs.isNil: return default return node.attrs.getOrDefault(attr) proc selectAttr(node: XmlNode; selector: string; attr: string; default=""): string = @@ -15,16 +15,21 @@ proc selectText(node: XmlNode; selector: string): string = let res = node.querySelector(selector) result = if res == nil: "" else: res.innerText() -proc parseProfile*(node: XmlNode): Profile = +proc parsePopupProfile*(node: XmlNode): Profile = let profile = node.querySelector(".profile-card") - result.fullname = profile.selectText(".fullname").strip() - result.username = profile.selectText(".username").strip(chars={'@', ' '}) - result.description = profile.selectText(".bio") - result.verified = profile.selectText(".Icon.Icon--verified").len > 0 - result.protected = profile.selectText(".Icon.Icon--protected").len > 0 - result.userpic = profile.selectAttr(".ProfileCard-avatarImage", "src").getUserpic() - result.banner = profile.selectAttr("svg > image", "xlink:href").replace("600x200", "1500x500") - if result.banner == "": + if profile.isNil: return + + result = Profile( + fullname: profile.selectText(".fullname").strip(), + username: profile.selectText(".username").strip(chars={'@', ' '}), + description: profile.selectText(".bio"), + verified: profile.selectText(".Icon.Icon--verified").len > 0, + protected: profile.selectText(".Icon.Icon--protected").len > 0, + userpic: profile.selectAttr(".ProfileCard-avatarImage", "src").getUserpic(), + banner: profile.selectAttr("svg > image", "xlink:href").replace("600x200", "1500x500") + ) + + if result.banner.len == 0: result.banner = profile.selectAttr(".ProfileCard-bg", "style") let stats = profile.querySelectorAll(".ProfileCardStats-statLink") @@ -35,12 +40,29 @@ proc parseProfile*(node: XmlNode): Profile = of "following": result.following = text else: result.tweets = text -proc parseTweetProfile*(tweet: XmlNode): Profile = +proc parseIntentProfile*(profile: XmlNode): Profile = result = Profile( - fullname: tweet.getAttr("data-name"), - username: tweet.getAttr("data-screen-name"), - userpic: tweet.selectAttr(".avatar", "src").getUserpic(), - verified: tweet.selectText(".Icon.Icon--verified").len > 0 + fullname: profile.selectText("a.fn.url.alternate-context").strip(), + username: profile.selectText(".nickname").strip(chars={'@', ' '}), + userpic: profile.querySelector(".profile.summary").selectAttr("img.photo", "src").getUserPic(), + description: profile.selectText("p.note").strip(), + verified: not profile.querySelector("li.verified").isNil, + protected: not profile.querySelector("li.protected").isNil, + banner: "background-color: #161616", + tweets: "?" + ) + + for stat in profile.querySelectorAll("dd.count > a"): + case stat.getAttr("href").split("/")[^1] + of "followers": result.followers = stat.innerText() + of "following": result.following = stat.innerText() + +proc parseTweetProfile*(profile: XmlNode): Profile = + result = Profile( + fullname: profile.getAttr("data-name"), + username: profile.getAttr("data-screen-name"), + userpic: profile.selectAttr(".avatar", "src").getUserpic(), + verified: profile.selectText(".Icon.Icon--verified").len > 0 ) proc parseTweet*(tweet: XmlNode): Tweet =