diff --git a/src/api.nim b/src/api.nim index 4f36dd8..cbd481f 100644 --- a/src/api.nim +++ b/src/api.nim @@ -163,6 +163,37 @@ proc getConversationPolls*(convo: Conversation) {.async.} = futs.add convo.replies.map(getPolls) await all(futs) +proc getCard*(tweet: Tweet) {.async.} = + if tweet.card.isNone(): return + + let headers = newHttpHeaders({ + "Accept": cardAccept, + "Referer": $(base / getLink(tweet)), + "User-Agent": agent, + "Authority": "twitter.com", + "Accept-Language": lang, + }) + + let url = base / get(tweet.card).query + let html = await fetchHtml(url, headers) + if html == nil: return + + parseCard(get(tweet.card), html) + # echo tweet.card.get() + +proc getCards*(thread: Thread) {.async.} = + if thread == nil: return + var cards = thread.tweets.filterIt(it.card.isSome) + await all(cards.map(getCard)) + +proc getConversationCards*(convo: Conversation) {.async.} = + var futs: seq[Future[void]] + futs.add getCard(convo.tweet) + futs.add getCards(convo.before) + futs.add getCards(convo.after) + futs.add convo.replies.map(getCards) + await all(futs) + proc getPhotoRail*(username: string): Future[seq[GalleryPhoto]] {.async.} = let headers = newHttpHeaders({ "Accept": jsonAccept, @@ -234,9 +265,12 @@ proc getTweet*(username, id: string): Future[Conversation] {.async.} = result = parseConversation(html) - let vidsFut = getConversationVideos(result) - let pollFut = getConversationPolls(result) - await all(vidsFut, pollFut) + let + vidsFut = getConversationVideos(result) + pollFut = getConversationPolls(result) + cardFut = getConversationCards(result) + + await all(vidsFut, pollFut, cardFut) proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future[Timeline] {.async.} = if json == nil: return Timeline() @@ -257,8 +291,9 @@ proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future thread = parseThread(html) vidsFut = getVideos(thread) pollFut = getPolls(thread) + cardFut = getCards(thread) - await all(vidsFut, pollFut) + await all(vidsFut, pollFut, cardFut) result.tweets = thread.tweets proc getTimeline*(username, after: string): Future[Timeline] {.async.} = diff --git a/src/formatters.nim b/src/formatters.nim index 56b366f..a21f4c1 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -81,3 +81,7 @@ proc getTime*(tweet: Tweet): string = proc getLink*(tweet: Tweet | Quote): string = &"/{tweet.profile.username}/status/{tweet.id}" + +proc getUrls*(text: string): seq[string] = + # temporary + text.findAll(urlRegex).mapIt(text[it.group(0)[0]]) diff --git a/src/parser.nim b/src/parser.nim index 5deb817..a1289ae 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -178,3 +178,32 @@ proc parsePhotoRail*(node: XmlNode): seq[GalleryPhoto] = tweetId: img.attr("data-tweet-id"), color: img.attr("background-color").replace("style: ", "") ) + +proc parseCard*(card: var Card; node: XmlNode) = + let cardKind = node.select("head > meta[name*=card_name]").attr("content") + + if "summary_large_image" in cardKind: + card.kind = summaryLarge + elif "summary" in cardKind: + card.kind = summary + elif "live_event" in cardKind: + card.kind = liveEvent + elif "player" in cardKind: + card.kind = player + elif "promo_website" in cardKind: + card.kind = promoWebsite + + card.title = node.selectText("h2.TwitterCard-title") + card.text = node.selectText("p.tcu-resetMargin") + card.dest = node.selectText("span.SummaryCard-destination") + + let image = node.select(".tcu-imageWrapper > img") + if image != nil: + # workaround for issue 11713 + card.image = image.attr("data-src").replace("gname", "g&name") + else: + echo card.id + + if card.kind == liveEvent: + card.text = card.title + card.title = node.selectText(".TwitterCard-attribution--category") diff --git a/src/parserutils.nim b/src/parserutils.nim index 35a086c..d5b1c4e 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -169,8 +169,26 @@ proc getQuoteMedia*(quote: var Quote; node: XmlNode) = proc getTweetCards*(tweet: Tweet; node: XmlNode) = if node.attr("data-has-cards") == "false": return - if "poll" in node.attr("data-card2-type"): + let cardType = node.attr("data-card2-type") + + if "poll" in cardType: tweet.poll = some(Poll()) + return + + let cardDiv = node.select(".card2 > div") + if cardDiv == nil: return + + var card = Card( + id: tweet.id, + query: cardDiv.attr("data-src") + ) + + # temporary solution + let text = node.selectText(".tweet-text") + let urls = getUrls(text) + card.url = urls[0] + + tweet.card = some(card) proc getMoreReplies*(node: XmlNode): int = let text = node.innerText().strip() diff --git a/src/search.nim b/src/search.nim index 216fbe3..cc53eb2 100644 --- a/src/search.nim +++ b/src/search.nim @@ -21,7 +21,7 @@ const proc initQuery*(filters, includes, excludes, separator: string; name=""): Query = var sep = separator.strip().toUpper() Query( - queryType: custom, + kind: custom, filters: filters.split(",").filterIt(it in validFilters), includes: includes.split(",").filterIt(it in validFilters), excludes: excludes.split(",").filterIt(it in validFilters), @@ -31,7 +31,7 @@ proc initQuery*(filters, includes, excludes, separator: string; name=""): Query proc getMediaQuery*(name: string): Query = Query( - queryType: media, + kind: media, filters: @["twimg", "native_video"], fromUser: name, sep: "OR" @@ -39,7 +39,7 @@ proc getMediaQuery*(name: string): Query = proc getReplyQuery*(name: string): Query = Query( - queryType: replies, + kind: replies, includes: @["nativeretweets"], fromUser: name ) @@ -61,8 +61,8 @@ proc genQueryParam*(query: Query): string = return strip(param & filters.join(&" {query.sep} ")) proc genQueryUrl*(query: Query): string = - result = &"/{query.queryType}?" - if query.queryType != custom: return + result = &"/{query.kind}?" + if query.kind != custom: return var params: seq[string] if query.filters.len > 0: diff --git a/src/types.nim b/src/types.nim index 4aac3b5..1f6f573 100644 --- a/src/types.nim +++ b/src/types.nim @@ -31,11 +31,11 @@ db("cache.db", "", "", ""): .}: Time type - QueryType* = enum + QueryKind* = enum replies, media, custom = "search" Query* = object - queryType*: QueryType + kind*: QueryKind filters*: seq[string] includes*: seq[string] excludes*: seq[string] @@ -70,6 +70,19 @@ type status*: string leader*: int + CardKind* = enum + summary, summaryLarge, liveEvent, player, promoWebsite + + Card* = object + kind*: CardKind + id*: string + query*: string + url*: string + title*: string + dest*: string + text*: string + image*: string + Quote* = object id*: string profile*: Profile @@ -104,6 +117,7 @@ type stats*: TweetStats retweet*: Option[Retweet] quote*: Option[Quote] + card*: Option[Card] gif*: Option[Gif] video*: Option[Video] photos*: seq[string] diff --git a/src/views/timeline.nim b/src/views/timeline.nim index cb1d32e..bacd326 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -14,7 +14,7 @@ proc getTabClass(timeline: Timeline; tab: string): string = if timeline.query.isNone: if tab == "tweets": classes.add "active" - elif $timeline.query.get().queryType == tab: + elif $timeline.query.get().kind == tab: classes.add "active" return classes.join(" ")