From 7d2fa1567600b62c37747c5f5fd1ff3d04f6e257 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 26 Jun 2019 19:59:28 +0200 Subject: [PATCH] Improve tweet selectors --- src/parser.nim | 14 +++++++------- src/parserutils.nim | 5 +++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/parser.nim b/src/parser.nim index a2c501f..1c12bec 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -83,8 +83,7 @@ proc parseTweet*(node: XmlNode): Tweet = proc parseTweets*(node: XmlNode): Tweets = if node.isNil or node.kind == xnText: return - for n in node.selectAll(".stream-item"): - result.add parseTweet(n) + node.selectAll(".stream-item").map(parseTweet) proc parseConversation*(node: XmlNode): Conversation = result = Conversation( @@ -92,13 +91,14 @@ proc parseConversation*(node: XmlNode): Conversation = before: parseTweets(node.select(".in-reply-to")) ) - let replies = node.select(".replies-to") + let replies = node.select(".replies-to", ".stream-items") if replies.isNil: return - result.after = parseTweets(replies.select(".ThreadedConversation--selfThread")) - - for reply in replies.select(".stream-items"): - result.replies.add parseTweets(reply) + for reply in replies.filterIt(it.kind != xnText): + if "selfThread" in reply.attr("class"): + result.after = parseTweets(reply.select(".stream-items")) + else: + result.replies.add parseTweets(reply) proc parseVideo*(node: JsonNode): Video = let track = node{"track"} diff --git a/src/parserutils.nim b/src/parserutils.nim index 3feb2d2..d1e89de 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -16,6 +16,11 @@ proc select*(node: XmlNode; selector: string): XmlNode = let nodes = node.selectAll(selector) if nodes.len > 0: nodes[0] else: nil +proc select*(node: XmlNode; parent, child: string): XmlNode = + let parentNode = node.select(parent) + if parentNode.isNil(): return + result = parentNode.select(child) + proc getAttr*(node: XmlNode; attr: string; default=""): string = if node.isNil or node.attrs.isNil: return default return node.attrs.getOrDefault(attr)