From 4dac9f0798b885130b01f9a5a5535c729d351467 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 25 Nov 2023 05:31:15 +0000 Subject: [PATCH] Add simple job_details card support --- src/experimental/parser/graphql.nim | 4 ++-- src/experimental/parser/unifiedcard.nim | 11 +++++++++ src/experimental/parser/user.nim | 7 +++++- src/experimental/types/graphuser.nim | 4 ++-- src/experimental/types/timeline.nim | 4 ++-- src/experimental/types/unifiedcard.nim | 32 ++++++++++++++++++++----- src/parser.nim | 4 ++-- src/parserutils.nim | 9 +++---- src/types.nim | 1 + src/utils.nim | 5 ++-- 10 files changed, 60 insertions(+), 21 deletions(-) diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index b9da7c4..0e9a678 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -12,7 +12,7 @@ proc parseGraphUser*(json: string): User = if raw.data.userResult.result.unavailableReason.get("") == "Suspended": return User(suspended: true) - result = toUser raw.data.userResult.result.legacy + result = raw.data.userResult.result.legacy result.id = raw.data.userResult.result.restId result.verified = result.verified or raw.data.userResult.result.isBlueVerified @@ -30,7 +30,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] = of TimelineTimelineItem: let userResult = entry.content.itemContent.userResults.result if userResult.restId.len > 0: - result.content.add toUser userResult.legacy + result.content.add userResult.legacy of TimelineTimelineCursor: if entry.content.cursorType == "Bottom": result.bottom = entry.content.value diff --git a/src/experimental/parser/unifiedcard.nim b/src/experimental/parser/unifiedcard.nim index c9af437..4a50e48 100644 --- a/src/experimental/parser/unifiedcard.nim +++ b/src/experimental/parser/unifiedcard.nim @@ -1,5 +1,6 @@ import std/[options, tables, strutils, strformat, sugar] import jsony +import user import ../types/unifiedcard from ../../types import Card, CardKind, Video from ../../utils import twimg, https @@ -27,6 +28,14 @@ proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card) result.text = data.topicDetail.title result.dest = "Topic" +proc parseJobDetails(data: ComponentData; card: UnifiedCard; result: var Card) = + data.destination.parseDestination(card, result) + + result.kind = jobDetails + result.title = data.title + result.text = data.shortDescriptionText + result.dest = &"@{data.profileUser.username} ยท {data.location}" + proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) = let app = card.appStoreData[data.appId][0] @@ -84,6 +93,8 @@ proc parseUnifiedCard*(json: string): Card = component.parseMedia(card, result) of buttonGroup: discard + of jobDetails: + component.data.parseJobDetails(card, result) of ComponentType.hidden: result.kind = CardKind.hidden of ComponentType.unknown: diff --git a/src/experimental/parser/user.nim b/src/experimental/parser/user.nim index 5962a87..78f596e 100644 --- a/src/experimental/parser/user.nim +++ b/src/experimental/parser/user.nim @@ -68,6 +68,11 @@ proc toUser*(raw: RawUser): User = result.expandUserEntities(raw) +proc parseHook*(s: string; i: var int; v: var User) = + var u: RawUser + parseHook(s, i, u) + v = toUser u + proc parseUser*(json: string; username=""): User = handleErrors: case error.code @@ -75,7 +80,7 @@ proc parseUser*(json: string; username=""): User = of userNotFound: return else: echo "[error - parseUser]: ", error - result = toUser json.fromJson(RawUser) + result = json.fromJson(User) proc parseUsers*(json: string; after=""): Result[User] = result = Result[User](beginning: after.len == 0) diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim index c30eed9..08100f9 100644 --- a/src/experimental/types/graphuser.nim +++ b/src/experimental/types/graphuser.nim @@ -1,5 +1,5 @@ import options -import user +from ../../types import User type GraphUser* = object @@ -9,7 +9,7 @@ type result*: UserResult UserResult = object - legacy*: RawUser + legacy*: User restId*: string isBlueVerified*: bool unavailableReason*: Option[string] diff --git a/src/experimental/types/timeline.nim b/src/experimental/types/timeline.nim index 28239ad..5ce6d9f 100644 --- a/src/experimental/types/timeline.nim +++ b/src/experimental/types/timeline.nim @@ -1,5 +1,5 @@ import std/tables -import user +from ../../types import User type Search* = object @@ -7,7 +7,7 @@ type timeline*: Timeline GlobalObjects = object - users*: Table[string, RawUser] + users*: Table[string, User] Timeline = object instructions*: seq[Instructions] diff --git a/src/experimental/types/unifiedcard.nim b/src/experimental/types/unifiedcard.nim index 6e83cad..e540a64 100644 --- a/src/experimental/types/unifiedcard.nim +++ b/src/experimental/types/unifiedcard.nim @@ -1,7 +1,10 @@ -import options, tables -from ../../types import VideoType, VideoVariant +import std/[options, tables, times] +import jsony +from ../../types import VideoType, VideoVariant, User type + Text* = distinct string + UnifiedCard* = object componentObjects*: Table[string, Component] destinationObjects*: Table[string, Destination] @@ -13,6 +16,7 @@ type media swipeableMedia buttonGroup + jobDetails appStoreDetails twitterListDetails communityDetails @@ -29,12 +33,15 @@ type appId*: string mediaId*: string destination*: string + location*: string title*: Text subtitle*: Text name*: Text memberCount*: int mediaList*: seq[MediaItem] topicDetail*: tuple[title: Text] + profileUser*: User + shortDescriptionText*: string MediaItem* = object id*: string @@ -69,12 +76,9 @@ type title*: Text category*: Text - Text = object - content: string - TypeField = Component | Destination | MediaEntity | AppStoreData -converter fromText*(text: Text): string = text.content +converter fromText*(text: Text): string = string(text) proc renameHook*(v: var TypeField; fieldName: var string) = if fieldName == "type": @@ -86,6 +90,7 @@ proc enumHook*(s: string; v: var ComponentType) = of "media": media of "swipeable_media": swipeableMedia of "button_group": buttonGroup + of "job_details": jobDetails of "app_store_details": appStoreDetails of "twitter_list_details": twitterListDetails of "community_details": communityDetails @@ -106,3 +111,18 @@ proc enumHook*(s: string; v: var MediaType) = of "photo": photo of "model3d": model3d else: echo "ERROR: Unknown enum value (MediaType): ", s; photo + +proc parseHook*(s: string; i: var int; v: var DateTime) = + var str: string + parseHook(s, i, str) + v = parse(str, "yyyy-MM-dd hh:mm:ss") + +proc parseHook*(s: string; i: var int; v: var Text) = + if s[i] == '"': + var str: string + parseHook(s, i, str) + v = Text(str) + else: + var t: tuple[content: string] + parseHook(s, i, t) + v = Text(t.content) diff --git a/src/parser.nim b/src/parser.nim index fcee13f..d7ba613 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -219,8 +219,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = ) ) - result.expandTweetEntities(js) - # fix for pinned threads if result.hasThread and result.threadId == 0: result.threadId = js{"self_thread", "id_str"}.getId @@ -254,6 +252,8 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = else: result.card = some parseCard(jsCard, js{"entities", "urls"}) + result.expandTweetEntities(js) + with jsMedia, js{"extended_entities", "media"}: for m in jsMedia: case m{"type"}.getStr diff --git a/src/parserutils.nim b/src/parserutils.nim index 6b8263f..00ea6f4 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -246,7 +246,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) = .replacef(htRegex, htReplace) proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int]; - replyTo=""; hasQuote=false) = + replyTo=""; hasRedundantLink=false) = let hasCard = tweet.card.isSome var replacements = newSeq[ReplaceSlice]() @@ -257,7 +257,7 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic if urlStr.len == 0 or urlStr notin text: continue - replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote) + replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink) if hasCard and u{"url"}.getStr == get(tweet.card).url: get(tweet.card).url = u{"expanded_url"}.getStr @@ -297,9 +297,10 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = let entities = ? js{"entities"} - hasQuote = js{"is_quote_status"}.getBool textRange = js{"display_text_range"} textSlice = textRange{0}.getInt .. textRange{1}.getInt + hasQuote = js{"is_quote_status"}.getBool + hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails var replyTo = "" if tweet.replyId != 0: @@ -307,7 +308,7 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = replyTo = reply.getStr tweet.reply.add replyTo - tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote) + tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard) proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) = let diff --git a/src/types.nim b/src/types.nim index 9ddf283..bc791b1 100644 --- a/src/types.nim +++ b/src/types.nim @@ -162,6 +162,7 @@ type imageDirectMessage = "image_direct_message" audiospace = "audiospace" newsletterPublication = "newsletter_publication" + jobDetails = "job_details" hidden unknown diff --git a/src/utils.nim b/src/utils.nim index 9002bbf..c96a6dd 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -16,7 +16,8 @@ const "twimg.com", "abs.twimg.com", "pbs.twimg.com", - "video.twimg.com" + "video.twimg.com", + "x.com" ] proc setHmacKey*(key: string) = @@ -57,4 +58,4 @@ proc isTwitterUrl*(uri: Uri): bool = uri.hostname in twitterDomains proc isTwitterUrl*(url: string): bool = - parseUri(url).hostname in twitterDomains + isTwitterUrl(parseUri(url))