From 111927a21cfdebbe3b67d81f3336ae7d342b4f8b Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 13 Jan 2022 00:36:30 +0100 Subject: [PATCH] Add experimental support for unified_card Closes #345 --- nitter.nimble | 1 + src/experimental/parser/unifiedcard.nim | 91 +++++++++++++++++++++++++ src/experimental/types/unifiedcard.nim | 79 +++++++++++++++++++++ src/formatters.nim | 2 +- src/parser.nim | 22 +++--- src/types.nim | 5 +- src/views/tweet.nim | 2 +- 7 files changed, 185 insertions(+), 17 deletions(-) create mode 100644 src/experimental/parser/unifiedcard.nim create mode 100644 src/experimental/types/unifiedcard.nim diff --git a/nitter.nimble b/nitter.nimble index ce9d783..a2c0615 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -22,6 +22,7 @@ requires "redpool#f880f49" requires "https://github.com/zedeus/redis#d0a0e6f" requires "zippy#0.7.3" requires "flatty#0.2.3" +requires "jsony#1.1.3" # Tasks diff --git a/src/experimental/parser/unifiedcard.nim b/src/experimental/parser/unifiedcard.nim new file mode 100644 index 0000000..337c3b9 --- /dev/null +++ b/src/experimental/parser/unifiedcard.nim @@ -0,0 +1,91 @@ +import std/[options, tables, strutils, strformat, sugar] +import jsony +import ../types/unifiedcard +from ../../types import Card, CardKind, Video +from ../../utils import twimg, https + +proc getImageUrl(entity: MediaEntity): string = + entity.mediaUrlHttps.dup(removePrefix(twimg), removePrefix(https)) + +proc parseDestination(id: string; card: UnifiedCard; result: var Card) = + let destination = card.destinationObjects[id].data + result.dest = destination.urlData.vanity + result.url = destination.urlData.url + +proc parseDetails(data: ComponentData; card: UnifiedCard; result: var Card) = + data.destination.parseDestination(card, result) + + result.text = data.title + if result.text.len == 0: + result.text = data.name + +proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card) = + data.destination.parseDestination(card, result) + + result.kind = summary + result.image = card.mediaEntities[data.mediaId].getImageUrl + result.text = data.topicDetail.title + result.dest = "Topic" + +proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) = + let app = card.appStoreData[data.appId][0] + + case app.kind + of androidApp: + result.url = "http://play.google.com/store/apps/details?id=" & app.id + of iPhoneApp, iPadApp: + result.url = "https://itunes.apple.com/app/id" & app.id + + result.text = app.title + result.dest = app.category + +proc parseListDetails(data: ComponentData; result: var Card) = + result.dest = &"List · {data.memberCount} Members" + +proc parseCommunityDetails(data: ComponentData; result: var Card) = + result.dest = &"Community · {data.memberCount} Members" + +proc parseMedia(component: Component; card: UnifiedCard; result: var Card) = + let mediaId = + if component.kind == swipeableMedia: + component.data.mediaList[0].id + else: + component.data.id + + let rMedia = card.mediaEntities[mediaId] + case rMedia.kind: + of photo: + result.kind = summaryLarge + result.image = rMedia.getImageUrl + of video: + let videoInfo = rMedia.videoInfo.get + result.kind = promoVideo + result.video = some Video( + available: true, + thumb: rMedia.getImageUrl, + durationMs: videoInfo.durationMillis, + variants: videoInfo.variants + ) + +proc parseUnifiedCard*(json: string): Card = + let card = json.fromJson(UnifiedCard) + + for component in card.componentObjects.values: + case component.kind + of details, communityDetails, twitterListDetails: + component.data.parseDetails(card, result) + of appStoreDetails: + component.data.parseAppDetails(card, result) + of mediaWithDetailsHorizontal: + component.data.parseMediaDetails(card, result) + of media, swipeableMedia: + component.parseMedia(card, result) + of buttonGroup: + discard + + case component.kind + of twitterListDetails: + component.data.parseListDetails(result) + of communityDetails: + component.data.parseCommunityDetails(result) + else: discard diff --git a/src/experimental/types/unifiedcard.nim b/src/experimental/types/unifiedcard.nim new file mode 100644 index 0000000..16500df --- /dev/null +++ b/src/experimental/types/unifiedcard.nim @@ -0,0 +1,79 @@ +import options, tables +from ../../types import VideoType, VideoVariant + +type + UnifiedCard* = object + componentObjects*: Table[string, Component] + destinationObjects*: Table[string, Destination] + mediaEntities*: Table[string, MediaEntity] + appStoreData*: Table[string, seq[AppStoreData]] + + ComponentType* = enum + details + media + swipeableMedia + buttonGroup + appStoreDetails + twitterListDetails + communityDetails + mediaWithDetailsHorizontal + + Component* = object + kind*: ComponentType + data*: ComponentData + + ComponentData* = object + id*: string + appId*: string + mediaId*: string + destination*: string + title*: Text + subtitle*: Text + name*: Text + memberCount*: int + mediaList*: seq[MediaItem] + topicDetail*: tuple[title: Text] + + MediaItem* = object + id*: string + destination*: string + + Destination* = object + kind*: string + data*: tuple[urlData: UrlData] + + UrlData* = object + url*: string + vanity*: string + + MediaType* = enum + photo, video + + MediaEntity* = object + kind*: MediaType + mediaUrlHttps*: string + videoInfo*: Option[VideoInfo] + + VideoInfo* = object + durationMillis*: int + variants*: seq[VideoVariant] + + AppType* = enum + androidApp, iPhoneApp, iPadApp + + AppStoreData* = object + kind*: AppType + id*: string + title*: Text + category*: Text + + Text = object + content: string + + HasTypeField = Component | Destination | MediaEntity | AppStoreData + +converter fromText*(text: Text): string = text.content + +proc renameHook*(v: var HasTypeField; fieldName: var string) = + if fieldName == "type": + fieldName = "kind" diff --git a/src/formatters.nim b/src/formatters.nim index 31dfb76..b251ccf 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -61,7 +61,7 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string = result = result.replace(tco, https & prefs.replaceTwitter & "/t.co") result = result.replace(cards, prefs.replaceTwitter & "/cards") result = result.replace(twRegex, prefs.replaceTwitter) - result = result.replace(twLinkRegex, a( + result = result.replacef(twLinkRegex, a( prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1")) if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result): diff --git a/src/parser.nim b/src/parser.nim index 3b16786..6aa6a7a 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -1,8 +1,8 @@ # SPDX-License-Identifier: AGPL-3.0-only import strutils, options, tables, times, math -import packedjson -import packedjson / deserialiser +import packedjson, packedjson/deserialiser import types, parserutils, utils +import experimental/parser/unifiedcard proc parseProfile(js: JsonNode; id=""): Profile = if js.isNull: return @@ -102,7 +102,6 @@ proc parseGif(js: JsonNode): Gif = proc parseVideo(js: JsonNode): Video = result = Video( - videoId: js{"id_str"}.getStr, thumb: js{"media_url_https"}.getImageStr, views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr, available: js{"ext_media_availability", "status"}.getStr == "available", @@ -119,7 +118,7 @@ proc parseVideo(js: JsonNode): Video = for v in js{"video_info", "variants"}: result.variants.add VideoVariant( - videoType: parseEnum[VideoType](v{"content_type"}.getStr("summary")), + contentType: parseEnum[VideoType](v{"content_type"}.getStr("summary")), bitrate: v{"bitrate"}.getInt, url: v{"url"}.getStr ) @@ -129,19 +128,17 @@ proc parsePromoVideo(js: JsonNode): Video = thumb: js{"player_image_large"}.getImageVal, available: true, durationMs: js{"content_duration_seconds"}.getStrVal("0").parseInt * 1000, - playbackType: vmap, - videoId: js{"player_content_id"}.getStrVal(js{"card_id"}.getStrVal( - js{"amplify_content_id"}.getStrVal())), + playbackType: vmap ) var variant = VideoVariant( - videoType: vmap, + contentType: vmap, url: js{"player_hls_url"}.getStrVal(js{"player_stream_url"}.getStrVal( js{"amplify_url_vmap"}.getStrVal())) ) if "m3u8" in variant.url: - variant.videoType = m3u8 + variant.contentType = m3u8 result.playbackType = m3u8 result.variants.add variant @@ -154,7 +151,7 @@ proc parseBroadcast(js: JsonNode): Card = title: js{"broadcaster_display_name"}.getStrVal, text: js{"broadcast_title"}.getStrVal, image: image, - video: some Video(videoId: js{"broadcast_media_id"}.getStrVal, thumb: image) + video: some Video(thumb: image) ) proc parseCard(js: JsonNode; urls: JsonNode): Card = @@ -166,6 +163,9 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card = name = js{"name"}.getStr kind = parseEnum[CardKind](name[(name.find(":") + 1) ..< name.len], unknown) + if kind == unified: + return parseUnifiedCard(vals{"unified_card", "string_value"}.getStr) + result = Card( kind: kind, url: vals.getCardUrl(kind), @@ -190,7 +190,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card = result.url = vals{"player_url"}.getStrVal if "youtube.com" in result.url: result.url = result.url.replace("/embed/", "/watch?v=") - of audiospace, unified, unknown: + of audiospace, unknown: result.title = "This card type is not supported." else: discard diff --git a/src/types.nim b/src/types.nim index 5b329b7..8bb1956 100644 --- a/src/types.nim +++ b/src/types.nim @@ -70,12 +70,11 @@ type vmap = "video/vmap" VideoVariant* = object - videoType*: VideoType + contentType*: VideoType url*: string bitrate*: int Video* = object - videoId*: string durationMs*: int url*: string thumb*: string @@ -147,8 +146,6 @@ type Card* = object kind*: CardKind - id*: string - query*: string url*: string title*: string dest*: string diff --git a/src/views/tweet.nim b/src/views/tweet.nim index d435469..cede58c 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -97,7 +97,7 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode = img(src=thumb) renderVideoDisabled(video, path) else: - let vid = video.variants.filterIt(it.videoType == video.playbackType) + let vid = video.variants.filterIt(it.contentType == video.playbackType) let source = getVidUrl(vid[0].url) case video.playbackType of mp4: