Cache videos

This commit is contained in:
Zed 2019-08-06 19:02:38 +02:00
parent eeead99e32
commit bfcb74c6cc
4 changed files with 47 additions and 20 deletions

View File

@ -117,7 +117,7 @@ proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
result = json["guest_token"].to(string) result = json["guest_token"].to(string)
guestToken = result guestToken = result
proc getVideo*(tweet: Tweet; token, agent: string) {.async.} = proc getVideoFetch*(tweet: Tweet; token, agent: string) {.async.} =
if tweet.video.isNone(): return if tweet.video.isNone(): return
let headers = newHttpHeaders({ let headers = newHttpHeaders({
@ -135,16 +135,32 @@ proc getVideo*(tweet: Tweet; token, agent: string) {.async.} =
if getTime() - tokenUpdated > initDuration(seconds=1): if getTime() - tokenUpdated > initDuration(seconds=1):
tokenUpdated = getTime() tokenUpdated = getTime()
discard await getGuestToken(agent, force=true) discard await getGuestToken(agent, force=true)
await getVideo(tweet, guestToken, agent) await getVideoFetch(tweet, guestToken, agent)
return return
if tweet.card.isNone: if tweet.card.isNone:
tweet.video = some(parseVideo(json)) tweet.video = some(parseVideo(json, tweet.id))
else: else:
get(tweet.card).video = some(parseVideo(json)) get(tweet.card).video = some(parseVideo(json, tweet.id))
tweet.video = none(Video) tweet.video = none(Video)
tokenUses.inc tokenUses.inc
proc getVideoVar*(tweet: Tweet): var Option[Video] =
if tweet.card.isSome():
return get(tweet.card).video
else:
return tweet.video
proc getVideo*(tweet: Tweet; token, agent: string; force=false) {.async.} =
withDb:
try:
getVideoVar(tweet) = some(Video.getOne("videoId = ?", tweet.id))
except KeyError:
await getVideoFetch(tweet, token, agent)
var video = getVideoVar(tweet)
if video.isSome():
get(video).insert()
proc getPoll*(tweet: Tweet; agent: string) {.async.} = proc getPoll*(tweet: Tweet; agent: string) {.async.} =
if tweet.poll.isNone(): return if tweet.poll.isNone(): return

View File

@ -1,4 +1,4 @@
import asyncdispatch, asyncfile, httpclient, strutils, strformat, uri, os import asyncdispatch, asyncfile, httpclient, sequtils, strutils, strformat, uri, os
from net import Port from net import Port
import jester, regex import jester, regex
@ -34,11 +34,11 @@ proc showMultiTimeline(names: seq[string]; after, agent: string; query: Option[Q
q = some(Query(kind: multi, fromUser: names, excludes: @["replies"])) q = some(Query(kind: multi, fromUser: names, excludes: @["replies"]))
var timeline = renderMulti(await getTimelineSearch(get(q), after, agent), names.join(",")) var timeline = renderMulti(await getTimelineSearch(get(q), after, agent), names.join(","))
return renderMain(timeline, title=cfg.title, titleText=names.join(" | ")) return renderMain(timeline, title=cfg.title, titleText="Multi")
proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} = proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} =
let agent = getAgent() let agent = getAgent()
let names = name.strip(chars={'/'}).split(",") let names = name.strip(chars={'/'}).split(",").filterIt(it.len > 0)
if names.len == 1: if names.len == 1:
return await showSingleTimeline(names[0], after, agent, query) return await showSingleTimeline(names[0], after, agent, query)

View File

@ -125,7 +125,7 @@ proc parseConversation*(node: XmlNode): Conversation =
else: else:
result.replies.add parseThread(thread) result.replies.add parseThread(thread)
proc parseVideo*(node: JsonNode): Video = proc parseVideo*(node: JsonNode; tweetId: string): Video =
let let
track = node{"track"} track = node{"track"}
cType = track["contentType"].to(string) cType = track["contentType"].to(string)
@ -148,6 +148,7 @@ proc parseVideo*(node: JsonNode): Video =
else: else:
echo "Can't parse video of type ", cType echo "Can't parse video of type ", cType
result.videoId = tweetId
result.thumb = node["posterImage"].to(string) result.thumb = node["posterImage"].to(string)
proc parsePoll*(node: XmlNode): Poll = proc parsePoll*(node: XmlNode): Poll =

View File

@ -3,6 +3,10 @@ import norm/sqlite
export sqlite, options export sqlite, options
type
VideoType* = enum
vmap, m3u8, mp4
db("cache.db", "", "", ""): db("cache.db", "", "", ""):
type type
Profile* = object Profile* = object
@ -30,6 +34,24 @@ db("cache.db", "", "", ""):
formatIt: getTime().toUnix() formatIt: getTime().toUnix()
.}: Time .}: Time
Video* = object
videoId*: string
contentId*: string
durationMs*: int
url*: string
thumb*: string
views*: string
playbackType* {.
dbType: "STRING",
parseIt: parseEnum[VideoType](it.s),
formatIt: $it,
.}: VideoType
available* {.
dbType: "STRING",
parseIt: parseBool(it.s)
formatIt: $it
.}: bool
type type
QueryKind* = enum QueryKind* = enum
replies, media, multi, custom = "search" replies, media, multi, custom = "search"
@ -42,18 +64,6 @@ type
fromUser*: seq[string] fromUser*: seq[string]
sep*: string sep*: string
VideoType* = enum
vmap, m3u8, mp4
Video* = object
contentId*: string
playbackType*: VideoType
durationMs*: int
url*: string
thumb*: string
views*: string
available*: bool
Gif* = object Gif* = object
url*: string url*: string
thumb*: string thumb*: string