Cache profiles

This commit is contained in:
Zed 2019-06-20 20:04:18 +02:00
parent 63d7528b8f
commit 6103db6893
8 changed files with 78 additions and 100 deletions

3
.gitignore vendored
View File

@ -1,2 +1,3 @@
nitter
*.html
*.html
*.db

View File

@ -1,74 +1,30 @@
import sharedtables, times, hashes
import asyncdispatch, times
import types, api
# var
# profileCache: SharedTable[int, Profile]
# profileCacheTime = initDuration(seconds=10)
withDb:
try:
createTables()
except DbError:
discard
# profileCache.init()
var profileCacheTime = initDuration(seconds=60)
proc getCachedProfile*(username: string; force=false): Profile =
return getProfile(username)
# let index = username.hash
# try:
# result = profileCache.mget(index)
# # if force or getTime() - result.lastUpdated > profileCacheTime:
# # result = getProfile(username)
# # profileCache[username.hash] = deepCopy(result)
# # return
# except KeyError:
# # result = getProfile(username)
# # profileCache.add(username.hash, deepCopy(result))
# var profile: Profile
# profileCache.withKey(index) do (k: int, v: var Profile, pairExists: var bool):
# v = getProfile(username)
# profile = v
# echo v
# pairExists = true
# echo profile.username
# return profile
# profileCache.withValue(hash(username), value) do:
# if getTime() - value.lastUpdated > profileCacheTime or force:
# result = getProfile(username)
# value = result
# else:
# result = value
# do:
# result = getProfile(username)
# value = result
# var profile: Profile
# profileCache.withKey(username.hash) do (k: int, v: var Profile, pairExists: var bool):
# if pairExists and getTime() - v.lastUpdated < profileCacheTime and not force:
# profile = deepCopy(v)
# echo "cached"
# else:
# profile = getProfile(username)
# v = deepCopy(profile)
# pairExists = true
# echo "fetched"
# return profile
# try:
# result = profileCache.mget(username.hash)
# if force or getTime() - result.lastUpdated > profileCacheTime:
# result = getProfile(username)
# profileCache[username.hash] = deepCopy(result)
# return
# except KeyError:
# result = getProfile(username)
# profileCache.add(username.hash, deepCopy(result))
# if not result.isNil or force or
# getTime() - result.lastUpdated > profileCacheTime:
# result = getProfile(username)
# profileCache[username] = result
# return
proc outdated(profile: Profile): bool =
getTime() - profile.updated > profileCacheTime
proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} =
withDb:
try:
result.getOne("username = ?", username)
doAssert(not result.outdated())
except:
if result.id == 0:
result = await getProfile(username)
result.insert()
elif result.outdated():
let
profile = await getProfile(username)
oldId = result.id
result = profile
result.id = oldId
result.update()

View File

@ -64,10 +64,10 @@ proc getUserpic*(profile: Profile; style=""): string =
getUserPic(profile.userpic, style)
proc getGifSrc*(tweet: Tweet): string =
fmt"https://video.twimg.com/tweet_video/{tweet.gif}.mp4"
fmt"https://video.twimg.com/tweet_video/{tweet.gif.get()}.mp4"
proc getGifThumb*(tweet: Tweet): string =
fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif}.jpg"
fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif.get()}.jpg"
proc formatName*(profile: Profile): string =
result = xmltree.escape(profile.fullname)

View File

@ -1,13 +1,13 @@
import asyncdispatch, httpclient, times, strutils, hashes, random, uri
import jester, regex
import api, utils, types
import api, utils, types, cache
import views/[user, general, conversation]
proc showTimeline(name: string; num=""): Future[string] {.async.} =
let
username = name.strip(chars={'/'})
profileFut = getProfile(username)
profileFut = getCachedProfile(username)
tweetsFut = getTimeline(username, after=num)
let profile = await profileFut

View File

@ -47,7 +47,6 @@ proc parseTweet*(tweet: XmlNode): Tweet =
result.id = tweet.getAttr("data-item-id")
result.link = tweet.getAttr("data-permalink-path")
result.text = tweet.selectText(".tweet-text").stripTwitterUrls()
result.retweetBy = tweet.selectText(".js-retweet-text > a > b")
result.pinned = "pinned" in tweet.getAttr("class")
result.profile = parseTweetProfile(tweet)
@ -70,6 +69,10 @@ proc parseTweet*(tweet: XmlNode): Tweet =
of "retweets": result.retweets = num
else: discard
let by = tweet.selectText(".js-retweet-text > a > b")
if by.len > 0:
result.retweetBy = some(by)
for photo in tweet.querySelectorAll(".AdaptiveMedia-photoContainer"):
result.photos.add photo.attrs["data-image-url"]
@ -77,9 +80,9 @@ proc parseTweet*(tweet: XmlNode): Tweet =
if player.len > 0:
let thumb = player.replace(re".+:url\('([^']+)'\)", "$1")
if "tweet_video" in thumb:
result.gif = thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1")
result.gif = some(thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1"))
else:
result.videoThumb = thumb
result.videoThumb = some(thumb)
proc parseTweets*(node: XmlNode): Tweets =
if node.isNil: return

View File

@ -1,18 +1,36 @@
import times, sequtils
import times, sequtils, strutils, options
import norm/sqlite
export sqlite, options
db("cache.db", "", "", ""):
type
Profile* = object
username*: string
fullname*: string
description*: string
userpic*: string
banner*: string
following*: string
followers*: string
tweets*: string
verified* {.
dbType: "STRING",
parseIt: parseBool(it.s)
formatIt: $it
.}: bool
protected* {.
dbType: "STRING",
parseIt: parseBool(it.s)
formatIt: $it
.}: bool
updated* {.
dbType: "INTEGER",
parseIt: it.i.fromUnix(),
formatIt: getTime().toUnix()
.}: Time
type
Profile* = object
username*: string
fullname*: string
description*: string
userpic*: string
banner*: string
following*: string
followers*: string
tweets*: string
verified*: bool
protected*: bool
Tweet* = object
id*: string
profile*: Profile
@ -23,12 +41,12 @@ type
replies*: string
retweets*: string
likes*: string
retweetBy*: string
pinned*: bool
photos*: seq[string]
gif*: string
video*: string
videoThumb*: string
retweetBy*: Option[string]
gif*: Option[string]
video*: Option[string]
videoThumb*: Option[string]
Tweets* = seq[Tweet]

View File

@ -3,9 +3,9 @@
#import ../types, ../formatters, ../utils
#
#proc renderHeading(tweet: Tweet): string =
#if tweet.retweetBy != "":
#if tweet.retweetBy.isSome:
<div class="retweet">
<span>🔄 ${tweet.retweetBy} retweeted</span>
<span>🔄 ${tweet.retweetBy.get()} retweeted</span>
</div>
#end if
#if tweet.pinned:
@ -59,7 +59,7 @@
<div class="attachments media-body">
<div class="gallery-row" style="max-height: unset;">
<div class="attachment image">
<video poster=${tweet.videoThumb} style="width: 100%; height: 100%;" autoplay muted loop></video>
<video poster=${tweet.videoThumb.get()} style="width: 100%; height: 100%;" autoplay muted loop></video>
<div class="video-overlay">
<p>Video playback not supported</p>
</div>
@ -102,9 +102,9 @@
</div>
#if tweet.photos.len > 0:
${renderMediaGroup(tweet)}
#elif tweet.videoThumb.len > 0:
#elif tweet.videoThumb.isSome:
${renderVideo(tweet)}
#elif tweet.gif.len > 0:
#elif tweet.gif.isSome:
${renderGif(tweet)}
#end if
${renderStats(tweet)}

View File

@ -73,7 +73,7 @@
#for tweet in tweets:
#if tweet in retweets: continue
#end if
#if tweet.retweetBy.len > 0: retweets.add tweet
#if tweet.retweetBy.isSome: retweets.add tweet
#end if
${renderTweet(tweet, "timeline-tweet")}
#end for