Cache profiles
This commit is contained in:
parent
63d7528b8f
commit
6103db6893
|
@ -1,2 +1,3 @@
|
|||
nitter
|
||||
*.html
|
||||
*.html
|
||||
*.db
|
|
@ -1,74 +1,30 @@
|
|||
import sharedtables, times, hashes
|
||||
import asyncdispatch, times
|
||||
import types, api
|
||||
|
||||
# var
|
||||
# profileCache: SharedTable[int, Profile]
|
||||
# profileCacheTime = initDuration(seconds=10)
|
||||
withDb:
|
||||
try:
|
||||
createTables()
|
||||
except DbError:
|
||||
discard
|
||||
|
||||
# profileCache.init()
|
||||
var profileCacheTime = initDuration(seconds=60)
|
||||
|
||||
proc getCachedProfile*(username: string; force=false): Profile =
|
||||
return getProfile(username)
|
||||
# let index = username.hash
|
||||
|
||||
# try:
|
||||
# result = profileCache.mget(index)
|
||||
# # if force or getTime() - result.lastUpdated > profileCacheTime:
|
||||
# # result = getProfile(username)
|
||||
# # profileCache[username.hash] = deepCopy(result)
|
||||
# # return
|
||||
# except KeyError:
|
||||
# # result = getProfile(username)
|
||||
# # profileCache.add(username.hash, deepCopy(result))
|
||||
|
||||
|
||||
|
||||
# var profile: Profile
|
||||
# profileCache.withKey(index) do (k: int, v: var Profile, pairExists: var bool):
|
||||
# v = getProfile(username)
|
||||
# profile = v
|
||||
# echo v
|
||||
# pairExists = true
|
||||
# echo profile.username
|
||||
# return profile
|
||||
|
||||
# profileCache.withValue(hash(username), value) do:
|
||||
# if getTime() - value.lastUpdated > profileCacheTime or force:
|
||||
# result = getProfile(username)
|
||||
# value = result
|
||||
# else:
|
||||
# result = value
|
||||
# do:
|
||||
# result = getProfile(username)
|
||||
# value = result
|
||||
|
||||
# var profile: Profile
|
||||
|
||||
# profileCache.withKey(username.hash) do (k: int, v: var Profile, pairExists: var bool):
|
||||
# if pairExists and getTime() - v.lastUpdated < profileCacheTime and not force:
|
||||
# profile = deepCopy(v)
|
||||
# echo "cached"
|
||||
# else:
|
||||
# profile = getProfile(username)
|
||||
# v = deepCopy(profile)
|
||||
# pairExists = true
|
||||
# echo "fetched"
|
||||
|
||||
# return profile
|
||||
|
||||
# try:
|
||||
# result = profileCache.mget(username.hash)
|
||||
# if force or getTime() - result.lastUpdated > profileCacheTime:
|
||||
# result = getProfile(username)
|
||||
# profileCache[username.hash] = deepCopy(result)
|
||||
# return
|
||||
# except KeyError:
|
||||
# result = getProfile(username)
|
||||
# profileCache.add(username.hash, deepCopy(result))
|
||||
|
||||
# if not result.isNil or force or
|
||||
# getTime() - result.lastUpdated > profileCacheTime:
|
||||
# result = getProfile(username)
|
||||
# profileCache[username] = result
|
||||
# return
|
||||
proc outdated(profile: Profile): bool =
|
||||
getTime() - profile.updated > profileCacheTime
|
||||
|
||||
proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} =
|
||||
withDb:
|
||||
try:
|
||||
result.getOne("username = ?", username)
|
||||
doAssert(not result.outdated())
|
||||
except:
|
||||
if result.id == 0:
|
||||
result = await getProfile(username)
|
||||
result.insert()
|
||||
elif result.outdated():
|
||||
let
|
||||
profile = await getProfile(username)
|
||||
oldId = result.id
|
||||
result = profile
|
||||
result.id = oldId
|
||||
result.update()
|
||||
|
|
|
@ -64,10 +64,10 @@ proc getUserpic*(profile: Profile; style=""): string =
|
|||
getUserPic(profile.userpic, style)
|
||||
|
||||
proc getGifSrc*(tweet: Tweet): string =
|
||||
fmt"https://video.twimg.com/tweet_video/{tweet.gif}.mp4"
|
||||
fmt"https://video.twimg.com/tweet_video/{tweet.gif.get()}.mp4"
|
||||
|
||||
proc getGifThumb*(tweet: Tweet): string =
|
||||
fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif}.jpg"
|
||||
fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif.get()}.jpg"
|
||||
|
||||
proc formatName*(profile: Profile): string =
|
||||
result = xmltree.escape(profile.fullname)
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
import asyncdispatch, httpclient, times, strutils, hashes, random, uri
|
||||
import jester, regex
|
||||
|
||||
import api, utils, types
|
||||
import api, utils, types, cache
|
||||
import views/[user, general, conversation]
|
||||
|
||||
proc showTimeline(name: string; num=""): Future[string] {.async.} =
|
||||
let
|
||||
username = name.strip(chars={'/'})
|
||||
profileFut = getProfile(username)
|
||||
profileFut = getCachedProfile(username)
|
||||
tweetsFut = getTimeline(username, after=num)
|
||||
|
||||
let profile = await profileFut
|
||||
|
|
|
@ -47,7 +47,6 @@ proc parseTweet*(tweet: XmlNode): Tweet =
|
|||
result.id = tweet.getAttr("data-item-id")
|
||||
result.link = tweet.getAttr("data-permalink-path")
|
||||
result.text = tweet.selectText(".tweet-text").stripTwitterUrls()
|
||||
result.retweetBy = tweet.selectText(".js-retweet-text > a > b")
|
||||
result.pinned = "pinned" in tweet.getAttr("class")
|
||||
result.profile = parseTweetProfile(tweet)
|
||||
|
||||
|
@ -70,6 +69,10 @@ proc parseTweet*(tweet: XmlNode): Tweet =
|
|||
of "retweets": result.retweets = num
|
||||
else: discard
|
||||
|
||||
let by = tweet.selectText(".js-retweet-text > a > b")
|
||||
if by.len > 0:
|
||||
result.retweetBy = some(by)
|
||||
|
||||
for photo in tweet.querySelectorAll(".AdaptiveMedia-photoContainer"):
|
||||
result.photos.add photo.attrs["data-image-url"]
|
||||
|
||||
|
@ -77,9 +80,9 @@ proc parseTweet*(tweet: XmlNode): Tweet =
|
|||
if player.len > 0:
|
||||
let thumb = player.replace(re".+:url\('([^']+)'\)", "$1")
|
||||
if "tweet_video" in thumb:
|
||||
result.gif = thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1")
|
||||
result.gif = some(thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1"))
|
||||
else:
|
||||
result.videoThumb = thumb
|
||||
result.videoThumb = some(thumb)
|
||||
|
||||
proc parseTweets*(node: XmlNode): Tweets =
|
||||
if node.isNil: return
|
||||
|
|
|
@ -1,18 +1,36 @@
|
|||
import times, sequtils
|
||||
import times, sequtils, strutils, options
|
||||
import norm/sqlite
|
||||
|
||||
export sqlite, options
|
||||
|
||||
db("cache.db", "", "", ""):
|
||||
type
|
||||
Profile* = object
|
||||
username*: string
|
||||
fullname*: string
|
||||
description*: string
|
||||
userpic*: string
|
||||
banner*: string
|
||||
following*: string
|
||||
followers*: string
|
||||
tweets*: string
|
||||
verified* {.
|
||||
dbType: "STRING",
|
||||
parseIt: parseBool(it.s)
|
||||
formatIt: $it
|
||||
.}: bool
|
||||
protected* {.
|
||||
dbType: "STRING",
|
||||
parseIt: parseBool(it.s)
|
||||
formatIt: $it
|
||||
.}: bool
|
||||
updated* {.
|
||||
dbType: "INTEGER",
|
||||
parseIt: it.i.fromUnix(),
|
||||
formatIt: getTime().toUnix()
|
||||
.}: Time
|
||||
|
||||
type
|
||||
Profile* = object
|
||||
username*: string
|
||||
fullname*: string
|
||||
description*: string
|
||||
userpic*: string
|
||||
banner*: string
|
||||
following*: string
|
||||
followers*: string
|
||||
tweets*: string
|
||||
verified*: bool
|
||||
protected*: bool
|
||||
|
||||
Tweet* = object
|
||||
id*: string
|
||||
profile*: Profile
|
||||
|
@ -23,12 +41,12 @@ type
|
|||
replies*: string
|
||||
retweets*: string
|
||||
likes*: string
|
||||
retweetBy*: string
|
||||
pinned*: bool
|
||||
photos*: seq[string]
|
||||
gif*: string
|
||||
video*: string
|
||||
videoThumb*: string
|
||||
retweetBy*: Option[string]
|
||||
gif*: Option[string]
|
||||
video*: Option[string]
|
||||
videoThumb*: Option[string]
|
||||
|
||||
Tweets* = seq[Tweet]
|
||||
|
||||
|
|
|
@ -3,9 +3,9 @@
|
|||
#import ../types, ../formatters, ../utils
|
||||
#
|
||||
#proc renderHeading(tweet: Tweet): string =
|
||||
#if tweet.retweetBy != "":
|
||||
#if tweet.retweetBy.isSome:
|
||||
<div class="retweet">
|
||||
<span>🔄 ${tweet.retweetBy} retweeted</span>
|
||||
<span>🔄 ${tweet.retweetBy.get()} retweeted</span>
|
||||
</div>
|
||||
#end if
|
||||
#if tweet.pinned:
|
||||
|
@ -59,7 +59,7 @@
|
|||
<div class="attachments media-body">
|
||||
<div class="gallery-row" style="max-height: unset;">
|
||||
<div class="attachment image">
|
||||
<video poster=${tweet.videoThumb} style="width: 100%; height: 100%;" autoplay muted loop></video>
|
||||
<video poster=${tweet.videoThumb.get()} style="width: 100%; height: 100%;" autoplay muted loop></video>
|
||||
<div class="video-overlay">
|
||||
<p>Video playback not supported</p>
|
||||
</div>
|
||||
|
@ -102,9 +102,9 @@
|
|||
</div>
|
||||
#if tweet.photos.len > 0:
|
||||
${renderMediaGroup(tweet)}
|
||||
#elif tweet.videoThumb.len > 0:
|
||||
#elif tweet.videoThumb.isSome:
|
||||
${renderVideo(tweet)}
|
||||
#elif tweet.gif.len > 0:
|
||||
#elif tweet.gif.isSome:
|
||||
${renderGif(tweet)}
|
||||
#end if
|
||||
${renderStats(tweet)}
|
||||
|
|
|
@ -73,7 +73,7 @@
|
|||
#for tweet in tweets:
|
||||
#if tweet in retweets: continue
|
||||
#end if
|
||||
#if tweet.retweetBy.len > 0: retweets.add tweet
|
||||
#if tweet.retweetBy.isSome: retweets.add tweet
|
||||
#end if
|
||||
${renderTweet(tweet, "timeline-tweet")}
|
||||
#end for
|
||||
|
|
Loading…
Reference in New Issue