Cache profiles

.gitignore

@@ -1,2 +1,3 @@
 nitter
 *.html
+*.db

src/cache.nim

@@ -1,74 +1,30 @@
-import sharedtables, times, hashes
+import asyncdispatch, times
 import types, api
 
-# var
+withDb:
-#   profileCache: SharedTable[int, Profile]
+  try:
-#   profileCacheTime = initDuration(seconds=10)
+    createTables()
+  except DbError:
+    discard
 
-# profileCache.init()
+var profileCacheTime = initDuration(seconds=60)
 
-proc getCachedProfile*(username: string; force=false): Profile =
+proc outdated(profile: Profile): bool =
-  return getProfile(username)
+  getTime() - profile.updated > profileCacheTime
-  # let index = username.hash
-
-  # try:
-  #   result = profileCache.mget(index)
-  #   # if force or getTime() - result.lastUpdated > profileCacheTime:
-  #   #   result = getProfile(username)
-  #   #   profileCache[username.hash] = deepCopy(result)
-  #   #   return
-  # except KeyError:
-  #   # result = getProfile(username)
-  #   # profileCache.add(username.hash, deepCopy(result))
-
-
-
-  #   var profile: Profile
-  #   profileCache.withKey(index) do (k: int, v: var Profile, pairExists: var bool):
-  #     v = getProfile(username)
-  #     profile = v
-  #     echo v
-  #     pairExists = true
-  #   echo profile.username
-  #   return profile
-
-  # profileCache.withValue(hash(username), value) do:
-  #   if getTime() - value.lastUpdated > profileCacheTime or force:
-  #     result = getProfile(username)
-  #     value = result
-  #   else:
-  #     result = value
-  # do:
-  #   result = getProfile(username)
-  #   value = result
-
-  # var profile: Profile
-
-  # profileCache.withKey(username.hash) do (k: int, v: var Profile, pairExists: var bool):
-  #   if pairExists and getTime() - v.lastUpdated < profileCacheTime and not force:
-  #     profile = deepCopy(v)
-  #     echo "cached"
-  #   else:
-  #     profile = getProfile(username)
-  #     v = deepCopy(profile)
-  #     pairExists = true
-  #     echo "fetched"
-
-  # return profile
-
-  # try:
-  #   result = profileCache.mget(username.hash)
-  #   if force or getTime() - result.lastUpdated > profileCacheTime:
-  #     result = getProfile(username)
-  #     profileCache[username.hash] = deepCopy(result)
-  #     return
-  # except KeyError:
-  #   result = getProfile(username)
-  #   profileCache.add(username.hash, deepCopy(result))
-
-  # if not result.isNil or force or
-  #   getTime() - result.lastUpdated > profileCacheTime:
-  #   result = getProfile(username)
-  #   profileCache[username] = result
-    # return
 
+proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} =
+  withDb:
+    try:
+      result.getOne("username = ?", username)
+      doAssert(not result.outdated())
+    except:
+      if result.id == 0:
+        result = await getProfile(username)
+        result.insert()
+      elif result.outdated():
+        let
+          profile = await getProfile(username)
+          oldId = result.id
+        result = profile
+        result.id = oldId
+        result.update()

src/formatters.nim

@@ -64,10 +64,10 @@ proc getUserpic*(profile: Profile; style=""): string =
   getUserPic(profile.userpic, style)
 
 proc getGifSrc*(tweet: Tweet): string =
-  fmt"https://video.twimg.com/tweet_video/{tweet.gif}.mp4"
+  fmt"https://video.twimg.com/tweet_video/{tweet.gif.get()}.mp4"
 
 proc getGifThumb*(tweet: Tweet): string =
-  fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif}.jpg"
+  fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif.get()}.jpg"
 
 proc formatName*(profile: Profile): string =
   result = xmltree.escape(profile.fullname)

src/nitter.nim

@@ -1,13 +1,13 @@
 import asyncdispatch, httpclient, times, strutils, hashes, random, uri
 import jester, regex
 
-import api, utils, types
+import api, utils, types, cache
 import views/[user, general, conversation]
 
 proc showTimeline(name: string; num=""): Future[string] {.async.} =
   let
     username = name.strip(chars={'/'})
-    profileFut = getProfile(username)
+    profileFut = getCachedProfile(username)
     tweetsFut = getTimeline(username, after=num)
 
   let profile = await profileFut

src/parser.nim

@@ -47,7 +47,6 @@ proc parseTweet*(tweet: XmlNode): Tweet =
   result.id = tweet.getAttr("data-item-id")
   result.link = tweet.getAttr("data-permalink-path")
   result.text = tweet.selectText(".tweet-text").stripTwitterUrls()
-  result.retweetBy = tweet.selectText(".js-retweet-text > a > b")
   result.pinned = "pinned" in tweet.getAttr("class")
   result.profile = parseTweetProfile(tweet)
 
@@ -70,6 +69,10 @@ proc parseTweet*(tweet: XmlNode): Tweet =
     of "retweets": result.retweets = num
     else: discard
 
+  let by = tweet.selectText(".js-retweet-text > a > b")
+  if by.len > 0:
+    result.retweetBy = some(by)
+
   for photo in tweet.querySelectorAll(".AdaptiveMedia-photoContainer"):
     result.photos.add photo.attrs["data-image-url"]
 
@@ -77,9 +80,9 @@ proc parseTweet*(tweet: XmlNode): Tweet =
   if player.len > 0:
     let thumb = player.replace(re".+:url\('([^']+)'\)", "$1")
     if "tweet_video" in thumb:
-      result.gif = thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1")
+      result.gif = some(thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1"))
     else:
-      result.videoThumb = thumb
+      result.videoThumb = some(thumb)
 
 proc parseTweets*(node: XmlNode): Tweets =
   if node.isNil: return

src/types.nim

@@ -1,18 +1,36 @@
-import times, sequtils
+import times, sequtils, strutils, options
+import norm/sqlite
+
+export sqlite, options
+
+db("cache.db", "", "", ""):
+  type
+    Profile* = object
+      username*: string
+      fullname*: string
+      description*: string
+      userpic*: string
+      banner*: string
+      following*: string
+      followers*: string
+      tweets*: string
+      verified* {.
+          dbType: "STRING",
+          parseIt: parseBool(it.s)
+          formatIt: $it
+        .}: bool
+      protected* {.
+          dbType: "STRING",
+          parseIt: parseBool(it.s)
+          formatIt: $it
+        .}: bool
+      updated* {.
+          dbType: "INTEGER",
+          parseIt: it.i.fromUnix(),
+          formatIt: getTime().toUnix()
+        .}: Time
 
 type
-  Profile* = object
-    username*: string
-    fullname*: string
-    description*: string
-    userpic*: string
-    banner*: string
-    following*: string
-    followers*: string
-    tweets*: string
-    verified*: bool
-    protected*: bool
-
   Tweet* = object
     id*: string
     profile*: Profile
@@ -23,12 +41,12 @@ type
     replies*: string
     retweets*: string
     likes*: string
-    retweetBy*: string
     pinned*: bool
     photos*: seq[string]
-    gif*: string
+    retweetBy*: Option[string]
-    video*: string
+    gif*: Option[string]
-    videoThumb*: string
+    video*: Option[string]
+    videoThumb*: Option[string]
 
   Tweets* = seq[Tweet]
 

src/views/tweet.nim

@@ -3,9 +3,9 @@
 #import ../types, ../formatters, ../utils
 #
 #proc renderHeading(tweet: Tweet): string =
-#if tweet.retweetBy != "":
+#if tweet.retweetBy.isSome:
   <div class="retweet">
-    <span>🔄 ${tweet.retweetBy} retweeted</span>
+    <span>🔄 ${tweet.retweetBy.get()} retweeted</span>
   </div>
 #end if
 #if tweet.pinned:
@@ -59,7 +59,7 @@
 <div class="attachments media-body">
   <div class="gallery-row" style="max-height: unset;">
     <div class="attachment image">
-    <video poster=${tweet.videoThumb} style="width: 100%; height: 100%;" autoplay muted loop></video>
+    <video poster=${tweet.videoThumb.get()} style="width: 100%; height: 100%;" autoplay muted loop></video>
     <div class="video-overlay">
       <p>Video playback not supported</p>
     </div>
@@ -102,9 +102,9 @@
       </div>
       #if tweet.photos.len > 0:
         ${renderMediaGroup(tweet)}
-      #elif tweet.videoThumb.len > 0:
+      #elif tweet.videoThumb.isSome:
         ${renderVideo(tweet)}
-      #elif tweet.gif.len > 0:
+      #elif tweet.gif.isSome:
         ${renderGif(tweet)}
       #end if
       ${renderStats(tweet)}

src/views/user.nim

@@ -73,7 +73,7 @@
   #for tweet in tweets:
     #if tweet in retweets: continue
     #end if
-    #if tweet.retweetBy.len > 0: retweets.add tweet
+    #if tweet.retweetBy.isSome: retweets.add tweet
     #end if
     ${renderTweet(tweet, "timeline-tweet")}
   #end for