Routing updates

src/routes/embed.nim

@@ -1,17 +1,14 @@
 import asyncdispatch, strutils, sequtils, uri, options
-
 import jester
-
+import ".."/[types, api], ../views/embed
-import router_utils
-import ".."/[api, types, agents]
-import ../views/[embed]
-export getVideo
 
 export embed
 
 proc createEmbedRouter*(cfg: Config) =
   router embed:
     get "/i/videos/tweet/@id":
-      let tweet = Tweet(id: @"id".parseBiggestInt, video: some Video())
+      let convo = await getTweet(@"id")
-      await getVideo(tweet, getAgent(), "")
+      if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
-      resp renderVideoEmbed(cfg, tweet)
+        resp Http404
+
+      resp renderVideoEmbed(cfg, convo.tweet)

src/routes/list.nim

@@ -3,25 +3,43 @@ import strutils
 import jester
 
 import router_utils
-import ".."/[query, types, api, agents]
+import ".."/[query, types, redis_cache, api]
 import ../views/[general, timeline, list]
-export getListTimeline, getListMembers
+export getListTimeline, getGraphList
 
-template respList*(list, timeline: typed) =
+template respList*(list, timeline, vnode: typed) =
-  if list.minId.len == 0:
+  if list.id.len == 0:
     resp Http404, showError("List \"" & @"list" & "\" not found", cfg)
-  let html = renderList(timeline, list.query, @"name", @"list")
+
-  let rss = "/$1/lists/$2/rss" % [@"name", @"list"]
+  let
+    html = renderList(vnode, timeline.query, list)
+    rss = "/$1/lists/$2/rss" % [@"name", @"list"]
+
   resp renderMain(html, request, cfg, rss=rss)
 
 proc createListRouter*(cfg: Config) =
   router list:
     get "/@name/lists/@list":
       cond '.' notin @"name"
-      let list = await getListTimeline(@"name", @"list", @"max_position", getAgent())
+      cond @"name" != "i"
-      respList(list, renderTimelineTweets(list, cookiePrefs(), request.path))
+      let
+        list = await getCachedList(@"name", @"list")
+        timeline = await getListTimeline(list.id, getCursor())
+        vnode = renderTimelineTweets(timeline, cookiePrefs(), request.path)
+      respList(list, timeline, vnode)
 
     get "/@name/lists/@list/members":
       cond '.' notin @"name"
-      let list = await getListMembers(@"name", @"list", @"max_position", getAgent())
+      cond @"name" != "i"
-      respList(list, renderTimelineUsers(list, cookiePrefs(), request.path))
+      let
+        list = await getCachedList(@"name", @"list")
+        members = await getListMembers(list)
+      respList(list, members, renderTimelineUsers(members, cookiePrefs(), request.path))
+
+    get "/i/lists/@id":
+      cond '.' notin @"id"
+      let list = await getCachedList(id=(@"id"))
+      if list.id.len == 0:
+        resp Http404
+      await cache(list, time=listCacheTime)
+      redirect("/" & list.username & "/lists/" & list.name)

src/routes/media.nim

@@ -14,6 +14,7 @@ const m3u8Regex* = re"""url="(.+.m3u8)""""
 proc createMediaRouter*(cfg: Config) =
   router media:
     get "/pic/?":
+      echo "empty pic"
       resp Http404
 
     get "/pic/@url":

src/routes/resolver.nim

@@ -3,9 +3,8 @@ import strutils
 import jester
 
 import router_utils
-import ".."/[query, types, api, agents]
+import ".."/[query, types, api]
 import ../views/general
-export resolve
 
 template respResolved*(url, kind: string): untyped =
   let u = url

src/routes/router_utils.nim

@@ -1,9 +1,9 @@
-import strutils, sequtils, asyncdispatch, httpclient
+import strutils, sequtils, asyncdispatch, httpclient, uri
 from jester import Request
 import ../utils, ../prefs
 export utils, prefs
 
-template savePref*(pref, value: string; req: Request; expire=false): typed =
+template savePref*(pref, value: string; req: Request; expire=false) =
   if not expire or pref in cookies(req):
     setCookie(pref, value, daysForward(when expire: -10 else: 360),
               httpOnly=true, secure=cfg.useHttps)
@@ -17,6 +17,15 @@ template getPath*(): untyped {.dirty.} =
 template refPath*(): untyped {.dirty.} =
   if @"referer".len > 0: @"referer" else: "/"
 
+template getCursor*(): string =
+  let cursor = @"cursor"
+  decodeUrl(if cursor.len > 0: cursor else: @"max_position", false)
+
+template getCursor*(req: Request): string =
+  let cursor = req.params.getOrDefault("cursor")
+  decodeUrl(if cursor.len > 0: cursor
+            else: req.params.getOrDefault("max_position"), false)
+
 proc getNames*(name: string): seq[string] =
   name.strip(chars={'/'}).split(",").filterIt(it.len > 0)
 

src/routes/rss.nim

@@ -1,27 +1,28 @@
-import asyncdispatch, strutils
+import asyncdispatch, strutils, tables, times, sequtils
 
 import jester
 
 import router_utils, timeline
-import ".."/[cache, agents, query]
+import ".."/[redis_cache, query], ../views/general
-import ../views/general
 
 include "../views/rss.nimf"
 
+export times
+
 proc showRss*(req: Request; hostname: string; query: Query): Future[(string, string)] {.async.} =
   var profile: Profile
   var timeline: Timeline
   let
     name = req.params.getOrDefault("name")
-    after = req.params.getOrDefault("max_position")
+    after = getCursor(req)
     names = getNames(name)
 
   if names.len == 1:
     (profile, timeline) =
-      await fetchSingleTimeline(after, getAgent(), query, media=false)
+      await fetchSingleTimeline(after, query)
   else:
     let multiQuery = query.getMultiQuery(names)
-    timeline = await getSearch[Tweet](multiQuery, after, getAgent(), media=false)
+    timeline = await getSearch[Tweet](multiQuery, after)
     # this is kinda dumb
     profile = Profile(
       username: name,
@@ -32,16 +33,16 @@ proc showRss*(req: Request; hostname: string; query: Query): Future[(string, str
   if profile.suspended:
     return (profile.username, "suspended")
 
-  if timeline != nil:
+  if timeline.content.len > 0:
     let rss = renderTimelineRss(timeline, profile, hostname, multi=(names.len > 1))
-    return (rss, timeline.minId)
+    return (rss, timeline.bottom)
 
 template respRss*(rss, minId) =
   if rss.len == 0:
     resp Http404, showError("User \"" & @"name" & "\" not found", cfg)
   elif minId == "suspended":
     resp Http404, showError(getSuspended(rss), cfg)
-  let headers = {"Content-Type": "application/rss+xml;charset=utf-8", "Min-Id": minId}
+  let headers = {"Content-Type": "application/rss+xml; charset=utf-8", "Min-Id": minId}
   resp Http200, headers, rss
 
 proc createRssRouter*(cfg: Config) =
@@ -54,14 +55,36 @@ proc createRssRouter*(cfg: Config) =
       if query.kind != tweets:
         resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg)
 
-      let tweets = await getSearch[Tweet](query, @"max_position", getAgent(), media=false)
+      let
-      let rss = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg.hostname)
+        cursor = getCursor()
-      respRss(rss, tweets.minId)
+        key = genQueryParam(query) & cursor
+        (cRss, cCursor) = await getCachedRss(key)
+
+      if cRss.len > 0:
+        respRss(cRss, cCursor)
+
+      let
+        tweets = await getSearch[Tweet](query, cursor)
+        rss = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg.hostname)
+
+      await cacheRss(key, rss, tweets.bottom)
+      respRss(rss, tweets.bottom)
 
     get "/@name/rss":
       cond '.' notin @"name"
-      let (rss, minId) = await showRss(request, cfg.hostname, Query(fromUser: @[@"name"]))
+      let
-      respRss(rss, minId)
+        cursor = getCursor()
+        name = @"name"
+        (cRss, cCursor) = await getCachedRss(name & cursor)
+
+      if cRss.len > 0:
+        respRss(cRss, cCursor)
+
+      let (rss, rssCursor) = await showRss(request, cfg.hostname,
+                                           Query(fromUser: @[name]))
+
+      await cacheRss(name & cursor, rss, rssCursor)
+      respRss(rss, rssCursor)
 
     get "/@name/@tab/rss":
       cond '.' notin @"name"
@@ -74,11 +97,31 @@ proc createRssRouter*(cfg: Config) =
         of "search": initQuery(params(request), name=name)
         else: Query(fromUser: @[name])
 
-      let (rss, minId) = await showRss(request, cfg.hostname, query)
+      let
-      respRss(rss, minId)
+        key = @"name" & "/" & @"tab" & getCursor()
+        (cRss, cCursor) = await getCachedRss(key)
+
+      if cRss.len > 0:
+        respRss(cRss, cCursor)
+
+      let (rss, rssCursor) = await showRss(request, cfg.hostname, query)
+      await cacheRss(key, rss, rssCursor)
+      respRss(rss, rssCursor)
 
     get "/@name/lists/@list/rss":
       cond '.' notin @"name"
-      let list = await getListTimeline(@"name", @"list", @"max_position", getAgent(), media=false)
+      let
-      let rss = renderListRss(list.content, @"name", @"list", cfg.hostname)
+        cursor = getCursor()
-      respRss(rss, list.minId)
+        key = @"name" & "/" & @"list" & cursor
+        (cRss, cCursor) = await getCachedRss(key)
+
+      if cRss.len > 0:
+        respRss(cRss, cCursor)
+
+      let
+        list = await getCachedList(@"name", @"list")
+        timeline = await getListTimeline(list.id, cursor)
+        rss = renderListRss(timeline.content, list, cfg.hostname)
+
+      await cacheRss(key, rss, timeline.bottom)
+      respRss(rss, timeline.bottom)

src/routes/search.nim

@@ -3,7 +3,7 @@ import strutils, sequtils, uri
 import jester
 
 import router_utils
-import ".."/[query, types, api, agents]
+import ".."/[query, types, api]
 import ../views/[general, search]
 
 include "../views/opensearch.nimf"
@@ -23,10 +23,10 @@ proc createSearchRouter*(cfg: Config) =
       of users:
         if "," in @"q":
           redirect("/" & @"q")
-        let users = await getSearch[Profile](query, @"max_position", getAgent())
+        let users = await getSearch[Profile](query, getCursor())
         resp renderMain(renderUserSearch(users, prefs), request, cfg)
       of tweets:
-        let tweets = await getSearch[Tweet](query, @"max_position", getAgent())
+        let tweets = await getSearch[Tweet](query, getCursor())
         let rss = "/search/rss?" & genQueryUrl(query)
         resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
                         request, cfg, rss=rss)

src/routes/status.nim

@@ -3,12 +3,12 @@ import asyncdispatch, strutils, sequtils, uri, options
 import jester, karax/vdom
 
 import router_utils
-import ".."/[api, types, formatters, agents]
+import ".."/[types, formatters, api]
 import ../views/[general, status]
 
 export uri, sequtils, options
 export router_utils
-export api, formatters, agents
+export api, formatters
 export status
 
 proc createStatusRouter*(cfg: Config) =
@@ -18,33 +18,36 @@ proc createStatusRouter*(cfg: Config) =
       let prefs = cookiePrefs()
 
       if @"scroll".len > 0:
-        let replies = await getReplies(@"name", @"id", @"max_position", getAgent())
+        let replies = await getReplies(@"id", getCursor())
-        if replies == nil:
+        if replies.content.len == 0:
           resp Http404, ""
         resp $renderReplies(replies, prefs, getPath())
 
-      let conversation = await getTweet(@"name", @"id", @"max_position", getAgent())
+      let conv = await getTweet(@"id", getCursor())
-      if conversation == nil or conversation.tweet.id == 0:
+      if conv == nil:
+        echo "nil conv"
+
+      if conv == nil or conv.tweet == nil or conv.tweet.id == 0:
         var error = "Tweet not found"
-        if conversation != nil and conversation.tweet.tombstone.len > 0:
+        if conv != nil and conv.tweet != nil and conv.tweet.tombstone.len > 0:
-          error = conversation.tweet.tombstone
+          error = conv.tweet.tombstone
         resp Http404, showError(error, cfg)
 
       var
-        title = pageTitle(conversation.tweet)
+        title = pageTitle(conv.tweet)
-        ogTitle = pageTitle(conversation.tweet.profile)
+        ogTitle = pageTitle(conv.tweet.profile)
-        desc = conversation.tweet.text
+        desc = conv.tweet.text
-        images = conversation.tweet.photos
+        images = conv.tweet.photos
         video = ""
 
-      if conversation.tweet.video.isSome():
+      if conv.tweet.video.isSome():
-        images = @[get(conversation.tweet.video).thumb]
+        images = @[get(conv.tweet.video).thumb]
-        video = getVideoEmbed(cfg, conversation.tweet.id)
+        video = getVideoEmbed(cfg, conv.tweet.id)
-      elif conversation.tweet.gif.isSome():
+      elif conv.tweet.gif.isSome():
-        images = @[get(conversation.tweet.gif).thumb]
+        images = @[get(conv.tweet.gif).thumb]
-        video = getGifUrl(get(conversation.tweet.gif).url)
+        video = getGifUrl(get(conv.tweet.gif).url)
 
-      let html = renderConversation(conversation, prefs, getPath() & "#m")
+      let html = renderConversation(conv, prefs, getPath() & "#m")
       resp renderMain(html, request, cfg, title, desc,
                       images=images, video=video, ogTitle=ogTitle)
 

src/routes/timeline.nim

@@ -1,14 +1,14 @@
-import asyncdispatch, strutils, sequtils, uri, options
+import asyncdispatch, strutils, sequtils, uri, options, times
 import jester, karax/vdom
 
 import router_utils
-import ".."/[api, types, cache, formatters, agents, query]
+import ".."/[types, redis_cache, formatters, query, api]
 import ../views/[general, profile, timeline, status, search]
 
 export vdom
 export uri, sequtils
 export router_utils
-export api, cache, formatters, query, agents
+export redis_cache, formatters, query, api
 export profile, timeline, status
 
 proc getQuery*(request: Request; tab, name: string): Query =
@@ -18,33 +18,50 @@ proc getQuery*(request: Request; tab, name: string): Query =
   of "search": initQuery(params(request), name=name)
   else: Query(fromUser: @[name])
 
-proc fetchTimeline*(after, agent: string; query: Query): Future[Timeline] =
+proc fetchSingleTimeline*(after: string; query: Query; skipRail=false):
-  case query.kind
+                        Future[(Profile, Timeline, PhotoRail)] {.async.} =
-  of QueryKind.media: getMediaTimeline(query.fromUser[0], after, agent)
-  of posts: getTimeline(query.fromUser[0], after, agent)
-  else: getSearch[Tweet](query, after, agent)
-
-proc fetchSingleTimeline*(after, agent: string; query: Query;
-                          media=true): Future[(Profile, Timeline)] {.async.} =
   let name = query.fromUser[0]
-  var timeline: Timeline
-  var profile: Profile
-  var cachedProfile = hasCachedProfile(name)
 
-  if cachedProfile.isSome:
+  var
-    profile = get(cachedProfile)
+    profile = await getCachedProfile(name, fetch=false)
+    profileId = await getProfileId(name)
 
-  if query.kind == posts and cachedProfile.isNone:
+  if profile.username.len == 0 and profileId.len == 0:
-    (profile, timeline) = await getProfileAndTimeline(name, after, agent, media)
+    profile = await getCachedProfile(name)
-    cache(profile)
+    profileId = profile.id
+
+  if profile.suspended or profileId.len == 0:
+    result[0] = profile
+    return
+
+  var rail: Future[PhotoRail]
+  if skipRail or query.kind == media or profile.suspended or profile.protected:
+    rail = newFuture[PhotoRail]()
+    rail.complete(@[])
   else:
-    let timelineFut = fetchTimeline(after, agent, query)
+    rail = getCachedPhotoRail(profileId)
-    if cachedProfile.isNone:
-      profile = await getCachedProfile(name, agent)
-    timeline = await timelineFut
 
-  if profile.username.len == 0: return
+  var timeline =
-  return (profile, timeline)
+    case query.kind
+    of posts: await getTimeline(profileId, after)
+    of replies: await getTimeline(profileId, after, replies=true)
+    of media: await getMediaTimeline(profileId, after)
+    else: await getSearch[Tweet](query, after)
+
+  timeline.query = query
+
+  for tweet in timeline.content:
+    if tweet.profile.id == profileId or
+       tweet.profile.username.cmpIgnoreCase(name) == 0:
+      profile = tweet.profile
+      break
+
+  if profile.username.len == 0:
+    profile = await getCachedProfile(name)
+  else:
+    await cache(profile)
+
+  return (profile, timeline, await rail)
 
 proc getMultiQuery*(q: Query; names: seq[string]): Query =
   result = q
@@ -57,25 +74,20 @@ proc get*(req: Request; key: string): string =
 
 proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
                    rss, after: string): Future[string] {.async.} =
-  let agent = getAgent()
-
   if query.fromUser.len != 1:
     let
-      timeline = await getSearch[Tweet](query, after, agent)
+      timeline = await getSearch[Tweet](query, after)
       html = renderTweetSearch(timeline, prefs, getPath())
     return renderMain(html, request, cfg, "Multi", rss=rss)
 
-  let
+  var (p, t, r) = await fetchSingleTimeline(after, query)
-    rail = getPhotoRail(query.fromUser[0], agent, skip=(query.kind == media))
+
-    (p, t) = await fetchSingleTimeline(after, agent, query)
+  if p.suspended: return showError(getSuspended(p.username), cfg)
-    r = await rail
+  if p.id.len == 0: return
-  if p.username.len == 0: return
-  if p.suspended:
-    return showError(getSuspended(p.username), cfg)
 
   let pHtml = renderProfile(p, t, r, prefs, getPath())
-  return renderMain(pHtml, request, cfg, pageTitle(p), pageDesc(p),
+  result = renderMain(pHtml, request, cfg, pageTitle(p), pageDesc(p),
-                    rss=rss, images = @[p.getUserpic("_200x200")])
+                      rss=rss, images = @[p.getUserpic("_200x200")])
 
 template respTimeline*(timeline: typed) =
   let t = timeline
@@ -84,8 +96,6 @@ template respTimeline*(timeline: typed) =
   resp t
 
 proc createTimelineRouter*(cfg: Config) =
-  setProfileCacheTime(cfg.profileCacheTime)
-
   router timeline:
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
@@ -93,7 +103,7 @@ proc createTimelineRouter*(cfg: Config) =
       cond @"tab" in ["with_replies", "media", "search", ""]
       let
         prefs = cookiePrefs()
-        after = @"max_position"
+        after = getCursor()
         names = getNames(@"name")
 
       var query = request.getQuery(@"tab", @"name")
@@ -102,11 +112,13 @@ proc createTimelineRouter*(cfg: Config) =
 
       if @"scroll".len > 0:
         if query.fromUser.len != 1:
-          let timeline = await getSearch[Tweet](query, after, getAgent())
+          var timeline = await getSearch[Tweet](query, after)
+          if timeline.content.len == 0: resp Http404
           timeline.beginning = true
           resp $renderTweetSearch(timeline, prefs, getPath())
         else:
-          let timeline = await fetchTimeline(after, getAgent(), query)
+          var (_, timeline, _) = await fetchSingleTimeline(after, query, skipRail=true)
+          if timeline.content.len == 0: resp Http404
           timeline.beginning = true
           resp $renderTimelineTweets(timeline, prefs, getPath())
 

src/routes/unsupported.nim

@@ -16,5 +16,5 @@ proc createUnsupportedRouter*(cfg: Config) =
       resp renderMain(renderFeature(), request, cfg)
 
     get "/i/@i?/?@j?":
-      cond @"i" != "status"
+      cond @"i" notin ["status", "lists"]
       resp renderMain(renderFeature(), request, cfg)