blankie
/
nitter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use random user agents

This commit is contained in:
Zed 2019-07-31 08:36:24 +02:00
parent d0ee8e8403
commit 6b6e5b3a40
4 changed files with 143 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
src/agents.nim Normal file
View File

@ -0,0 +1,90 @@
import random
const userAgents = [
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0",
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0",
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/38.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 9.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/38.0",
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/40.1",
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/43.0",
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/50.0",
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
"Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
"Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
"Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16"
]
proc getAgent*(): string = sample(userAgents)

87
src/api.nim
View File

@ -4,7 +4,6 @@ import sequtils, strutils, json, xmltree, uri
import types, parser, parserutils, formatters, search import types, parser, parserutils, formatters, search
const const
agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
lang = "en-US,en;q=0.9" lang = "en-US,en;q=0.9"
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3" cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
@ -61,7 +60,7 @@ proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
except: except:
return nil return nil
proc getGuestToken(force=false): Future[string] {.async.} = proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
if getTime() - tokenUpdated < tokenLifetime and if getTime() - tokenUpdated < tokenLifetime and
not force and tokenUses < tokenMaxUses: not force and tokenUses < tokenMaxUses:
return guestToken return guestToken
@ -85,7 +84,7 @@ proc getGuestToken(force=false): Future[string] {.async.} =
result = json["guest_token"].to(string) result = json["guest_token"].to(string)
guestToken = result guestToken = result
proc getVideo*(tweet: Tweet; token: string) {.async.} = proc getVideo*(tweet: Tweet; token, agent: string) {.async.} =
if tweet.video.isNone(): return if tweet.video.isNone(): return
let headers = newHttpHeaders({ let headers = newHttpHeaders({
@ -102,8 +101,8 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
if json == nil: if json == nil:
if getTime() - tokenUpdated > initDuration(seconds=1): if getTime() - tokenUpdated > initDuration(seconds=1):
tokenUpdated = getTime() tokenUpdated = getTime()
discard await getGuestToken(force=true) discard await getGuestToken(agent, force=true)
await getVideo(tweet, guestToken) await getVideo(tweet, guestToken, agent)
return return
if tweet.card.isNone: if tweet.card.isNone:
@ -113,31 +112,31 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
tweet.video = none(Video) tweet.video = none(Video)
tokenUses.inc tokenUses.inc
proc getVideos*(thread: Thread; token="") {.async.} = proc getVideos*(thread: Thread; agent: string; token="") {.async.} =
if thread == nil: return if thread == nil: return
var gToken = token var gToken = token
if gToken.len == 0: if gToken.len == 0:
gToken = await getGuestToken() gToken = await getGuestToken(agent)
var videoFuts: seq[Future[void]] var videoFuts: seq[Future[void]]
for tweet in thread.tweets.filterIt(it.video.isSome): for tweet in thread.tweets.filterIt(it.video.isSome):
videoFuts.add getVideo(tweet, gToken) videoFuts.add getVideo(tweet, gToken, agent)
await all(videoFuts) await all(videoFuts)
proc getConversationVideos*(convo: Conversation) {.async.} = proc getConversationVideos*(convo: Conversation; agent: string) {.async.} =
var token = await getGuestToken() var token = await getGuestToken(agent)
var futs: seq[Future[void]] var futs: seq[Future[void]]
futs.add getVideo(convo.tweet, token) futs.add getVideo(convo.tweet, token, agent)
futs.add convo.replies.mapIt(getVideos(it, token)) futs.add convo.replies.mapIt(getVideos(it, token, agent))
futs.add getVideos(convo.before, token) futs.add getVideos(convo.before, token, agent)
futs.add getVideos(convo.after, token) futs.add getVideos(convo.after, token, agent)
await all(futs) await all(futs)
proc getPoll*(tweet: Tweet) {.async.} = proc getPoll*(tweet: Tweet; agent: string) {.async.} =
if tweet.poll.isNone(): return if tweet.poll.isNone(): return
let headers = newHttpHeaders({ let headers = newHttpHeaders({
@ -154,20 +153,20 @@ proc getPoll*(tweet: Tweet) {.async.} =
tweet.poll = some(parsePoll(html)) tweet.poll = some(parsePoll(html))
proc getPolls*(thread: Thread) {.async.} = proc getPolls*(thread: Thread; agent: string) {.async.} =
if thread == nil: return if thread == nil: return
var polls = thread.tweets.filterIt(it.poll.isSome) var polls = thread.tweets.filterIt(it.poll.isSome)
await all(polls.map(getPoll)) await all(polls.mapIt(getPoll(it, agent)))
proc getConversationPolls*(convo: Conversation) {.async.} = proc getConversationPolls*(convo: Conversation; agent: string) {.async.} =
var futs: seq[Future[void]] var futs: seq[Future[void]]
futs.add getPoll(convo.tweet) futs.add getPoll(convo.tweet, agent)
futs.add getPolls(convo.before) futs.add getPolls(convo.before, agent)
futs.add getPolls(convo.after) futs.add getPolls(convo.after, agent)
futs.add convo.replies.map(getPolls) futs.add convo.replies.mapIt(getPolls(it, agent))
await all(futs) await all(futs)
proc getCard*(tweet: Tweet) {.async.} = proc getCard*(tweet: Tweet; agent: string) {.async.} =
if tweet.card.isNone(): return if tweet.card.isNone(): return
let headers = newHttpHeaders({ let headers = newHttpHeaders({
@ -184,20 +183,20 @@ proc getCard*(tweet: Tweet) {.async.} =
parseCard(get(tweet.card), html) parseCard(get(tweet.card), html)
proc getCards*(thread: Thread) {.async.} = proc getCards*(thread: Thread; agent: string) {.async.} =
if thread == nil: return if thread == nil: return
var cards = thread.tweets.filterIt(it.card.isSome) var cards = thread.tweets.filterIt(it.card.isSome)
await all(cards.map(getCard)) await all(cards.mapIt(getCard(it, agent)))
proc getConversationCards*(convo: Conversation) {.async.} = proc getConversationCards*(convo: Conversation; agent: string) {.async.} =
var futs: seq[Future[void]] var futs: seq[Future[void]]
futs.add getCard(convo.tweet) futs.add getCard(convo.tweet, agent)
futs.add getCards(convo.before) futs.add getCards(convo.before, agent)
futs.add getCards(convo.after) futs.add getCards(convo.after, agent)
futs.add convo.replies.map(getCards) futs.add convo.replies.mapIt(getCards(it, agent))
await all(futs) await all(futs)
proc getPhotoRail*(username: string): Future[seq[GalleryPhoto]] {.async.} = proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} =
let headers = newHttpHeaders({ let headers = newHttpHeaders({
"Accept": jsonAccept, "Accept": jsonAccept,
"Referer": $(base / username), "Referer": $(base / username),
@ -222,7 +221,7 @@ proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile]
result = parseIntentProfile(html) result = parseIntentProfile(html)
proc getProfile*(username: string): Future[Profile] {.async.} = proc getProfile*(username, agent: string): Future[Profile] {.async.} =
let headers = newHttpHeaders({ let headers = newHttpHeaders({
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
"Referer": $(base / username), "Referer": $(base / username),
@ -248,7 +247,7 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
result = parsePopupProfile(html) result = parsePopupProfile(html)
proc getTweet*(username, id: string): Future[Conversation] {.async.} = proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} =
let headers = newHttpHeaders({ let headers = newHttpHeaders({
"Accept": jsonAccept, "Accept": jsonAccept,
"Referer": $base, "Referer": $base,
@ -269,13 +268,13 @@ proc getTweet*(username, id: string): Future[Conversation] {.async.} =
result = parseConversation(html) result = parseConversation(html)
let let
vidsFut = getConversationVideos(result) vidsFut = getConversationVideos(result, agent)
pollFut = getConversationPolls(result) pollFut = getConversationPolls(result, agent)
cardFut = getConversationCards(result) cardFut = getConversationCards(result, agent)
await all(vidsFut, pollFut, cardFut) await all(vidsFut, pollFut, cardFut)
proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future[Timeline] {.async.} = proc finishTimeline(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} =
if json == nil: return Timeline() if json == nil: return Timeline()
result = Timeline( result = Timeline(
@ -292,14 +291,14 @@ proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future
let let
html = parseHtml(json["items_html"].to(string)) html = parseHtml(json["items_html"].to(string))
thread = parseThread(html) thread = parseThread(html)
vidsFut = getVideos(thread) vidsFut = getVideos(thread, agent)
pollFut = getPolls(thread) pollFut = getPolls(thread, agent)
cardFut = getCards(thread) cardFut = getCards(thread, agent)
await all(vidsFut, pollFut, cardFut) await all(vidsFut, pollFut, cardFut)
result.tweets = thread.tweets result.tweets = thread.tweets
proc getTimeline*(username, after: string): Future[Timeline] {.async.} = proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
let headers = newHttpHeaders({ let headers = newHttpHeaders({
"Accept": jsonAccept, "Accept": jsonAccept,
"Referer": $(base / username), "Referer": $(base / username),
@ -320,9 +319,9 @@ proc getTimeline*(username, after: string): Future[Timeline] {.async.} =
params.add {"max_position": after} params.add {"max_position": after}
let json = await fetchJson(base / (timelineUrl % username) ? params, headers) let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
result = await finishTimeline(json, none(Query), after) result = await finishTimeline(json, none(Query), after, agent)
proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline] {.async.} = proc getTimelineSearch*(username, after, agent: string; query: Query): Future[Timeline] {.async.} =
let queryParam = genQueryParam(query) let queryParam = genQueryParam(query)
let queryEncoded = encodeUrl(queryParam, usePlus=false) let queryEncoded = encodeUrl(queryParam, usePlus=false)
@ -347,4 +346,4 @@ proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline]
} }
let json = await fetchJson(base / timelineSearchUrl ? params, headers) let json = await fetchJson(base / timelineSearchUrl ? params, headers)
result = await finishTimeline(json, some(query), after) result = await finishTimeline(json, some(query), after, agent)

6
src/cache.nim
View File

@ -12,18 +12,18 @@ var profileCacheTime = initDuration(minutes=10)
proc outdated(profile: Profile): bool = proc outdated(profile: Profile): bool =
getTime() - profile.updated > profileCacheTime getTime() - profile.updated > profileCacheTime
proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} = proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} =
withDb: withDb:
try: try:
result.getOne("username = ?", username) result.getOne("username = ?", username)
doAssert not result.outdated() doAssert not result.outdated()
except AssertionError: except AssertionError:
var profile = await getProfile(username) var profile = await getProfile(username, agent)
profile.id = result.id profile.id = result.id
result = profile result = profile
result.update() result.update()
except KeyError: except KeyError:
result = await getProfile(username) result = await getProfile(username, agent)
if result.username.len > 0: if result.username.len > 0:
result.insert() result.insert()

13
src/nitter.nim
View File

@ -3,7 +3,7 @@ from net import Port
import jester, regex import jester, regex
import api, utils, types, cache, formatters, search, config import api, utils, types, cache, formatters, search, config, agents
import views/[general, profile, status] import views/[general, profile, status]
const configPath {.strdefine.} = "./nitter.conf" const configPath {.strdefine.} = "./nitter.conf"
@ -11,15 +11,16 @@ let cfg = getConfig(configPath)
proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} = proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} =
let let
agent = getAgent()
username = name.strip(chars={'/'}) username = name.strip(chars={'/'})
profileFut = getCachedProfile(username) profileFut = getCachedProfile(username, agent)
railFut = getPhotoRail(username) railFut = getPhotoRail(username, agent)
var timelineFut: Future[Timeline] var timelineFut: Future[Timeline]
if query.isNone: if query.isNone:
timelineFut = getTimeline(username, after) timelineFut = getTimeline(username, after, agent)
else: else:
timelineFut = getTimelineSearch(username, after, get(query)) timelineFut = getTimelineSearch(username, after, agent, get(query))
let profile = await profileFut let profile = await profileFut
if profile.username.len == 0: if profile.username.len == 0:
@ -69,7 +70,7 @@ routes:
get "/@name/status/@id": get "/@name/status/@id":
cond '.' notin @"name" cond '.' notin @"name"
let conversation = await getTweet(@"name", @"id") let conversation = await getTweet(@"name", @"id", getAgent())
if conversation == nil or conversation.tweet.id.len == 0: if conversation == nil or conversation.tweet.id.len == 0:
resp Http404, showError("Tweet not found", cfg.title) resp Http404, showError("Tweet not found", cfg.title)