Use random user agents

This commit is contained in:
Zed 2019-07-31 08:36:24 +02:00
parent d0ee8e8403
commit 6b6e5b3a40
4 changed files with 143 additions and 53 deletions

90
src/agents.nim Normal file
View File

@ -0,0 +1,90 @@
import random
const userAgents = [
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0",
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0",
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/38.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 9.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0",
"Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/38.0",
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/40.1",
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/43.0",
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/50.0",
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
"Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
"Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
"Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16"
]
proc getAgent*(): string = sample(userAgents)

View File

@ -4,7 +4,6 @@ import sequtils, strutils, json, xmltree, uri
import types, parser, parserutils, formatters, search
const
agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
lang = "en-US,en;q=0.9"
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
@ -61,7 +60,7 @@ proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
except:
return nil
proc getGuestToken(force=false): Future[string] {.async.} =
proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
if getTime() - tokenUpdated < tokenLifetime and
not force and tokenUses < tokenMaxUses:
return guestToken
@ -85,7 +84,7 @@ proc getGuestToken(force=false): Future[string] {.async.} =
result = json["guest_token"].to(string)
guestToken = result
proc getVideo*(tweet: Tweet; token: string) {.async.} =
proc getVideo*(tweet: Tweet; token, agent: string) {.async.} =
if tweet.video.isNone(): return
let headers = newHttpHeaders({
@ -102,8 +101,8 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
if json == nil:
if getTime() - tokenUpdated > initDuration(seconds=1):
tokenUpdated = getTime()
discard await getGuestToken(force=true)
await getVideo(tweet, guestToken)
discard await getGuestToken(agent, force=true)
await getVideo(tweet, guestToken, agent)
return
if tweet.card.isNone:
@ -113,31 +112,31 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
tweet.video = none(Video)
tokenUses.inc
proc getVideos*(thread: Thread; token="") {.async.} =
proc getVideos*(thread: Thread; agent: string; token="") {.async.} =
if thread == nil: return
var gToken = token
if gToken.len == 0:
gToken = await getGuestToken()
gToken = await getGuestToken(agent)
var videoFuts: seq[Future[void]]
for tweet in thread.tweets.filterIt(it.video.isSome):
videoFuts.add getVideo(tweet, gToken)
videoFuts.add getVideo(tweet, gToken, agent)
await all(videoFuts)
proc getConversationVideos*(convo: Conversation) {.async.} =
var token = await getGuestToken()
proc getConversationVideos*(convo: Conversation; agent: string) {.async.} =
var token = await getGuestToken(agent)
var futs: seq[Future[void]]
futs.add getVideo(convo.tweet, token)
futs.add convo.replies.mapIt(getVideos(it, token))
futs.add getVideos(convo.before, token)
futs.add getVideos(convo.after, token)
futs.add getVideo(convo.tweet, token, agent)
futs.add convo.replies.mapIt(getVideos(it, token, agent))
futs.add getVideos(convo.before, token, agent)
futs.add getVideos(convo.after, token, agent)
await all(futs)
proc getPoll*(tweet: Tweet) {.async.} =
proc getPoll*(tweet: Tweet; agent: string) {.async.} =
if tweet.poll.isNone(): return
let headers = newHttpHeaders({
@ -154,20 +153,20 @@ proc getPoll*(tweet: Tweet) {.async.} =
tweet.poll = some(parsePoll(html))
proc getPolls*(thread: Thread) {.async.} =
proc getPolls*(thread: Thread; agent: string) {.async.} =
if thread == nil: return
var polls = thread.tweets.filterIt(it.poll.isSome)
await all(polls.map(getPoll))
await all(polls.mapIt(getPoll(it, agent)))
proc getConversationPolls*(convo: Conversation) {.async.} =
proc getConversationPolls*(convo: Conversation; agent: string) {.async.} =
var futs: seq[Future[void]]
futs.add getPoll(convo.tweet)
futs.add getPolls(convo.before)
futs.add getPolls(convo.after)
futs.add convo.replies.map(getPolls)
futs.add getPoll(convo.tweet, agent)
futs.add getPolls(convo.before, agent)
futs.add getPolls(convo.after, agent)
futs.add convo.replies.mapIt(getPolls(it, agent))
await all(futs)
proc getCard*(tweet: Tweet) {.async.} =
proc getCard*(tweet: Tweet; agent: string) {.async.} =
if tweet.card.isNone(): return
let headers = newHttpHeaders({
@ -184,20 +183,20 @@ proc getCard*(tweet: Tweet) {.async.} =
parseCard(get(tweet.card), html)
proc getCards*(thread: Thread) {.async.} =
proc getCards*(thread: Thread; agent: string) {.async.} =
if thread == nil: return
var cards = thread.tweets.filterIt(it.card.isSome)
await all(cards.map(getCard))
await all(cards.mapIt(getCard(it, agent)))
proc getConversationCards*(convo: Conversation) {.async.} =
proc getConversationCards*(convo: Conversation; agent: string) {.async.} =
var futs: seq[Future[void]]
futs.add getCard(convo.tweet)
futs.add getCards(convo.before)
futs.add getCards(convo.after)
futs.add convo.replies.map(getCards)
futs.add getCard(convo.tweet, agent)
futs.add getCards(convo.before, agent)
futs.add getCards(convo.after, agent)
futs.add convo.replies.mapIt(getCards(it, agent))
await all(futs)
proc getPhotoRail*(username: string): Future[seq[GalleryPhoto]] {.async.} =
proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} =
let headers = newHttpHeaders({
"Accept": jsonAccept,
"Referer": $(base / username),
@ -222,7 +221,7 @@ proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile]
result = parseIntentProfile(html)
proc getProfile*(username: string): Future[Profile] {.async.} =
proc getProfile*(username, agent: string): Future[Profile] {.async.} =
let headers = newHttpHeaders({
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
"Referer": $(base / username),
@ -248,7 +247,7 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
result = parsePopupProfile(html)
proc getTweet*(username, id: string): Future[Conversation] {.async.} =
proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} =
let headers = newHttpHeaders({
"Accept": jsonAccept,
"Referer": $base,
@ -269,13 +268,13 @@ proc getTweet*(username, id: string): Future[Conversation] {.async.} =
result = parseConversation(html)
let
vidsFut = getConversationVideos(result)
pollFut = getConversationPolls(result)
cardFut = getConversationCards(result)
vidsFut = getConversationVideos(result, agent)
pollFut = getConversationPolls(result, agent)
cardFut = getConversationCards(result, agent)
await all(vidsFut, pollFut, cardFut)
proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future[Timeline] {.async.} =
proc finishTimeline(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} =
if json == nil: return Timeline()
result = Timeline(
@ -292,14 +291,14 @@ proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future
let
html = parseHtml(json["items_html"].to(string))
thread = parseThread(html)
vidsFut = getVideos(thread)
pollFut = getPolls(thread)
cardFut = getCards(thread)
vidsFut = getVideos(thread, agent)
pollFut = getPolls(thread, agent)
cardFut = getCards(thread, agent)
await all(vidsFut, pollFut, cardFut)
result.tweets = thread.tweets
proc getTimeline*(username, after: string): Future[Timeline] {.async.} =
proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
let headers = newHttpHeaders({
"Accept": jsonAccept,
"Referer": $(base / username),
@ -320,9 +319,9 @@ proc getTimeline*(username, after: string): Future[Timeline] {.async.} =
params.add {"max_position": after}
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
result = await finishTimeline(json, none(Query), after)
result = await finishTimeline(json, none(Query), after, agent)
proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline] {.async.} =
proc getTimelineSearch*(username, after, agent: string; query: Query): Future[Timeline] {.async.} =
let queryParam = genQueryParam(query)
let queryEncoded = encodeUrl(queryParam, usePlus=false)
@ -347,4 +346,4 @@ proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline]
}
let json = await fetchJson(base / timelineSearchUrl ? params, headers)
result = await finishTimeline(json, some(query), after)
result = await finishTimeline(json, some(query), after, agent)

View File

@ -12,18 +12,18 @@ var profileCacheTime = initDuration(minutes=10)
proc outdated(profile: Profile): bool =
getTime() - profile.updated > profileCacheTime
proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} =
proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} =
withDb:
try:
result.getOne("username = ?", username)
doAssert not result.outdated()
except AssertionError:
var profile = await getProfile(username)
var profile = await getProfile(username, agent)
profile.id = result.id
result = profile
result.update()
except KeyError:
result = await getProfile(username)
result = await getProfile(username, agent)
if result.username.len > 0:
result.insert()

View File

@ -3,7 +3,7 @@ from net import Port
import jester, regex
import api, utils, types, cache, formatters, search, config
import api, utils, types, cache, formatters, search, config, agents
import views/[general, profile, status]
const configPath {.strdefine.} = "./nitter.conf"
@ -11,15 +11,16 @@ let cfg = getConfig(configPath)
proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} =
let
agent = getAgent()
username = name.strip(chars={'/'})
profileFut = getCachedProfile(username)
railFut = getPhotoRail(username)
profileFut = getCachedProfile(username, agent)
railFut = getPhotoRail(username, agent)
var timelineFut: Future[Timeline]
if query.isNone:
timelineFut = getTimeline(username, after)
timelineFut = getTimeline(username, after, agent)
else:
timelineFut = getTimelineSearch(username, after, get(query))
timelineFut = getTimelineSearch(username, after, agent, get(query))
let profile = await profileFut
if profile.username.len == 0:
@ -69,7 +70,7 @@ routes:
get "/@name/status/@id":
cond '.' notin @"name"
let conversation = await getTweet(@"name", @"id")
let conversation = await getTweet(@"name", @"id", getAgent())
if conversation == nil or conversation.tweet.id.len == 0:
resp Http404, showError("Tweet not found", cfg.title)