parent
a7249080db
commit
13a9f6cd1f
|
@ -26,7 +26,7 @@ is on implementing missing features.
|
|||
## Todo (roughly in this order)
|
||||
|
||||
- Search (images/videos, hashtags, etc.)
|
||||
- Hiding retweets, showing replies, etc.
|
||||
- Custom timeline filter
|
||||
- Media carousel below profile
|
||||
- Media-only/gallery view
|
||||
- Nitter link previews
|
||||
|
|
|
@ -497,6 +497,39 @@ video {
|
|||
word-wrap: break-word;
|
||||
}
|
||||
|
||||
.tab {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
list-style: none;
|
||||
margin: 0 0 5px 0;
|
||||
background-color: #161616;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.tab .tab-item {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.tab-item {
|
||||
flex: 1 1 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.tab .tab-item a.active, .tab .tab-item.active a {
|
||||
border-bottom-color: #ff6c60;
|
||||
color: #ff6c60;
|
||||
}
|
||||
|
||||
.tab .tab-item a {
|
||||
border-bottom: .1rem solid transparent;
|
||||
color: inherit;
|
||||
display: block;
|
||||
padding: 8px 0;
|
||||
text-decoration: none;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.conversation {
|
||||
max-width: 580px;
|
||||
margin: 0 auto;
|
||||
|
|
BIN
screenshot.png
BIN
screenshot.png
Binary file not shown.
Before Width: | Height: | Size: 375 KiB After Width: | Height: | Size: 380 KiB |
124
src/api.nim
124
src/api.nim
|
@ -1,21 +1,20 @@
|
|||
import httpclient, asyncdispatch, htmlparser, times
|
||||
import sequtils, strutils, strformat, json, xmltree, uri
|
||||
import regex
|
||||
import sequtils, strutils, json, xmltree, uri
|
||||
|
||||
import ./types, ./parser, ./parserutils, ./formatters
|
||||
import types, parser, parserutils, formatters, search
|
||||
|
||||
const
|
||||
agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
|
||||
lang = "en-US,en;q=0.9"
|
||||
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
|
||||
cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
|
||||
jsonAccept = "application/json, text/javascript, */*; q=0.01"
|
||||
|
||||
base = parseUri("https://twitter.com/")
|
||||
apiBase = parseUri("https://api.twitter.com/1.1/")
|
||||
|
||||
timelineParams = "?include_available_features=1&include_entities=1&include_new_items_bar=false&reset_error_state=false"
|
||||
showUrl = "i/profiles/show/$1" & timelineParams
|
||||
timelineUrl = showUrl % "$1/timeline/tweets"
|
||||
timelineUrl = "i/profiles/show/$1/timeline/tweets"
|
||||
timelineSearchUrl = "i/search/timeline"
|
||||
profilePopupUrl = "i/profiles/popup"
|
||||
profileIntentUrl = "intent/user"
|
||||
tweetUrl = "status"
|
||||
|
@ -70,7 +69,7 @@ proc getGuestToken(force=false): Future[string] {.async.} =
|
|||
tokenUses = 0
|
||||
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": "application/json, text/javascript, */*; q=0.01",
|
||||
"Accept": jsonAccept,
|
||||
"Referer": $base,
|
||||
"User-Agent": agent,
|
||||
"Authorization": auth
|
||||
|
@ -89,7 +88,7 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
|
|||
if tweet.video.isNone(): return
|
||||
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": "application/json, text/javascript, */*; q=0.01",
|
||||
"Accept": jsonAccept,
|
||||
"Referer": $(base / getLink(tweet)),
|
||||
"User-Agent": agent,
|
||||
"Authorization": auth,
|
||||
|
@ -196,45 +195,9 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
|
|||
|
||||
result = parsePopupProfile(html)
|
||||
|
||||
proc getTimeline*(username: string; after=""): Future[Timeline] {.async.} =
|
||||
proc getTweet*(username, id: string): Future[Conversation] {.async.} =
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": "application/json, text/javascript, */*; q=0.01",
|
||||
"Referer": $(base / username),
|
||||
"User-Agent": agent,
|
||||
"X-Twitter-Active-User": "yes",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Accept-Language": lang
|
||||
})
|
||||
|
||||
var url = timelineUrl % username
|
||||
let cleanAfter = after.replace(re"[^\d]*(\d+)[^\d]*", "$1")
|
||||
if cleanAfter.len > 0:
|
||||
url &= "&max_position=" & cleanAfter
|
||||
|
||||
let json = await fetchJson(base / url, headers)
|
||||
if json == nil: return Timeline()
|
||||
|
||||
result = Timeline(
|
||||
hasMore: json["has_more_items"].to(bool),
|
||||
maxId: json.getOrDefault("max_position").getStr(""),
|
||||
minId: json.getOrDefault("min_position").getStr(""),
|
||||
)
|
||||
|
||||
if json["new_latent_count"].to(int) == 0: return
|
||||
if not json.hasKey("items_html"): return
|
||||
|
||||
let
|
||||
html = parseHtml(json["items_html"].to(string))
|
||||
thread = parseThread(html)
|
||||
vidsFut = getVideos(thread)
|
||||
pollFut = getPolls(thread)
|
||||
|
||||
await all(vidsFut, pollFut)
|
||||
result.tweets = thread.tweets
|
||||
|
||||
proc getTweet*(username: string; id: string): Future[Conversation] {.async.} =
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": "application/json, text/javascript, */*; q=0.01",
|
||||
"Accept": jsonAccept,
|
||||
"Referer": $base,
|
||||
"User-Agent": agent,
|
||||
"X-Twitter-Active-User": "yes",
|
||||
|
@ -255,3 +218,72 @@ proc getTweet*(username: string; id: string): Future[Conversation] {.async.} =
|
|||
let vidsFut = getConversationVideos(result)
|
||||
let pollFut = getConversationPolls(result)
|
||||
await all(vidsFut, pollFut)
|
||||
|
||||
proc finishTimeline(json: JsonNode; query: Option[Query]): Future[Timeline] {.async.} =
|
||||
if json == nil: return Timeline()
|
||||
|
||||
result = Timeline(
|
||||
hasMore: json["has_more_items"].to(bool),
|
||||
maxId: json.getOrDefault("max_position").getStr(""),
|
||||
minId: json.getOrDefault("min_position").getStr("").cleanPos(),
|
||||
)
|
||||
|
||||
if json["new_latent_count"].to(int) == 0: return
|
||||
if not json.hasKey("items_html"): return
|
||||
|
||||
let
|
||||
html = parseHtml(json["items_html"].to(string))
|
||||
thread = parseThread(html)
|
||||
vidsFut = getVideos(thread)
|
||||
pollFut = getPolls(thread)
|
||||
|
||||
await all(vidsFut, pollFut)
|
||||
result.tweets = thread.tweets
|
||||
result.query = query
|
||||
|
||||
proc getTimeline*(username, after: string): Future[Timeline] {.async.} =
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": jsonAccept,
|
||||
"Referer": $(base / username),
|
||||
"User-Agent": agent,
|
||||
"X-Twitter-Active-User": "yes",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Accept-Language": lang
|
||||
})
|
||||
|
||||
var params = toSeq({
|
||||
"include_available_features": "1",
|
||||
"include_entities": "1",
|
||||
"include_new_items_bar": "false",
|
||||
"reset_error_state": "false"
|
||||
})
|
||||
|
||||
if after.len > 0:
|
||||
params.add {"max_position": after}
|
||||
|
||||
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
|
||||
result = await finishTimeline(json, none(Query))
|
||||
|
||||
proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline] {.async.} =
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": jsonAccept,
|
||||
"Referer": $(base / ("search?f=tweets&q=from%3A$1&src=typd" % username)),
|
||||
"User-Agent": agent,
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Authority": "twitter.com",
|
||||
"Accept-Language": lang
|
||||
})
|
||||
|
||||
let params = {
|
||||
"f": "tweets",
|
||||
"vertical": "default",
|
||||
"q": genQueryParam(query),
|
||||
"src": "typd",
|
||||
"include_available_features": "1",
|
||||
"include_entities": "1",
|
||||
"max_position": if after.len > 0: genPos(after) else: "0",
|
||||
"reset_error_state": "false"
|
||||
}
|
||||
|
||||
let json = await fetchJson(base / timelineSearchUrl ? params, headers)
|
||||
result = await finishTimeline(json, some(query))
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import strutils, strformat, htmlgen, xmltree, times
|
||||
import regex
|
||||
|
||||
import ./types, ./utils
|
||||
import types, utils
|
||||
|
||||
from unicode import Rune, `$`
|
||||
|
||||
|
|
|
@ -1,26 +1,36 @@
|
|||
import asyncdispatch, asyncfile, httpclient, strutils, strformat, uri, os
|
||||
import jester
|
||||
import jester, regex
|
||||
|
||||
import api, utils, types, cache, formatters
|
||||
import api, utils, types, cache, formatters, search
|
||||
|
||||
include views/"user.nimf"
|
||||
include views/"general.nimf"
|
||||
|
||||
const cacheDir {.strdefine.} = "/tmp/nitter"
|
||||
|
||||
proc showTimeline(name: string; num=""): Future[string] {.async.} =
|
||||
proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} =
|
||||
let
|
||||
username = name.strip(chars={'/'})
|
||||
profileFut = getCachedProfile(username)
|
||||
tweetsFut = getTimeline(username, after=num)
|
||||
|
||||
var timelineFut: Future[Timeline]
|
||||
if query.isNone:
|
||||
timelineFut = getTimeline(username, after)
|
||||
else:
|
||||
timelineFut = getTimelineSearch(username, after, get(query))
|
||||
|
||||
let profile = await profileFut
|
||||
if profile.username.len == 0:
|
||||
return ""
|
||||
|
||||
let profileHtml = renderProfile(profile, await tweetsFut, num.len == 0)
|
||||
let profileHtml = renderProfile(profile, await timelineFut, after.len == 0)
|
||||
return renderMain(profileHtml, title=pageTitle(profile))
|
||||
|
||||
template respTimeline(timeline: typed) =
|
||||
if timeline.len == 0:
|
||||
resp Http404, showError("User \"" & @"name" & "\" not found")
|
||||
resp timeline
|
||||
|
||||
routes:
|
||||
get "/":
|
||||
resp renderMain(renderSearchPanel(), title=pageTitle("Search"))
|
||||
|
@ -28,17 +38,24 @@ routes:
|
|||
post "/search":
|
||||
if @"query".len == 0:
|
||||
resp Http404, showError("Please enter a username.")
|
||||
|
||||
redirect("/" & @"query")
|
||||
|
||||
get "/@name/?":
|
||||
cond '.' notin @"name"
|
||||
respTimeline(await showTimeline(@"name", @"after", none(Query)))
|
||||
|
||||
let timeline = await showTimeline(@"name", @"after")
|
||||
if timeline.len == 0:
|
||||
resp Http404, showError("User \"" & @"name" & "\" not found")
|
||||
get "/@name/search/?":
|
||||
cond '.' notin @"name"
|
||||
let query = initQuery(@"filter", @"sep", @"name")
|
||||
respTimeline(await showTimeline(@"name", @"after", some(query)))
|
||||
|
||||
resp timeline
|
||||
get "/@name/replies":
|
||||
cond '.' notin @"name"
|
||||
respTimeline(await showTimeline(@"name", @"after", some(getReplyQuery(@"name"))))
|
||||
|
||||
get "/@name/media":
|
||||
cond '.' notin @"name"
|
||||
respTimeline(await showTimeline(@"name", @"after", some(getMediaQuery(@"name"))))
|
||||
|
||||
get "/@name/status/@id":
|
||||
cond '.' notin @"name"
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import xmltree, sequtils, strtabs, strutils, strformat, json
|
||||
|
||||
import ./types, ./parserutils, ./formatters
|
||||
import types, parserutils, formatters
|
||||
|
||||
proc parsePopupProfile*(node: XmlNode): Profile =
|
||||
let profile = node.select(".profile-card")
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import xmltree, htmlparser, strtabs, strformat, times
|
||||
import regex
|
||||
|
||||
import ./types, ./formatters, ./api
|
||||
import types, formatters, api
|
||||
|
||||
from q import nil
|
||||
|
||||
|
|
|
@ -0,0 +1,80 @@
|
|||
import asyncdispatch, strutils, strformat, uri, tables
|
||||
|
||||
import types
|
||||
|
||||
const
|
||||
separators = @["AND", "OR"]
|
||||
validFilters = @[
|
||||
"media", "images", "videos", "native_video", "twimg",
|
||||
"links", "quote", "replies", "mentions",
|
||||
"news", "verified", "safe"
|
||||
]
|
||||
|
||||
# Experimental, this might break in the future
|
||||
# Till then, it results in shorter urls
|
||||
const
|
||||
posPrefix = "thGAVUV0VFVBa"
|
||||
posSuffix = "EjUAFQAlAFUAFQAA"
|
||||
|
||||
proc initQuery*(filter, separator: string; name=""): Query =
|
||||
var sep = separator.strip().toUpper()
|
||||
Query(
|
||||
filter: filter.split(",").filterIt(it in validFilters),
|
||||
sep: if sep in separators: sep else: "AND",
|
||||
fromUser: name,
|
||||
queryType: custom
|
||||
)
|
||||
|
||||
proc getMediaQuery*(name: string): Query =
|
||||
Query(
|
||||
filter: @["twimg", "native_video"],
|
||||
sep: "OR",
|
||||
fromUser: name,
|
||||
queryType: media
|
||||
)
|
||||
|
||||
proc getReplyQuery*(name: string): Query =
|
||||
Query(fromUser: name, queryType: replies)
|
||||
|
||||
proc genQueryParam*(query: Query): string =
|
||||
var filters: seq[string]
|
||||
var param: string
|
||||
|
||||
if query.fromUser.len > 0:
|
||||
param = &"from:{query.fromUser} "
|
||||
|
||||
for f in query.filter:
|
||||
filters.add "filter:" & f
|
||||
for e in query.exclude:
|
||||
filters.add "-filter:" & e
|
||||
|
||||
return strip(param & filters.join(&" {query.sep} "))
|
||||
|
||||
proc genQueryUrl*(query: Query): string =
|
||||
result = &"/{query.queryType}?"
|
||||
if query.queryType != custom: return
|
||||
|
||||
var params: seq[string]
|
||||
if query.filter.len > 0:
|
||||
params &= "filter=" & query.filter.join(",")
|
||||
if query.exclude.len > 0:
|
||||
params &= "not=" & query.exclude.join(",")
|
||||
if query.sep.len > 0:
|
||||
params &= "sep=" & query.sep
|
||||
if params.len > 0:
|
||||
result &= params.join("&") & "&"
|
||||
|
||||
proc cleanPos*(pos: string): string =
|
||||
pos.multiReplace((posPrefix, ""), (posSuffix, ""))
|
||||
|
||||
proc genPos*(pos: string): string =
|
||||
posPrefix & pos & posSuffix
|
||||
|
||||
proc tabClass*(timeline: Timeline; tab: string): string =
|
||||
result = '"' & "tab-item"
|
||||
if timeline.query.isNone:
|
||||
if tab == "tweets":
|
||||
result &= " active"
|
||||
elif $timeline.query.get().queryType == tab:
|
||||
result &= " active"
|
||||
result &= '"'
|
|
@ -31,6 +31,16 @@ db("cache.db", "", "", ""):
|
|||
.}: Time
|
||||
|
||||
type
|
||||
QueryType* = enum
|
||||
replies, media, custom = "search"
|
||||
|
||||
Query* = object
|
||||
filter*: seq[string]
|
||||
exclude*: seq[string]
|
||||
sep*: string
|
||||
fromUser*: string
|
||||
queryType*: QueryType
|
||||
|
||||
VideoType* = enum
|
||||
vmap, m3u8, mp4
|
||||
|
||||
|
@ -106,6 +116,7 @@ type
|
|||
minId*: string
|
||||
maxId*: string
|
||||
hasMore*: bool
|
||||
query*: Option[Query]
|
||||
|
||||
proc contains*(thread: Thread; tweet: Tweet): bool =
|
||||
thread.tweets.anyIt(it.id == tweet.id)
|
||||
|
|
|
@ -45,5 +45,5 @@
|
|||
#end proc
|
||||
#
|
||||
#proc showError*(error: string): string =
|
||||
${renderMain(renderError(error), title="Error | Nitter")}
|
||||
#renderMain(renderError(error), title="Error | Nitter")
|
||||
#end proc
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
#? stdtmpl(subsChar = '$', metaChar = '#')
|
||||
#import xmltree, strutils, uri
|
||||
#import ../types, ../formatters, ../utils
|
||||
#import ../types, ../formatters, ../utils, ../search
|
||||
#include "tweet.nimf"
|
||||
#
|
||||
#proc renderProfileCard*(profile: Profile): string =
|
||||
|
@ -52,10 +52,13 @@
|
|||
#
|
||||
#proc renderTimeline*(timeline: Timeline; profile: Profile; beginning: bool): string =
|
||||
#var retweets: seq[string]
|
||||
#var query = "?"
|
||||
#if timeline.query.isSome: query = genQueryUrl(get(timeline.query))
|
||||
#end if
|
||||
<div id="tweets">
|
||||
#if not beginning:
|
||||
<div class="show-more status-el">
|
||||
<a href="/${profile.username}">Load newest tweets</a>
|
||||
<a href="/${profile.username}${query.strip(chars={'?'})}">Load newest tweets</a>
|
||||
</div>
|
||||
#end if
|
||||
#
|
||||
|
@ -66,9 +69,9 @@
|
|||
${renderTweet(tweet, "timeline-tweet")}
|
||||
#end for
|
||||
#
|
||||
#if timeline.hasMore:
|
||||
#if timeline.hasMore or timeline.query.isSome and timeline.tweets.len > 0:
|
||||
<div class="show-more">
|
||||
<a href="/${profile.username}?after=${timeline.minId}">Load older tweets</a>
|
||||
<a href="/${profile.username}${query}after=${timeline.minId}">Load older tweets</a>
|
||||
</div>
|
||||
#elif timeline.tweets.len > 0:
|
||||
<div class="timeline-footer">
|
||||
|
@ -96,6 +99,13 @@
|
|||
${renderProfileCard(profile)}
|
||||
</div>
|
||||
<div class="timeline-tab">
|
||||
#let link = "/" & profile.username
|
||||
<ul class="tab">
|
||||
<li class=${timeline.tabClass("tweets")}><a href="${link}">Tweets</a></li>
|
||||
<li class=${timeline.tabClass("replies")}><a href="${link}/replies">Tweets & Replies</a></li>
|
||||
<li class=${timeline.tabClass("media")}><a href="${link}/media">Media</a></li>
|
||||
#discard "<li class=tab-item><a href=${link}/search>Custom</a></li>"
|
||||
</ul>
|
||||
${renderTimeline(timeline, profile, beginning)}
|
||||
</div>
|
||||
</div>
|
||||
|
|
Loading…
Reference in New Issue