Optional base64 support for proxy urls

This commit is contained in:
Zed 2020-06-09 15:04:38 +02:00
parent 1b9fa40237
commit 310c5e936d
7 changed files with 61 additions and 39 deletions

View File

@ -20,6 +20,7 @@ redisMaxConnections = 30
[Config]
hmacKey = "secretkey" # random key for cryptographic signing of video urls
base64Media = false # use base64 encoding for proxied media urls
tokenCount = 10
# minimum amount of usable tokens. tokens are used to authorize API requests,
# but they expire after ~1 hour, and have a limit of 187 requests.

View File

@ -20,6 +20,10 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
hostname: cfg.get("Server", "hostname", "nitter.net"),
staticDir: cfg.get("Server", "staticDir", "./public"),
hmacKey: cfg.get("Config", "hmacKey", "secretkey"),
base64Media: cfg.get("Config", "base64Media", false),
minTokens: cfg.get("Config", "tokenCount", 10),
cacheDir: cfg.get("Cache", "directory", "/tmp/nitter"),
listCacheTime: cfg.get("Cache", "listMinutes", 120),
rssCacheTime: cfg.get("Cache", "rssMinutes", 10),
@ -27,10 +31,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
redisHost: cfg.get("Cache", "redisHost", "localhost"),
redisPort: cfg.get("Cache", "redisPort", 6379),
redisConns: cfg.get("Cache", "redisConnections", 20),
redisMaxConns: cfg.get("Cache", "redisMaxConnections", 30),
hmacKey: cfg.get("Config", "hmacKey", "secretkey"),
minTokens: cfg.get("Config", "tokenCount", 10),
redisMaxConns: cfg.get("Cache", "redisMaxConnections", 30)
)
return (conf, cfg)

View File

@ -15,6 +15,7 @@ const
nbsp = $Rune(0x000A0)
wwwRegex = re"https?://(www[0-9]?\.)?"
m3u8Regex = re"""url="(.+.m3u8)""""
manifestRegex = re"(.+(.ts|.m3u8|.vmap))"
userpicRegex = re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$"
extRegex = re"(\.[A-z]+)$"
@ -52,6 +53,11 @@ proc replaceUrl*(url: string; prefs: Prefs; absolute=""): string =
if absolute.len > 0:
result = result.replace("href=\"/", "href=\"https://" & absolute & "/")
proc getM3u8Url*(content: string): string =
var m: RegexMatch
if content.find(m3u8Regex, m):
result = content[m.group(0)[0]]
proc proxifyVideo*(manifest: string; proxy: bool): string =
proc cb(m: RegexMatch; s: string): string =
result = "https://video.twimg.com" & s[m.group(0)[0]]

View File

@ -23,6 +23,7 @@ stdout.flushFile
updateDefaultPrefs(fullCfg)
setCacheTimes(cfg)
setHmacKey(cfg.hmacKey)
setProxyEncoding(cfg.base64Media)
waitFor initRedisPool(cfg)
stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"

View File

@ -1,17 +1,16 @@
import uri, strutils, httpclient, os, hashes
import uri, strutils, httpclient, os, hashes, base64, re
import asynchttpserver, asyncstreams, asyncfile, asyncnet
import jester, regex
import jester
import router_utils
import ".."/[types, formatters, agents, utils]
import ../views/general
export asynchttpserver, asyncstreams, asyncfile, asyncnet
export httpclient, os, strutils, asyncstreams, regex
export httpclient, os, strutils, asyncstreams, base64, re
const
m3u8Regex* = re"""url="(.+.m3u8)""""
m3u8Mime* = "application/vnd.apple.mpegurl"
maxAge* = "max-age=604800"
@ -60,13 +59,27 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
finally:
client.safeClose()
template check*(code): untyped =
if code != Http200:
resp code
else:
enableRawMode()
break route
proc decoded*(req: jester.Request; index: int): string =
let
based = req.matches[0].len > 1
encoded = req.matches[index]
if based: decode(encoded)
else: decodeUrl(encoded)
proc createMediaRouter*(cfg: Config) =
router media:
get "/pic/?":
resp Http404
get "/pic/@url":
var url = decodeUrl(@"url")
get re"^\/pic\/(enc)?\/?(.+)":
var url = decoded(request, 1)
if "twimg.com" notin url:
url.insert(twimg)
if not url.startsWith(https):
@ -75,42 +88,32 @@ proc createMediaRouter*(cfg: Config) =
let uri = parseUri(url)
cond isTwitterUrl(uri) == true
enableRawMode()
let code = await proxyMedia(request, $uri)
if code == Http200:
enableRawMode()
break route
else:
resp code
let code = await proxyMedia(request, url)
check code
get "/video/@sig/@url":
cond "http" in @"url"
var url = decodeUrl(@"url")
let prefs = cookiePrefs()
get re"^\/video\/(enc)?\/?(.+)\/(.+)$":
let url = decoded(request, 2)
cond "http" in url
if getHmac(url) != @"sig":
if getHmac(url) != request.matches[1]:
resp showError("Failed to verify signature", cfg)
if ".mp4" in url or ".ts" in url:
let code = await proxyMedia(request, url)
if code == Http200:
enableRawMode()
break route
else:
resp code
check code
var content: string
if ".vmap" in url:
var m: RegexMatch
content = await safeFetch(url, mediaAgent)
if content.find(m3u8Regex, m):
url = decodeUrl(content[m.group(0)[0]])
let m3u8 = getM3u8Url(await safeFetch(url, mediaAgent))
if m3u8.len > 0:
content = await safeFetch(url, mediaAgent)
else:
resp Http404
if ".m3u8" in url:
let vid = await safeFetch(url, mediaAgent)
let
vid = await safeFetch(url, mediaAgent)
prefs = cookiePrefs()
content = proxifyVideo(vid, prefs.proxyVideos)
resp content, m3u8Mime

View File

@ -201,6 +201,7 @@ type
staticDir*: string
hmacKey*: string
base64Media*: bool
minTokens*: int
cacheDir*: string

View File

@ -1,7 +1,9 @@
import strutils, strformat, sequtils, uri, tables
import strutils, strformat, sequtils, uri, tables, base64
import nimcrypto, regex
var hmacKey = "secretkey"
var
hmacKey = "secretkey"
base64Media = false
const
https* = "https://"
@ -20,17 +22,24 @@ const
proc setHmacKey*(key: string) =
hmacKey = key
proc setProxyEncoding*(state: bool) =
base64Media = state
proc getHmac*(data: string): string =
($hmac(sha256, hmacKey, data))[0 .. 12]
proc getVidUrl*(link: string): string =
if link.len == 0: return
let
sig = getHmac(link)
url = encodeUrl(link)
&"/video/{sig}/{url}"
let sig = getHmac(link)
if base64Media:
&"/video/enc/{sig}/{encode(link, safe=true)}"
else:
&"/video/{sig}/{encodeUrl(link)}"
proc getPicUrl*(link: string): string =
if base64Media:
&"/pic/enc/{encode(link, safe=true)}"
else:
&"/pic/{encodeUrl(link)}"
proc cleanFilename*(filename: string): string =