dongge
/
AndroidCatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean code

This commit is contained in:
FongMi 2023-11-21 09:12:28 +08:00
parent 9f73e574dc
commit ff4c5be051
5 changed files with 115 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/src/main/java/com/github/catvod/api/AliYun.java
View File

@ -364,7 +364,7 @@ public class AliYun {
} else if (flag.split("#")[0].equals("原畫")) { } else if (flag.split("#")[0].equals("原畫")) {
return Result.get().url(getDownloadUrl(ids[0], ids[1])).octet().subs(getSubs(ids)).header(getHeader()).string(); return Result.get().url(getDownloadUrl(ids[0], ids[1])).octet().subs(getSubs(ids)).header(getHeader()).string();
} else if (flag.split("#")[0].equals("極速")) { } else if (flag.split("#")[0].equals("極速")) {
return Result.get().url(MultiThread.url(getDownloadUrl(ids[0], ids[1]), 4)).octet().subs(getSubs(ids)).header(getHeader()).string(); return Result.get().url(MultiThread.url(getDownloadUrl(ids[0], ids[1]), 5)).octet().subs(getSubs(ids)).header(getHeader()).string();
} else { } else {
return ""; return "";
} }

19
app/src/main/java/com/github/catvod/utils/MultiThread.java
View File

@ -17,13 +17,20 @@ public class MultiThread {
public static Object[] proxy(Map<String, String> params) throws Exception { public static Object[] proxy(Map<String, String> params) throws Exception {
String url = params.get("url"); String url = params.get("url");
int thread = Integer.parseInt(params.get("thread")); int thread = Integer.parseInt(params.get("thread"));
Map<String, String> reqHeaders = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
for (String key : params.keySet()) reqHeaders.put(key, params.get(key)); for (String key : params.keySet()) headers.put(key, params.get(key));
if (reqHeaders.containsKey("do")) reqHeaders.remove("do"); MultiThreadedDownloader downloader = new MultiThreadedDownloader(url, removeHeaders(headers), thread);
if (reqHeaders.containsKey("url")) reqHeaders.remove("url");
if (reqHeaders.containsKey("thread")) reqHeaders.remove("thread");
MultiThreadedDownloader downloader = new MultiThreadedDownloader(url, reqHeaders, thread);
NanoHTTPD.Response response = downloader.start(); NanoHTTPD.Response response = downloader.start();
return new Object[]{response}; return new Object[]{response};
} }
private static Map<String, String> removeHeaders(Map<String, String> headers) {
headers.remove("do");
headers.remove("url");
headers.remove("host");
headers.remove("thread");
headers.remove("remote-addr");
headers.remove("http-client-ip");
return headers;
}
} }

212
app/src/main/java/com/github/catvod/utils/MultiThreadedDownloader.java
View File

@ -1,13 +1,15 @@
package com.github.catvod.utils; package com.github.catvod.utils;
import static java.lang.Thread.sleep;
import static fi.iki.elonen.NanoHTTPD.newFixedLengthResponse; import static fi.iki.elonen.NanoHTTPD.newFixedLengthResponse;
import android.os.SystemClock;
import com.github.catvod.net.OkHttp; import com.github.catvod.net.OkHttp;
import java.io.PipedInputStream; import java.io.PipedInputStream;
import java.io.PipedOutputStream; import java.io.PipedOutputStream;
import java.util.HashMap; import java.util.HashMap;
import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
@ -20,71 +22,76 @@ import java.util.regex.Pattern;
import fi.iki.elonen.NanoHTTPD; import fi.iki.elonen.NanoHTTPD;
import okhttp3.Response; import okhttp3.Response;
// 多线程内存下载器
public class MultiThreadedDownloader { public class MultiThreadedDownloader {
private String url; // 资源URL //已开始下载的 chunk 队列
private Map<String, String> headers; // HTTP Headers. 主要关注`Range-Bytes`这个字段. private final BlockingQueue<Chunk> readyChunkQueue;
private int chunkSize = 1024 * 128; // 每个线程每轮下载的字节数. private final Map<String, String> headers;
private int maxBufferedChunk = 1024; // 最多缓存多少个未被取走的chunk. 避免内存溢出. 过多chunk未被取走时，下载线程可以暂时休息等待. private final String url;
private int numThreads; // 线程数. private final Lock lock;
private int timeout = 10; // 读超时（秒） //最多缓存多少个未被取走的chunk
private boolean running = false; // 多线程是否在运行中. private final int maxBufferedChunk = 1024;
private long startOffset = -1; // 下载起始的偏移量. //线程数
private long endOffset = -1; // 下载结束的偏移量. private final int numThreads;
private long currentOffset = -1; // 当前读取的偏移量. //读超时
private long nextChunkStartOffset = -1; // 下一个chunk的起始偏移量. private final int timeout = 10;
private BlockingQueue<Chunk> readyChunkQueue = new LinkedBlockingQueue<>(); // 已开始下载的chunk队列（有序）. //每个线程每轮下载的字节数
private Lock lock = new ReentrantLock(); // 锁. private int chunkSize = 1024 * 64;
//下载起始的偏移量
private long startOffset = -1;
//下载结束的偏移量
private long endOffset = -1;
//当前读取的偏移量
private long currentOffset = -1;
//下一个 chunk 的起始偏移量
private long nextChunkStartOffset = -1;
//多线程是否在运行中
private boolean running;
public MultiThreadedDownloader(String url, Map<String, String> headers, int numThreads) { public MultiThreadedDownloader(String url, Map<String, String> headers, int numThreads) {
this.url = url; this.url = url;
this.headers = headers; this.headers = headers;
this.numThreads = numThreads; this.numThreads = numThreads;
this.lock = new ReentrantLock();
this.readyChunkQueue = new LinkedBlockingQueue<>();
} }
// 开始下载 //开始下载
public NanoHTTPD.Response start() throws Exception { public NanoHTTPD.Response start() throws Exception {
// 确保请求header包含Range //确保请求header包含Range
NanoHTTPD.Response.Status status = NanoHTTPD.Response.Status.PARTIAL_CONTENT; NanoHTTPD.Response.Status status = NanoHTTPD.Response.Status.PARTIAL_CONTENT;
if (!this.headers.containsKey("Range")) { if (!headers.containsKey("Range")) {
this.headers.put("Range", "bytes=0-"); headers.put("Range", "bytes=0-");
status = NanoHTTPD.Response.Status.OK; status = NanoHTTPD.Response.Status.OK;
} }
// 尝试从Range判断下载的起始偏移量和结束偏移量 //尝试从Range判断下载的起始偏移量和结束偏移量
Matcher matcher = Pattern.compile("bytes=(\\d+)-(\\d+)?").matcher(this.headers.get("Range")); Matcher matcher = Pattern.compile("bytes=(\\d+)-(\\d+)?").matcher(headers.get("Range"));
if (matcher.find()) { if (matcher.find()) {
this.startOffset = Long.parseLong(matcher.group(1)); startOffset = Long.parseLong(matcher.group(1));
if (matcher.group(2) != null) { if (matcher.group(2) != null) {
this.endOffset = Long.parseLong(matcher.group(2)); endOffset = Long.parseLong(matcher.group(2));
} }
} else { } else {
throw new Exception("invalid Range: " + this.headers.get("Range")); throw new Exception("invalid Range: " + headers.get("Range"));
} }
this.nextChunkStartOffset = this.startOffset; nextChunkStartOffset = startOffset;
this.currentOffset = this.startOffset; currentOffset = startOffset;
// 建立连接 //建立连接
Map<String, String> mHeaders = new HashMap<>(); Response response = OkHttp.newCall(getDownloadUrl(), headers);
for (Map.Entry<String, String> entry : this.headers.entrySet()) {
if (!shouldFilterRequestHeaderKey(entry.getKey())) {
mHeaders.put(entry.getKey(), entry.getValue());
}
}
Response response = OkHttp.newCall(getDownloadUrl(), mHeaders);
// 尽早关闭连接，我们不需要body的数据 //尽早关闭连接，我们不需要body的数据
response.body().close(); if (response.body() != null) response.body().close();
// 检查状态码 //检查状态码
int responseCode = response.code(); int responseCode = response.code();
if (responseCode < 200 || responseCode >= 300) { if (responseCode < 200 || responseCode >= 300) {
throw new Exception("response code: " + responseCode); throw new Exception("response code: " + responseCode);
} }
// 获取header //获取header
String contentType = response.headers().get("Content-Type"); String contentType = response.headers().get("Content-Type");
String contentDisposition = response.headers().get("Content-Disposition"); String contentDisposition = response.headers().get("Content-Disposition");
if (contentDisposition != null) { if (contentDisposition != null) {
@ -118,13 +125,13 @@ public class MultiThreadedDownloader {
} }
long contentLength = Long.parseLong(hContentLength); long contentLength = Long.parseLong(hContentLength);
// 尝试从Content-Range获取下载结束的偏移量 //尝试从Content-Range获取下载结束的偏移量
if (this.endOffset <= 0) { if (endOffset <= 0) {
String hContentRange = response.headers().get("Content-Range"); String hContentRange = response.headers().get("Content-Range");
if (hContentRange != null) { if (hContentRange != null) {
matcher = Pattern.compile(".*/(\\d+)").matcher(hContentRange); matcher = Pattern.compile(".*/(\\d+)").matcher(hContentRange);
if (matcher.find()) { if (matcher.find()) {
this.endOffset = Long.parseLong(matcher.group(1)) - 1; endOffset = Long.parseLong(matcher.group(1)) - 1;
} else { } else {
throw new Exception("invalid `Content-Range`: " + hContentRange); throw new Exception("invalid `Content-Range`: " + hContentRange);
} }
@ -133,20 +140,20 @@ public class MultiThreadedDownloader {
} }
} }
// 如果下载的内容过小，那么减少chunkSize，使得每个线程刚好只需要下载一轮 //如果下载的内容过小，那么减少chunkSize，使得每个线程刚好只需要下载一轮
long downloadSize = this.endOffset - this.startOffset + 1; long downloadSize = endOffset - startOffset + 1;
long onetimeChunkSize = downloadSize / this.numThreads + 1; long onetimeChunkSize = downloadSize / numThreads + 1;
if (this.chunkSize > onetimeChunkSize) { if (chunkSize > onetimeChunkSize) {
this.chunkSize = (int) onetimeChunkSize; chunkSize = (int) onetimeChunkSize;
} }
// 开启多线程下载 //开启多线程下载
this.running = true; running = true;
for (int i = 0; i < this.numThreads; ++i) { for (int i = 0; i < numThreads; ++i) {
new Thread(MultiThreadedDownloader.this::worker).start(); new Thread(MultiThreadedDownloader.this::worker).start();
} }
// 构造response //构造response
PipedInputStream input = new PipedInputStream(); PipedInputStream input = new PipedInputStream();
PipedOutputStream output = new PipedOutputStream(input); PipedOutputStream output = new PipedOutputStream(input);
NanoHTTPD.Response mResponse = newFixedLengthResponse(status, contentType, input, contentLength); NanoHTTPD.Response mResponse = newFixedLengthResponse(status, contentType, input, contentLength);
@ -157,7 +164,7 @@ public class MultiThreadedDownloader {
} }
} }
// 搬运数据流 //搬运数据流
new Thread(() -> { new Thread(() -> {
try { try {
while (true) { while (true) {
@ -182,32 +189,28 @@ public class MultiThreadedDownloader {
return mResponse; return mResponse;
} }
// 读取文件内容 //读取文件内容
private byte[] read() throws Exception { private byte[] read() throws Exception {
// 判断文件是否下载结束 //判断文件是否下载结束
if (this.currentOffset > this.endOffset) { if (currentOffset > endOffset) {
this.stop(); stop();
return null; return null;
} }
// 获取当前的chunk的数据 //获取当前的chunk的数据
Chunk currentChunk = this.readyChunkQueue.poll(this.timeout, TimeUnit.SECONDS); Chunk currentChunk = readyChunkQueue.poll(timeout, TimeUnit.SECONDS);
if (currentChunk == null) { if (currentChunk == null) {
this.stop(); stop();
throw new Exception("read timeout"); throw new Exception("read timeout");
} }
while (this.running) { while (running) {
byte[] buffer = currentChunk.get(); byte[] buffer = currentChunk.get();
if (buffer != null) { if (buffer != null) {
this.currentOffset += buffer.length; currentOffset += buffer.length;
return buffer; return buffer;
} else { } else {
try { SystemClock.sleep(100);
sleep(100);
} catch (Exception e) {
e.printStackTrace();
}
} }
} }
@ -215,94 +218,79 @@ public class MultiThreadedDownloader {
} }
private void worker() { private void worker() {
while (this.running) { while (running) {
// 生成下一个chunk //生成下一个chunk
Chunk chunk = null; Chunk chunk = null;
this.lock.lock(); lock.lock();
long startOffset = this.nextChunkStartOffset; long startOffset = nextChunkStartOffset;
this.nextChunkStartOffset += this.chunkSize; nextChunkStartOffset += chunkSize;
if (startOffset <= this.endOffset) { if (startOffset <= endOffset) {
long endOffset = startOffset + this.chunkSize - 1; long endOffset = startOffset + chunkSize - 1;
if (endOffset > this.endOffset) { if (endOffset > this.endOffset) {
endOffset = this.endOffset; endOffset = this.endOffset;
} }
chunk = new Chunk(startOffset, endOffset); chunk = new Chunk(startOffset, endOffset);
this.readyChunkQueue.add(chunk); readyChunkQueue.add(chunk);
} }
this.lock.unlock(); lock.unlock();
// 所有chunk已下载完 //所有chunk已下载完
if (chunk == null) { if (chunk == null) {
break; break;
} }
while (this.running) { while (running) {
// 过多的数据未被取走，先休息一下，避免内存溢出 //过多的数据未被取走，先休息一下，避免内存溢出
if (chunk.startOffset - this.currentOffset >= this.chunkSize * this.maxBufferedChunk) { if (chunk.startOffset - currentOffset >= (long) chunkSize * maxBufferedChunk) {
try { SystemClock.sleep(1000);
sleep(1000);
} catch (Exception e) {
e.printStackTrace();
}
} else { } else {
break; break;
} }
} }
while (this.running) { while (running) {
try { try {
// 建立连接 //建立连接
Map<String, String> mHeaders = new HashMap<>(); Map<String, String> mHeaders = new HashMap<>();
for (Map.Entry<String, String> entry : this.headers.entrySet()) { for (Map.Entry<String, String> entry : headers.entrySet()) {
if (!shouldFilterRequestHeaderKey(entry.getKey())) { if (entry.getKey().equalsIgnoreCase("Range")) {
if (entry.getKey().equalsIgnoreCase("Range")) { //设置下载范围
// 设置下载范围 mHeaders.put("Range", String.format(Locale.getDefault(), "bytes=%d-%d", chunk.startOffset, chunk.endOffset));
mHeaders.put("Range", String.format("bytes=%d-%d", chunk.startOffset, chunk.endOffset)); } else {
} else { mHeaders.put(entry.getKey(), entry.getValue());
mHeaders.put(entry.getKey(), entry.getValue());
}
} }
} }
Response response = OkHttp.newCall(getDownloadUrl(), mHeaders); Response response = OkHttp.newCall(getDownloadUrl(), mHeaders);
// 检查状态码 //检查状态码
int responseCode = response.code(); int responseCode = response.code();
if (responseCode < 200 || responseCode >= 300) { if (responseCode < 200 || responseCode >= 300) {
throw new Exception("response code: " + responseCode); throw new Exception("response code: " + responseCode);
} }
//接收数据
if (response.body() != null) {
chunk.put(response.body().bytes());
}
// 接收数据
byte[] buffer = response.body().bytes();
chunk.put(buffer);
break; break;
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
try { SystemClock.sleep(1000);
sleep(1000);
} catch (Exception e1) {
e1.printStackTrace();
}
} }
} }
} }
} }
private String getDownloadUrl() { private String getDownloadUrl() {
if (this.url.contains("/proxy?")) { if (url.contains("/proxy?")) {
return OkHttp.string(this.url); return OkHttp.string(url);
} else { } else {
return this.url; return url;
} }
} }
private boolean shouldFilterRequestHeaderKey(String key) {
if (key == null) return true;
key = key.toLowerCase();
return key.equals("host") || key.equals("http-client-ip") || key.equals("remote-addr");
}
public void stop() { public void stop() {
this.running = false; running = false;
} }
private static class Chunk { private static class Chunk {

BIN
jar/custom_spider.jar
View File

Binary file not shown.

2
jar/custom_spider.jar.md5
View File

@ -1 +1 @@
41ef471afad9198e7273a12f9912578a 4bc4aada582cd00fafe20d12508dc2a7