dongge
/
AndroidCatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

多线程测试

This commit is contained in:
lushunming 2025-06-18 14:32:03 +08:00
parent 25cd597ae6
commit 68c4b89344
6 changed files with 87 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/src/main/java/com/github/catvod/spider/Proxy.java
View File

@ -55,7 +55,7 @@ public class Proxy extends Spider {
/*for (Map.Entry<String, String> entry : params.entrySet()) {
if (!keys.contains(entry.getKey())) header.put(entry.getKey(), entry.getValue());
}*/
return ProxyVideo.proxy(url, header);
return ProxyVideo.proxyMultiThread(url, header);
}

158
app/src/main/java/com/github/catvod/utils/ProxyVideo.java
View File

@ -7,6 +7,7 @@ import com.github.catvod.net.OkHttp;
import com.github.catvod.spider.Proxy;
import com.google.gson.Gson;
import okhttp3.Response;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.json.JSONObject;
@ -78,82 +79,113 @@ public class ProxyVideo {
public static Object[] proxyMultiThread(String url, Map<String, String> headers) throws Exception {
Map<String, String> newHeaders = new HashMap<>(headers);
newHeaders.put("range", "bytes=0-0");
Object[] info = proxy(url, newHeaders);
int code = (int) info[0];
if (code != 206) {
return proxy(url, headers);
}
String contentRange = ((Map<String, String>) info[3]).get("Content-Range");
//文件总大小
String total = StringUtils.split(contentRange, "/")[1];
String range = headers.get("range");
SpiderDebug.log("---proxyMultiThread,Range:" + range);
Range rangeObj = parseRange(range);
Map<String, String> rangeObj = parseRange(range);
//没有range,无需分割
if (rangeObj == null) {
SpiderDebug.log("没有range,无需分割");
return proxy(url, headers);
} else {
//end 为空，测试请求
if (StringUtils.isAllBlank(rangeObj.getEnd())) {
return proxy(url, headers);
} else {
long start = Long.parseLong(rangeObj.getStart());
long end = Long.parseLong(rangeObj.getEnd());
List<long[]> partList = generatePart(rangeObj, total);
long size = end - start;
//每块大小
long partSize = size / THREAD_NUM;
ExecutorService service = Executors.newFixedThreadPool(THREAD_NUM);
// 存储执行结果的List
List<Future<Response>> results = new ArrayList<Future<Response>>();
for (int i = 0; i < THREAD_NUM; i++) {
long partEnd = start + partSize >= end ? end : start + partSize;
String newRange = "range=" + start + "-" + partEnd;
start = partEnd;
ExecutorService service = Executors.newFixedThreadPool(THREAD_NUM);
// 存储执行结果的List
List<Future<Response>> results = new ArrayList<Future<Response>>();
for (long[] part : partList) {
headers.put("Range", newRange);
Future<Response> result = service.submit(() -> {
try {
return OkHttp.newCall(url, headers);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
results.add(result);
}
byte[] bytes = new byte[(int) size];
Response response = null;
for (int i = 0; i < THREAD_NUM; i++) {
// 获取包含返回结果的future对象
Future<Response> future = results.get(i);
// 从future中取出执行结果（若尚未返回结果，则get方法被阻塞，直到结果被返回为止）
response = future.get();
response.body().byteStream().read(bytes, (int) (i * partSize), (int) partSize);
SpiderDebug.log("---第" + i + "块下载完成" + ";headers:" + Json.toJson(response.headers()));
String newRange = "bytes=" + part[0] + "-" + part[1];
SpiderDebug.log("下载开始" + ";newRange:" + newRange);
}
String contentType = response.headers().get("Content-Type");
String contentDisposition = response.headers().get("Content-Disposition");
if (contentDisposition != null) contentType = getMimeType(contentDisposition);
Map<String, String> respHeaders = new HashMap<>();
Map<String, String> headerNew = new HashMap<>(headers);
headerNew.put("range", newRange);
Future<Response> result = service.submit(() -> {
try {
return OkHttp.newCall(url, headerNew);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
results.add(result);
}
byte[] bytes = null;
Response response = null;
for (int i = 0; i < THREAD_NUM; i++) {
// 获取包含返回结果的future对象
Future<Response> future = results.get(i);
// 从future中取出执行结果（若尚未返回结果，则get方法被阻塞，直到结果被返回为止）
response = future.get();
bytes = ArrayUtils.addAll(bytes, response.body().bytes());
SpiderDebug.log("---第" + i + "块下载完成" + ";Content-Range:" + response.headers().get("Content-Range"));
}
service.shutdown();
String contentType = response.headers().get("Content-Type");
String contentDisposition = response.headers().get("Content-Disposition");
if (contentDisposition != null) contentType = getMimeType(contentDisposition);
Map<String, String> respHeaders = new HashMap<>();
/* respHeaders.put("Access-Control-Allow-Credentials", "true");
respHeaders.put("Access-Control-Allow-Origin", "*");*/
for (String key : response.headers().names()) {
respHeaders.put(key, response.headers().get(key));
}
SpiderDebug.log("++proxy res contentType:" + contentType);
// SpiderDebug.log("++proxy res body:" + response.body());
SpiderDebug.log("++proxy res respHeaders:" + Json.toJson(respHeaders));
return new Object[]{response.code(), contentType, new ByteArrayInputStream(bytes), respHeaders};
for (String key : response.headers().names()) {
respHeaders.put(key, response.headers().get(key));
}
respHeaders.put("Content-Length", String.valueOf(bytes.length));
respHeaders.put("Content-Range", String.format("bytes %s-%s/%s", partList.get(0)[0], partList.get(THREAD_NUM - 1)[1], total));
SpiderDebug.log("++proxy res contentType:" + contentType);
// SpiderDebug.log("++proxy res body:" + response.body());
SpiderDebug.log("++proxy res respHeaders:" + Json.toJson(respHeaders));
return new Object[]{response.code(), contentType, new ByteArrayInputStream(bytes), respHeaders};
}
}
private static Range parseRange(String range) {
private static List<long[]> generatePart(Map<String, String> rangeObj, String total) {
long start = Long.parseLong(rangeObj.get("start"));
long end = StringUtils.isAllBlank(rangeObj.get("end")) ? start + 1024 * 1024 * 1 * 4 : Long.parseLong(rangeObj.get("end"));
long totalSize = Long.parseLong(total);
end = Math.min(end, totalSize - 1);
long length = end - start + 1;
long size = length / THREAD_NUM;
List<long[]> partList = new ArrayList<>();
for (int i = 0; i < THREAD_NUM; i++) {
long partEnd = Math.min(start + size, end);
partList.add(new long[]{start, partEnd});
start = partEnd + 1;
}
return partList;
}
private static Map<String, String> parseRange(String range) {
SpiderDebug.log("parseRange:" + range);
if (StringUtils.isNoneBlank(range)) {
String[] ranges = StringUtils.split(range.replace("bytes=", ""), "-");
String start = ranges[0];
String end = ranges.length > 1 ? ranges[1] : "";
return new Range(start, end);
return Map.of("start", start, "end", end);
}
return null;
}
@ -193,29 +225,5 @@ public class ProxyVideo {
/**
* 视频range
*/
private static class Range {
private String start;
private String end;
public Range(String start, String end) {
start = start;
end = end;
}
public String getStart() {
return start;
}
public void setStart(String start) {
this.start = start;
}
public String getEnd() {
return end;
}
public void setEnd(String end) {
this.end = end;
}
}
}

2
app/src/test/java/com/github/catvod/api/QuarkApiTest.java
View File

@ -32,7 +32,7 @@ public class QuarkApiTest {
public void testdownload() throws Exception {
String url = "https://video-play-p-zb.cdn.yun.cn/P7r95SEr/1997440970/7986fbd7419840ba83d70e7ec36f933867d2fadf/67d2fadf98f5dd83fcd64481858236a79b4c3384?auth_key=1741952223-3304496-16098-d233ccbc65c0321102d36db56f3db9c2&sp=642&token=3-08917a23ee79367eab5e9dcfbd898751-3-2-963-5cbf_3bd039d6d54ec6a8737515d6f20a488c-0-0-0-0-e9fc047ff0aa9b590be130819e1e82f2&ud=9-0-1-2-1-5-8-N-0-4-0-N";
OkResult okResult1 = OkHttp.get(url, new HashMap<>(), Map.of("Range", "bytes=0-0"));
OkResult okResult1 = OkHttp.get(url, new HashMap<>(), Map.of("Range", "bytes=0-"));
assert okResult1.getCode() == 206;

BIN
jar/custom_spider.jar
View File

Binary file not shown.

2
jar/custom_spider.jar.md5
View File

@ -1 +1 @@
c2f5d58d6de2397d7e8955a449b38835
5977963210de316129a4b64a7261d469

2
json/test.json
View File

@ -1,5 +1,5 @@
{
"spider": "https://ghproxy.net/https://raw.githubusercontent.com/lushunming/AndroidCatVodSpider/multiThread/jar/custom_spider.jar;md5;c2f5d58d6de2397d7e8955a449b38835",
"spider": "https://ghproxy.net/https://raw.githubusercontent.com/lushunming/AndroidCatVodSpider/multiThread/jar/custom_spider.jar;md5;5977963210de316129a4b64a7261d469",
"lives": [
{
"name": "电视直播",