多线程测试
This commit is contained in:
parent
0d5753a208
commit
459038157d
|
|
@ -507,7 +507,7 @@ public class AliYun {
|
||||||
Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
|
Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
|
||||||
List<String> keys = Arrays.asList("referer", "icy-metadata", "range", "connection", "accept-encoding", "user-agent");
|
List<String> keys = Arrays.asList("referer", "icy-metadata", "range", "connection", "accept-encoding", "user-agent");
|
||||||
for (String key : params.keySet()) if (keys.contains(key)) headers.put(key, params.get(key));
|
for (String key : params.keySet()) if (keys.contains(key)) headers.put(key, params.get(key));
|
||||||
return ProxyVideo.proxy(downloadUrl, headers);
|
return ProxyVideo.proxyMultiThread(downloadUrl, headers);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getM3u8Url(String shareId, String fileId, String templateId) {
|
private String getM3u8Url(String shareId, String fileId, String templateId) {
|
||||||
|
|
|
||||||
|
|
@ -73,7 +73,7 @@ public class QuarkApi {
|
||||||
if (Util.getExt(url).contains("m3u8")) {
|
if (Util.getExt(url).contains("m3u8")) {
|
||||||
return getM3u8(url, header);
|
return getM3u8(url, header);
|
||||||
}
|
}
|
||||||
return ProxyVideo.proxy(url, header);
|
return ProxyVideo.proxyMultiThread(url, header);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -7,17 +7,23 @@ import com.github.catvod.net.OkHttp;
|
||||||
import com.github.catvod.spider.Proxy;
|
import com.github.catvod.spider.Proxy;
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.Gson;
|
||||||
import okhttp3.Response;
|
import okhttp3.Response;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
import java.net.URLEncoder;
|
import java.net.URLEncoder;
|
||||||
import java.nio.charset.Charset;
|
import java.nio.charset.Charset;
|
||||||
import java.util.HashMap;
|
import java.util.*;
|
||||||
import java.util.Locale;
|
import java.util.concurrent.ExecutorService;
|
||||||
import java.util.Map;
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.Future;
|
||||||
|
|
||||||
public class ProxyVideo {
|
public class ProxyVideo {
|
||||||
|
|
||||||
private static final String GO_SERVER = "http://127.0.0.1:7777/";
|
private static final String GO_SERVER = "http://127.0.0.1:7777/";
|
||||||
|
//线程数4
|
||||||
|
private static final int THREAD_NUM = 4;
|
||||||
|
|
||||||
|
|
||||||
public static String buildCommonProxyUrl(String url, Map<String, String> headers) {
|
public static String buildCommonProxyUrl(String url, Map<String, String> headers) {
|
||||||
return Proxy.getUrl() + "?do=proxy&url=" + Util.base64Encode(url.getBytes(Charset.defaultCharset())) + "&header=" + Util.base64Encode((new Gson().toJson(headers)).getBytes(Charset.defaultCharset()));
|
return Proxy.getUrl() + "?do=proxy&url=" + Util.base64Encode(url.getBytes(Charset.defaultCharset())) + "&header=" + Util.base64Encode((new Gson().toJson(headers)).getBytes(Charset.defaultCharset()));
|
||||||
|
|
@ -70,6 +76,84 @@ public class ProxyVideo {
|
||||||
return new Object[]{response.code(), contentType, response.body().byteStream(), respHeaders};
|
return new Object[]{response.code(), contentType, response.body().byteStream(), respHeaders};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static Object[] proxyMultiThread(String url, Map<String, String> headers) throws Exception {
|
||||||
|
|
||||||
|
String range = headers.get("Range");
|
||||||
|
Range rangeObj = parseRange(range);
|
||||||
|
//没有range,无需分割
|
||||||
|
if (rangeObj == null) {
|
||||||
|
return proxy(url, headers);
|
||||||
|
} else {
|
||||||
|
//end 为空,测试请求
|
||||||
|
if (StringUtils.isAllBlank(rangeObj.getEnd())) {
|
||||||
|
return proxy(url, headers);
|
||||||
|
} else {
|
||||||
|
long start = Long.parseLong(rangeObj.getStart());
|
||||||
|
long end = Long.parseLong(rangeObj.getEnd());
|
||||||
|
|
||||||
|
long size = end - start;
|
||||||
|
//每块大小
|
||||||
|
long partSize = size / THREAD_NUM;
|
||||||
|
ExecutorService service = Executors.newFixedThreadPool(THREAD_NUM);
|
||||||
|
// 存储执行结果的List
|
||||||
|
List<Future<Response>> results = new ArrayList<Future<Response>>();
|
||||||
|
for (int i = 0; i < THREAD_NUM; i++) {
|
||||||
|
long partEnd = start + partSize >= end ? end : start + partSize;
|
||||||
|
String newRange = "range=" + start + "-" + partEnd;
|
||||||
|
start = partEnd;
|
||||||
|
|
||||||
|
headers.put("Range", newRange);
|
||||||
|
Future<Response> result = service.submit(() -> {
|
||||||
|
try {
|
||||||
|
return OkHttp.newCall(url, headers);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
results.add(result);
|
||||||
|
}
|
||||||
|
byte[] bytes = new byte[(int) size];
|
||||||
|
Response response = null;
|
||||||
|
for (int i = 0; i < THREAD_NUM; i++) {
|
||||||
|
// 获取包含返回结果的future对象
|
||||||
|
Future<Response> future = results.get(i);
|
||||||
|
// 从future中取出执行结果(若尚未返回结果,则get方法被阻塞,直到结果被返回为止)
|
||||||
|
response = future.get();
|
||||||
|
response.body().byteStream().read(bytes, (int) (i * partSize), (int) partSize);
|
||||||
|
SpiderDebug.log("---第" + i + "块下载完成" + ";headers:" + Json.toJson(response.headers()));
|
||||||
|
|
||||||
|
}
|
||||||
|
String contentType = response.headers().get("Content-Type");
|
||||||
|
String contentDisposition = response.headers().get("Content-Disposition");
|
||||||
|
if (contentDisposition != null) contentType = getMimeType(contentDisposition);
|
||||||
|
Map<String, String> respHeaders = new HashMap<>();
|
||||||
|
/* respHeaders.put("Access-Control-Allow-Credentials", "true");
|
||||||
|
respHeaders.put("Access-Control-Allow-Origin", "*");*/
|
||||||
|
|
||||||
|
for (String key : response.headers().names()) {
|
||||||
|
respHeaders.put(key, response.headers().get(key));
|
||||||
|
}
|
||||||
|
SpiderDebug.log("++proxy res contentType:" + contentType);
|
||||||
|
// SpiderDebug.log("++proxy res body:" + response.body());
|
||||||
|
SpiderDebug.log("++proxy res respHeaders:" + Json.toJson(respHeaders));
|
||||||
|
return new Object[]{response.code(), contentType, new ByteArrayInputStream(bytes), respHeaders};
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Range parseRange(String range) {
|
||||||
|
if (StringUtils.isNoneBlank(range)) {
|
||||||
|
String[] ranges = range.replace("bytes=", "").split("-");
|
||||||
|
String start = ranges[0];
|
||||||
|
String end = ranges[1];
|
||||||
|
return new Range(start, end);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
private static String getMimeType(String contentDisposition) {
|
private static String getMimeType(String contentDisposition) {
|
||||||
if (contentDisposition.endsWith(".mp4")) {
|
if (contentDisposition.endsWith(".mp4")) {
|
||||||
return "video/mp4";
|
return "video/mp4";
|
||||||
|
|
@ -101,4 +185,33 @@ public class ProxyVideo {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 视频range
|
||||||
|
*/
|
||||||
|
private static class Range {
|
||||||
|
private String start;
|
||||||
|
private String end;
|
||||||
|
|
||||||
|
public Range(String start, String end) {
|
||||||
|
start = start;
|
||||||
|
end = end;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getStart() {
|
||||||
|
return start;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setStart(String start) {
|
||||||
|
this.start = start;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getEnd() {
|
||||||
|
return end;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setEnd(String end) {
|
||||||
|
this.end = end;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Binary file not shown.
|
|
@ -1 +1 @@
|
||||||
4a9814b359e0f39cf4613cd801871c06
|
e815d386e2777a1c728d0133698de99a
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
{
|
{
|
||||||
"spider": "https://ghproxy.net/https://raw.githubusercontent.com/lushunming/AndroidCatVodSpider/refs/heads/tianyiPassword/jar/custom_spider.jar;md5;bb54c03fd035f011b948b4ccf1273819",
|
"spider": "https://ghproxy.net/https://raw.githubusercontent.com/lushunming/AndroidCatVodSpider/refs/heads/multiThread/jar/custom_spider.jar;md5;e815d386e2777a1c728d0133698de99a",
|
||||||
"lives": [
|
"lives": [
|
||||||
{
|
{
|
||||||
"name": "电视直播",
|
"name": "电视直播",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
pluginManagement {
|
pluginManagement {
|
||||||
repositories {
|
repositories {
|
||||||
mavenCentral()
|
|
||||||
maven { url 'https://maven.aliyun.com/repository/public/' }
|
maven { url 'https://maven.aliyun.com/repository/public/' }
|
||||||
maven { url 'https://maven.aliyun.com/repository/google/' }
|
maven { url 'https://maven.aliyun.com/repository/google/' }
|
||||||
maven { url 'https://maven.aliyun.com/repository/gradle-plugin/' }
|
maven { url 'https://maven.aliyun.com/repository/gradle-plugin/' }
|
||||||
|
|
@ -9,6 +9,7 @@ pluginManagement {
|
||||||
maven { url 'https://maven.aliyun.com/repository/mapr-public/' }
|
maven { url 'https://maven.aliyun.com/repository/mapr-public/' }
|
||||||
gradlePluginPortal()
|
gradlePluginPortal()
|
||||||
google()
|
google()
|
||||||
|
mavenCentral()
|
||||||
|
|
||||||
maven { url "https://plugins.gradle.org/m2/" }
|
maven { url "https://plugins.gradle.org/m2/" }
|
||||||
maven { url "https://www.jitpack.io" }
|
maven { url "https://www.jitpack.io" }
|
||||||
|
|
@ -17,7 +18,7 @@ pluginManagement {
|
||||||
dependencyResolutionManagement {
|
dependencyResolutionManagement {
|
||||||
repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
|
repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
|
||||||
repositories {
|
repositories {
|
||||||
mavenCentral()
|
|
||||||
maven { url 'https://maven.aliyun.com/repository/public/' }
|
maven { url 'https://maven.aliyun.com/repository/public/' }
|
||||||
maven { url 'https://maven.aliyun.com/repository/google/' }
|
maven { url 'https://maven.aliyun.com/repository/google/' }
|
||||||
maven { url 'https://maven.aliyun.com/repository/gradle-plugin/' }
|
maven { url 'https://maven.aliyun.com/repository/gradle-plugin/' }
|
||||||
|
|
@ -25,6 +26,7 @@ dependencyResolutionManagement {
|
||||||
maven { url 'https://maven.aliyun.com/repository/central/'}
|
maven { url 'https://maven.aliyun.com/repository/central/'}
|
||||||
maven { url 'https://maven.aliyun.com/repository/mapr-public/' }
|
maven { url 'https://maven.aliyun.com/repository/mapr-public/' }
|
||||||
google()
|
google()
|
||||||
|
mavenCentral()
|
||||||
maven { url "https://www.jitpack.io" }
|
maven { url "https://www.jitpack.io" }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue