package com.github.catvod.utils; import android.os.SystemClock; import android.text.TextUtils; import com.github.catvod.crawler.SpiderDebug; import com.github.catvod.net.OkHttp; import com.github.catvod.spider.Proxy; import com.google.gson.Gson; import okhttp3.Response; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.json.JSONObject; import java.io.ByteArrayInputStream; import java.net.URLEncoder; import java.nio.charset.Charset; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; public class ProxyVideo { private static final String GO_SERVER = "http://127.0.0.1:7777/"; //线程数4 private static final int THREAD_NUM = 16; public static String buildCommonProxyUrl(String url, Map headers) { return Proxy.getUrl() + "?do=proxy&url=" + Util.base64Encode(url.getBytes(Charset.defaultCharset())) + "&header=" + Util.base64Encode((new Gson().toJson(headers)).getBytes(Charset.defaultCharset())); } public static void go() { boolean close = OkHttp.string(GO_SERVER).isEmpty(); if (close) OkHttp.string("http://127.0.0.1:" + Proxy.getPort() + "/go"); if (close) while (OkHttp.string(GO_SERVER).isEmpty()) SystemClock.sleep(20); } public static String goVer() { try { go(); String result = OkHttp.string(GO_SERVER + "version"); return new JSONObject(result).optString("version"); } catch (Exception e) { return ""; } } public static String url(String url, int thread) { if (!TextUtils.isEmpty(goVer()) && url.contains("/proxy?")) url += "&response=url"; return String.format(Locale.getDefault(), "%s?url=%s&thread=%d", GO_SERVER, URLEncoder.encode(url), thread); } public static Object[] proxy(String url, Map headers) throws Exception { SpiderDebug.log(" ++start proxy:"); SpiderDebug.log(" ++proxy url:" + url); SpiderDebug.log(" ++proxy header:" + Json.toJson(headers)); Response response = OkHttp.newCall(url, headers); SpiderDebug.log(" ++end proxy:"); SpiderDebug.log(" ++proxy res code:" + response.code()); SpiderDebug.log(" ++proxy res header:" + Json.toJson(response.headers())); // SpiderDebug.log(" ++proxy res data:" + Json.toJson(response.body())); String contentType = response.headers().get("Content-Type"); String contentDisposition = response.headers().get("Content-Disposition"); if (contentDisposition != null) contentType = getMimeType(contentDisposition); Map respHeaders = new HashMap<>(); /* respHeaders.put("Access-Control-Allow-Credentials", "true"); respHeaders.put("Access-Control-Allow-Origin", "*");*/ for (String key : response.headers().names()) { respHeaders.put(key, response.headers().get(key)); } SpiderDebug.log("++proxy res contentType:" + contentType); // SpiderDebug.log("++proxy res body:" + response.body()); SpiderDebug.log("++proxy res respHeaders:" + Json.toJson(respHeaders)); return new Object[]{response.code(), contentType, response.body().byteStream(), respHeaders}; } public static Object[] proxyMultiThread(String url, Map headers) throws Exception { SpiderDebug.log("--proxyMultiThread: start "); Map newHeaders = new HashMap<>(headers); newHeaders.put("range", "bytes=0-0"); newHeaders.put("Range", "bytes=0-0"); Object[] info = proxy(url, newHeaders); int code = (int) info[0]; if (code != 206) { return proxy(url, headers); } String contentRange = StringUtils.isAllBlank(((Map) info[3]).get("Content-Range")) ? ((Map) info[3]).get("content-range") : ((Map) info[3]).get("Content-Range"); SpiderDebug.log("--contentRange:" + contentRange); //文件总大小 String total = StringUtils.split(contentRange, "/")[1]; SpiderDebug.log("--文件总大小:" + total); String range = StringUtils.isAllBlank(headers.get("range")) ? headers.get("Range") : headers.get("range"); SpiderDebug.log("---proxyMultiThread,Range:" + range); Map rangeObj = parseRange(range); //没有range,无需分割 if (rangeObj == null) { SpiderDebug.log("没有range,无需分割"); return proxy(url, headers); } else { List partList = generatePart(rangeObj, total); ExecutorService service = Executors.newFixedThreadPool(THREAD_NUM); // 存储执行结果的List List> results = new ArrayList>(); for (long[] part : partList) { String newRange = "bytes=" + part[0] + "-" + part[1]; SpiderDebug.log("下载开始" + ";newRange:" + newRange); Map headerNew = new HashMap<>(headers); headerNew.put("range", newRange); headerNew.put("Range", newRange); Future result = service.submit(() -> { try { return OkHttp.newCall(url, headerNew); } catch (Exception e) { throw new RuntimeException(e); } }); results.add(result); } byte[] bytes = null; Response response = null; for (int i = 0; i < THREAD_NUM; i++) { // 获取包含返回结果的future对象 Future future = results.get(i); // 从future中取出执行结果(若尚未返回结果,则get方法被阻塞,直到结果被返回为止) response = future.get(); bytes = ArrayUtils.addAll(bytes, response.body().bytes()); SpiderDebug.log("---第" + i + "块下载完成" + ";Content-Range:" + response.headers().get("Content-Range")); SpiderDebug.log("---第" + i + "块下载完成" + ";content-range:" + response.headers().get("content-range")); } service.shutdown(); String contentType = response.headers().get("Content-Type"); String contentDisposition = response.headers().get("Content-Disposition"); if (contentDisposition != null) contentType = getMimeType(contentDisposition); Map respHeaders = new HashMap<>(); /* respHeaders.put("Access-Control-Allow-Credentials", "true"); respHeaders.put("Access-Control-Allow-Origin", "*");*/ for (String key : response.headers().names()) { respHeaders.put(key, response.headers().get(key)); } respHeaders.put("Content-Length", String.valueOf(bytes.length)); respHeaders.put("content-length", String.valueOf(bytes.length)); respHeaders.put("Content-Range", String.format("bytes %s-%s/%s", partList.get(0)[0], partList.get(THREAD_NUM - 1)[1], total)); respHeaders.put("content-range", String.format("bytes %s-%s/%s", partList.get(0)[0], partList.get(THREAD_NUM - 1)[1], total)); SpiderDebug.log("++proxy res contentType:" + contentType); // SpiderDebug.log("++proxy res body:" + response.body()); SpiderDebug.log("++proxy res respHeaders:" + Json.toJson(respHeaders)); return new Object[]{response.code(), contentType, new ByteArrayInputStream(bytes), respHeaders}; } } private static List generatePart(Map rangeObj, String total) { long start = Long.parseLong(rangeObj.get("start")); long end = StringUtils.isAllBlank(rangeObj.get("end")) ? start + 1024 * 1024 *8 : Long.parseLong(rangeObj.get("end")); long totalSize = Long.parseLong(total); end = Math.min(end, totalSize - 1); long length = end - start + 1; long size = length / THREAD_NUM; List partList = new ArrayList<>(); for (int i = 0; i < THREAD_NUM; i++) { long partEnd = Math.min(start + size, end); partList.add(new long[]{start, partEnd}); start = partEnd + 1; } return partList; } private static Map parseRange(String range) { SpiderDebug.log("parseRange:" + range); if (StringUtils.isNoneBlank(range)) { String[] ranges = StringUtils.split(range.replace("bytes=", ""), "-"); String start = ranges[0]; String end = ranges.length > 1 ? ranges[1] : ""; return Map.of("start", start, "end", end); } return null; } private static String getMimeType(String contentDisposition) { if (contentDisposition.endsWith(".mp4")) { return "video/mp4"; } else if (contentDisposition.endsWith(".webm")) { return "video/webm"; } else if (contentDisposition.endsWith(".avi")) { return "video/x-msvideo"; } else if (contentDisposition.endsWith(".wmv")) { return "video/x-ms-wmv"; } else if (contentDisposition.endsWith(".flv")) { return "video/x-flv"; } else if (contentDisposition.endsWith(".mov")) { return "video/quicktime"; } else if (contentDisposition.endsWith(".mkv")) { return "video/x-matroska"; } else if (contentDisposition.endsWith(".mpeg")) { return "video/mpeg"; } else if (contentDisposition.endsWith(".3gp")) { return "video/3gpp"; } else if (contentDisposition.endsWith(".ts")) { return "video/MP2T"; } else if (contentDisposition.endsWith(".mp3")) { return "audio/mp3"; } else if (contentDisposition.endsWith(".wav")) { return "audio/wav"; } else if (contentDisposition.endsWith(".aac")) { return "audio/aac"; } else { return null; } } /** * 视频range */ }