diff --git a/app/src/main/java/com/github/catvod/utils/DownloadMT.kt b/app/src/main/java/com/github/catvod/utils/DownloadMT.kt index 3dff3935..b6b38f8b 100644 --- a/app/src/main/java/com/github/catvod/utils/DownloadMT.kt +++ b/app/src/main/java/com/github/catvod/utils/DownloadMT.kt @@ -2,7 +2,6 @@ package com.github.catvod.utils import com.github.catvod.crawler.SpiderDebug import com.github.catvod.net.OkHttp -import com.github.catvod.utils.ProxyVideo.generatePart import com.github.catvod.utils.ProxyVideo.getInfo import com.github.catvod.utils.ProxyVideo.getMimeType import com.github.catvod.utils.ProxyVideo.parseRange @@ -19,30 +18,27 @@ import okhttp3.Response import org.apache.commons.lang3.StringUtils import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream -import java.io.PipedInputStream -import java.io.PipedOutputStream -import java.io.SequenceInputStream +import kotlin.math.min object DownloadMT { - val THREAD_NUM: Int = Runtime.getRuntime().availableProcessors() * 2 + val CORE_NUM: Int = Runtime.getRuntime().availableProcessors() private val infos = mutableMapOf>(); fun proxyMultiThread(url: String, headers: Map): Array? = runBlocking { - proxy(url, headers) + proxyAsync(url, headers) } - suspend fun proxy(url: String, headers: Map): Array? { + suspend fun proxyAsync(url: String, headers: Map): Array? { /* val service = Executors.newFixedThreadPool(THREAD_NUM)*/ - SpiderDebug.log("--proxyMultiThread: THREAD_NUM " + THREAD_NUM) - + SpiderDebug.log("--proxyMultiThread: CORE_NUM " + CORE_NUM) + //默认线程数核心数两倍 + var threadNum = CORE_NUM * 2 try { //缓存，避免每次都请求total等信息 - - var info = infos[url] if (info == null) { infos.clear() @@ -51,7 +47,7 @@ object DownloadMT { } val code = info?.get(0) as Int - SpiderDebug.log("-----------code:" + code) + SpiderDebug.log("-----------code:$code") if (code != 206) { return proxy(url, headers) } @@ -64,10 +60,19 @@ object DownloadMT { val total = StringUtils.split(contentRange, "/")[1] SpiderDebug.log("--文件总大小:$total") - //如果文件太小，也不走代理 - /* if (total.toLong() < 1024 * 1024 * 100) { - return proxy(url, headers) - }*/ + //如果文件小于50MB，也不走代理 + if (total.toLong() < 1024 * 1024 * 50L) { + return proxy(url, headers) + } else if (total.toLong() < 1024 * 1024 * 1024L * 10L) { + //10GB以下 + threadNum = CORE_NUM * 4 + } else if (total.toLong() < 1024 * 1024 * 1024L * 40L) { + //40GB以下 + threadNum = CORE_NUM * 8 + } else { + //40GB以上 + threadNum = CORE_NUM * 16 + } var range = if (StringUtils.isAllBlank(headers["range"])) headers["Range"] else headers["range"] if (StringUtils.isAllBlank(range)) range = "bytes=0-"; @@ -77,11 +82,11 @@ object DownloadMT { ) //没有range,无需分割 - val partList = generatePart(rangeObj, total) + val partList = generatePart(rangeObj, total, threadNum) // 存储执行结果的List val jobs = mutableListOf() - val channels = List(THREAD_NUM) { Channel() } + val channels = List(threadNum) { Channel() } for ((index, part) in partList.withIndex()) { @@ -146,11 +151,11 @@ object DownloadMT { /* respHeaders.put("Access-Control-Allow-Credentials", "true"); respHeaders.put("Access-Control-Allow-Origin", "*");*/ resHeader["Content-Length"] = - (partList[THREAD_NUM - 1][1] - partList[0][0] + 1).toString() + (partList[threadNum - 1][1] - partList[0][0] + 1).toString() resHeader.remove("content-length") resHeader["Content-Range"] = String.format( - "bytes %s-%s/%s", partList[0][0], partList[THREAD_NUM - 1][1], total + "bytes %s-%s/%s", partList[0][0], partList[threadNum - 1][1], total ) resHeader.remove("content-range") @@ -170,6 +175,43 @@ object DownloadMT { } } + private fun generatePart( + rangeObj: Map, total: String, threadNum: Int + ): List { + val totalSize = total.toLong() + + //默认8MB + var partSize = 1024 * 1024 * 8L + + if (totalSize < 1024 * 1024 * 1024L * 10L) { + //10GB以下，分片8MB + partSize = 1024 * 1024 * 8L + } else if (totalSize < 1024 * 1024 * 1024L * 40L) { + //40GB以下，分片32MB + partSize = 1024 * 1024 * 8L * 4 + } else { + //40GB以上，分片128MB + partSize = 1024 * 1024 * 8L * 4 * 4 + } + + var start = rangeObj["start"]!!.toLong() + var end = + if (StringUtils.isAllBlank(rangeObj["end"])) start + partSize else rangeObj["end"]!!.toLong() + + + end = min(end.toDouble(), (totalSize - 1).toDouble()).toLong() + val length = end - start + 1 + + val size = length / threadNum + val partList: MutableList = ArrayList() + for (i in 0.. diff --git a/app/src/main/java/com/github/catvod/utils/ProxyVideo.java b/app/src/main/java/com/github/catvod/utils/ProxyVideo.java index bf860e74..8e3034d6 100644 --- a/app/src/main/java/com/github/catvod/utils/ProxyVideo.java +++ b/app/src/main/java/com/github/catvod/utils/ProxyVideo.java @@ -100,28 +100,6 @@ public class ProxyVideo { return DownloadMT.INSTANCE.proxyMultiThread(url, headers); } - public static List generatePart(Map rangeObj, String total) { - long totalSize = Long.parseLong(total); - //超过10GB，分块是10Mb，不然是2MB - long partSize = totalSize > 8L * 1024L * 1024L * 1024L * 10L ? 1024 * 1024 * 8 * 10L : 1024 * 1024 * 8 * 2L; - - long start = Long.parseLong(rangeObj.get("start")); - long end = StringUtils.isAllBlank(rangeObj.get("end")) ? start + partSize : Long.parseLong(rangeObj.get("end")); - - - end = Math.min(end, totalSize - 1); - long length = end - start + 1; - - long size = length / THREAD_NUM; - List partList = new ArrayList<>(); - for (int i = 0; i < THREAD_NUM; i++) { - long partEnd = Math.min(start + size, end); - - partList.add(new long[]{start, partEnd}); - start = partEnd + 1; - } - return partList; - } public static Map parseRange(String range) { SpiderDebug.log("parseRange:" + range); diff --git a/jar/custom_spider.jar b/jar/custom_spider.jar index 5d172e05..3ad35c19 100644 Binary files a/jar/custom_spider.jar and b/jar/custom_spider.jar differ diff --git a/jar/custom_spider.jar.md5 b/jar/custom_spider.jar.md5 index a71f8628..1923fa1c 100644 --- a/jar/custom_spider.jar.md5 +++ b/jar/custom_spider.jar.md5 @@ -1 +1 @@ -cd1d6a277a73915237cf1ecb19d5f46a +a926c6e48bbffc73be39e17fcc6650a1 diff --git a/json/test.json b/json/test.json index f352b536..cf13edbf 100644 --- a/json/test.json +++ b/json/test.json @@ -1,5 +1,5 @@ { - "spider": "https://ghproxy.net/https://raw.githubusercontent.com/lushunming/AndroidCatVodSpider/multiThreadkt/jar/custom_spider.jar;md5;cd1d6a277a73915237cf1ecb19d5f46a", + "spider": "https://ghproxy.net/https://raw.githubusercontent.com/lushunming/AndroidCatVodSpider/multiThreadkt/jar/custom_spider.jar;md5;a926c6e48bbffc73be39e17fcc6650a1", "lives": [ { "name": "电视直播",