kt线程和分片优化
This commit is contained in:
lushunming
parent
22b771615f
commit
c7752dc1be
|
|
@ -2,7 +2,6 @@ package com.github.catvod.utils
|
|||
|
||||
import com.github.catvod.crawler.SpiderDebug
|
||||
import com.github.catvod.net.OkHttp
|
||||
import com.github.catvod.utils.ProxyVideo.generatePart
|
||||
import com.github.catvod.utils.ProxyVideo.getInfo
|
||||
import com.github.catvod.utils.ProxyVideo.getMimeType
|
||||
import com.github.catvod.utils.ProxyVideo.parseRange
|
||||
|
|
@ -19,30 +18,27 @@ import okhttp3.Response
|
|||
import org.apache.commons.lang3.StringUtils
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.io.PipedInputStream
|
||||
import java.io.PipedOutputStream
|
||||
import java.io.SequenceInputStream
|
||||
import kotlin.math.min
|
||||
|
||||
object DownloadMT {
|
||||
val THREAD_NUM: Int = Runtime.getRuntime().availableProcessors() * 2
|
||||
val CORE_NUM: Int = Runtime.getRuntime().availableProcessors()
|
||||
|
||||
private val infos = mutableMapOf<String, Array<Any>>();
|
||||
|
||||
fun proxyMultiThread(url: String, headers: Map<String, String>): Array<out Any?>? =
|
||||
runBlocking {
|
||||
proxy(url, headers)
|
||||
proxyAsync(url, headers)
|
||||
}
|
||||
|
||||
suspend fun proxy(url: String, headers: Map<String, String>): Array<out Any?>? {
|
||||
suspend fun proxyAsync(url: String, headers: Map<String, String>): Array<out Any?>? {
|
||||
|
||||
/* val service = Executors.newFixedThreadPool(THREAD_NUM)*/
|
||||
SpiderDebug.log("--proxyMultiThread: THREAD_NUM " + THREAD_NUM)
|
||||
|
||||
SpiderDebug.log("--proxyMultiThread: CORE_NUM " + CORE_NUM)
|
||||
//默认线程数核心数两倍
|
||||
var threadNum = CORE_NUM * 2
|
||||
|
||||
try {
|
||||
//缓存，避免每次都请求total等信息
|
||||
|
||||
|
||||
var info = infos[url]
|
||||
if (info == null) {
|
||||
infos.clear()
|
||||
|
|
@ -51,7 +47,7 @@ object DownloadMT {
|
|||
}
|
||||
|
||||
val code = info?.get(0) as Int
|
||||
SpiderDebug.log("-----------code:" + code)
|
||||
SpiderDebug.log("-----------code:$code")
|
||||
if (code != 206) {
|
||||
return proxy(url, headers)
|
||||
}
|
||||
|
|
@ -64,10 +60,19 @@ object DownloadMT {
|
|||
val total = StringUtils.split(contentRange, "/")[1]
|
||||
SpiderDebug.log("--文件总大小:$total")
|
||||
|
||||
//如果文件太小，也不走代理
|
||||
/* if (total.toLong() < 1024 * 1024 * 100) {
|
||||
return proxy(url, headers)
|
||||
}*/
|
||||
//如果文件小于50MB，也不走代理
|
||||
if (total.toLong() < 1024 * 1024 * 50L) {
|
||||
return proxy(url, headers)
|
||||
} else if (total.toLong() < 1024 * 1024 * 1024L * 10L) {
|
||||
//10GB以下
|
||||
threadNum = CORE_NUM * 4
|
||||
} else if (total.toLong() < 1024 * 1024 * 1024L * 40L) {
|
||||
//40GB以下
|
||||
threadNum = CORE_NUM * 8
|
||||
} else {
|
||||
//40GB以上
|
||||
threadNum = CORE_NUM * 16
|
||||
}
|
||||
var range =
|
||||
if (StringUtils.isAllBlank(headers["range"])) headers["Range"] else headers["range"]
|
||||
if (StringUtils.isAllBlank(range)) range = "bytes=0-";
|
||||
|
|
@ -77,11 +82,11 @@ object DownloadMT {
|
|||
)
|
||||
//没有range,无需分割
|
||||
|
||||
val partList = generatePart(rangeObj, total)
|
||||
val partList = generatePart(rangeObj, total, threadNum)
|
||||
|
||||
// 存储执行结果的List
|
||||
val jobs = mutableListOf<Job>()
|
||||
val channels = List(THREAD_NUM) { Channel<ByteArray>() }
|
||||
val channels = List(threadNum) { Channel<ByteArray>() }
|
||||
for ((index, part) in partList.withIndex()) {
|
||||
|
||||
|
||||
|
|
@ -146,11 +151,11 @@ object DownloadMT {
|
|||
/* respHeaders.put("Access-Control-Allow-Credentials", "true");
|
||||
respHeaders.put("Access-Control-Allow-Origin", "*");*/
|
||||
resHeader["Content-Length"] =
|
||||
(partList[THREAD_NUM - 1][1] - partList[0][0] + 1).toString()
|
||||
(partList[threadNum - 1][1] - partList[0][0] + 1).toString()
|
||||
resHeader.remove("content-length")
|
||||
|
||||
resHeader["Content-Range"] = String.format(
|
||||
"bytes %s-%s/%s", partList[0][0], partList[THREAD_NUM - 1][1], total
|
||||
"bytes %s-%s/%s", partList[0][0], partList[threadNum - 1][1], total
|
||||
)
|
||||
resHeader.remove("content-range")
|
||||
|
||||
|
|
@ -170,6 +175,43 @@ object DownloadMT {
|
|||
}
|
||||
}
|
||||
|
||||
private fun generatePart(
|
||||
rangeObj: Map<String?, String>, total: String, threadNum: Int
|
||||
): List<LongArray> {
|
||||
val totalSize = total.toLong()
|
||||
|
||||
//默认8MB
|
||||
var partSize = 1024 * 1024 * 8L
|
||||
|
||||
if (totalSize < 1024 * 1024 * 1024L * 10L) {
|
||||
//10GB以下，分片8MB
|
||||
partSize = 1024 * 1024 * 8L
|
||||
} else if (totalSize < 1024 * 1024 * 1024L * 40L) {
|
||||
//40GB以下，分片32MB
|
||||
partSize = 1024 * 1024 * 8L * 4
|
||||
} else {
|
||||
//40GB以上，分片128MB
|
||||
partSize = 1024 * 1024 * 8L * 4 * 4
|
||||
}
|
||||
|
||||
var start = rangeObj["start"]!!.toLong()
|
||||
var end =
|
||||
if (StringUtils.isAllBlank(rangeObj["end"])) start + partSize else rangeObj["end"]!!.toLong()
|
||||
|
||||
|
||||
end = min(end.toDouble(), (totalSize - 1).toDouble()).toLong()
|
||||
val length = end - start + 1
|
||||
|
||||
val size = length / threadNum
|
||||
val partList: MutableList<LongArray> = ArrayList()
|
||||
for (i in 0..<threadNum) {
|
||||
val partEnd = min((start + size).toDouble(), end.toDouble()).toLong()
|
||||
|
||||
partList.add(longArrayOf(start, partEnd))
|
||||
start = partEnd + 1
|
||||
}
|
||||
return partList
|
||||
}
|
||||
|
||||
private fun downloadRange(
|
||||
url: String, headerNew: MutableMap<String, String>
|
||||
|
|
|
|||
|
|
@ -100,28 +100,6 @@ public class ProxyVideo {
|
|||
return DownloadMT.INSTANCE.proxyMultiThread(url, headers);
|
||||
}
|
||||
|
||||
public static List<long[]> generatePart(Map<String, String> rangeObj, String total) {
|
||||
long totalSize = Long.parseLong(total);
|
||||
//超过10GB，分块是10Mb，不然是2MB
|
||||
long partSize = totalSize > 8L * 1024L * 1024L * 1024L * 10L ? 1024 * 1024 * 8 * 10L : 1024 * 1024 * 8 * 2L;
|
||||
|
||||
long start = Long.parseLong(rangeObj.get("start"));
|
||||
long end = StringUtils.isAllBlank(rangeObj.get("end")) ? start + partSize : Long.parseLong(rangeObj.get("end"));
|
||||
|
||||
|
||||
end = Math.min(end, totalSize - 1);
|
||||
long length = end - start + 1;
|
||||
|
||||
long size = length / THREAD_NUM;
|
||||
List<long[]> partList = new ArrayList<>();
|
||||
for (int i = 0; i < THREAD_NUM; i++) {
|
||||
long partEnd = Math.min(start + size, end);
|
||||
|
||||
partList.add(new long[]{start, partEnd});
|
||||
start = partEnd + 1;
|
||||
}
|
||||
return partList;
|
||||
}
|
||||
|
||||
public static Map<String, String> parseRange(String range) {
|
||||
SpiderDebug.log("parseRange:" + range);
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -1 +1 @@
|
|||
cd1d6a277a73915237cf1ecb19d5f46a
|
||||
a926c6e48bbffc73be39e17fcc6650a1
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"spider": "https://ghproxy.net/https://raw.githubusercontent.com/lushunming/AndroidCatVodSpider/multiThreadkt/jar/custom_spider.jar;md5;cd1d6a277a73915237cf1ecb19d5f46a",
|
||||
"spider": "https://ghproxy.net/https://raw.githubusercontent.com/lushunming/AndroidCatVodSpider/multiThreadkt/jar/custom_spider.jar;md5;a926c6e48bbffc73be39e17fcc6650a1",
|
||||
"lives": [
|
||||
{
|
||||
"name": "电视直播",
|
||||
|
|
|
|||
Loading…
Reference in New Issue