diff --git a/app/src/main/java/com/github/catvod/utils/DownloadMT.kt b/app/src/main/java/com/github/catvod/utils/DownloadMT.kt new file mode 100644 index 00000000..0c7d0add --- /dev/null +++ b/app/src/main/java/com/github/catvod/utils/DownloadMT.kt @@ -0,0 +1,150 @@ +package com.github.catvod.utils + +import com.github.catvod.crawler.SpiderDebug +import com.github.catvod.net.OkHttp +import com.github.catvod.utils.ProxyVideo.generatePart +import com.github.catvod.utils.ProxyVideo.getInfo +import com.github.catvod.utils.ProxyVideo.getMimeType +import com.github.catvod.utils.ProxyVideo.parseRange +import com.github.catvod.utils.ProxyVideo.proxy +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import okhttp3.Response +import org.apache.commons.lang3.StringUtils +import java.io.InputStream +import java.io.SequenceInputStream +import java.util.Vector + +object DownloadMT { + val THREAD_NUM: Int = Runtime.getRuntime().availableProcessors() * 2 + + private val infos = mutableMapOf>(); + fun proxyMultiThread(url: String, headers: Map): Array? { + + + /* val service = Executors.newFixedThreadPool(THREAD_NUM) + SpiderDebug.log("--proxyMultiThread: THREAD_NUM " + THREAD_NUM)*/ + + val `in`: SequenceInputStream? + try { + //缓存，避免每次都请求total等信息 + + + val info = infos[url] + if (info == null) { + infos.clear() + infos[url] = getInfo(url, headers) + } + + val code = info?.get(0) as Int + if (code != 206) { + return proxy(url, headers) + } + val resHeader = info[3] as MutableMap + val contentRange = + if (StringUtils.isAllBlank(resHeader["Content-Range"])) resHeader["content-range"] else resHeader["Content-Range"] + + SpiderDebug.log("--contentRange:$contentRange") + //文件总大小 + val total = StringUtils.split(contentRange, "/")[1] + SpiderDebug.log("--文件总大小:$total") + + //如果文件太小，也不走代理 + if (total.toLong() < 1024 * 1024 * 100) { + return proxy(url, headers) + } + var range = + if (StringUtils.isAllBlank(headers["range"])) headers["Range"] else headers["range"] + if (StringUtils.isAllBlank(range)) range = "bytes=0-"; + SpiderDebug.log("---proxyMultiThread,Range:$range") + val rangeObj = parseRange( + range!! + ) + //没有range,无需分割 + + val partList = generatePart(rangeObj, total) + + // 存储执行结果的List + val jobs = mutableListOf() + val channels = List(THREAD_NUM) { Channel() } + for ((index, part) in partList.withIndex()) { + + + val newRange = "bytes=" + part[0] + "-" + part[1] + SpiderDebug.log("下载开始;newRange:$newRange") + + val headerNew: MutableMap = HashMap(headers) + + headerNew["range"] = newRange + headerNew["Range"] = newRange + jobs += CoroutineScope(Dispatchers.IO).launch { + downloadRange(url, headerNew)?.let { channels[index].send(it) }; + } + + + } + + val inputStreams: MutableList = ArrayList() + CoroutineScope(Dispatchers.Default).launch { + repeat(jobs.size) { index -> + launch { + val response = channels[index].receive() + val body = response.body();/* int bytesRead; + while ((bytesRead = body.byteStream().read(bytes)) != -1) { + out.write(bytes, 0, bytesRead); + }*/ + inputStreams.add(body!!.byteStream()) + + + SpiderDebug.log("---第" + index + "块下载完成" + ";Content-Range:" + response.headers()["Content-Range"]) + SpiderDebug.log("---第" + index + "块下载完成" + ";content-range:" + response.headers()["content-range"]) + } + } + // 等待所有下载完成 + jobs.joinAll() + } + + `in` = SequenceInputStream(Vector(inputStreams).elements()) + + + // SpiderDebug.log(" ++proxy res data:" + Json.toJson(response.body())); + var contentType: String? = if (StringUtils.isAllBlank( + resHeader["Content-Type"] + ) + ) resHeader["content-type"] else resHeader["Content-Type"] + val contentDisposition: String = resHeader["Content-Disposition"]!!.toString() + if (StringUtils.isAllBlank(contentType)&& StringUtils.isNoneBlank(contentDisposition)) { + contentType = getMimeType(contentDisposition) + } + + + /* respHeaders.put("Access-Control-Allow-Credentials", "true"); + respHeaders.put("Access-Control-Allow-Origin", "*");*/ + resHeader["Content-Length"] = + (partList[THREAD_NUM - 1][1] - partList[0][0] + 1).toString() + // respHeaders.put("content-length", String.valueOf(bytes.length)); + resHeader["Content-Range"] = String.format( + "bytes %s-%s/%s", partList[0][0], partList[THREAD_NUM - 1][1], total + ) + // respHeaders.put("content-range", String.format("bytes %s-%s/%s", partList.get(0)[0], partList.get(THREAD_NUM - 1)[1], total)); + SpiderDebug.log("++proxy res contentType:$contentType") + // SpiderDebug.log("++proxy res body:" + response.body()); + SpiderDebug.log("++proxy res respHeaders:" + Json.toJson(resHeader)) + return arrayOf(206, contentType, `in`, resHeader) + + } catch (e: Exception) { + SpiderDebug.log("proxyMultiThread error:" + e.message) + e.printStackTrace() + return null + } + } + + + private fun downloadRange( + url: String, headerNew: MutableMap + ): Response? = OkHttp.newCall(url, headerNew) +} \ No newline at end of file diff --git a/jar/custom_spider.jar b/jar/custom_spider.jar index e4245e10..cdca7251 100644 Binary files a/jar/custom_spider.jar and b/jar/custom_spider.jar differ diff --git a/jar/custom_spider.jar.md5 b/jar/custom_spider.jar.md5 index be1bea08..64e60eca 100644 --- a/jar/custom_spider.jar.md5 +++ b/jar/custom_spider.jar.md5 @@ -1 +1 @@ -860afdcd1a39fd5c0bcfae2b0652ec7c +2d18b21a13a3e7b982f1d1642d975c9b diff --git a/json/test.json b/json/test.json index 29f7033f..e9cccc2c 100644 --- a/json/test.json +++ b/json/test.json @@ -1,5 +1,5 @@ { - "spider": "https://gh-proxy.com/https://raw.githubusercontent.com/lushunming/AndroidCatVodSpider/multiThreadkt/jar/custom_spider.jar;md5;860afdcd1a39fd5c0bcfae2b0652ec7c", + "spider": "https://gh-proxy.com/https://raw.githubusercontent.com/lushunming/AndroidCatVodSpider/multiThreadkt/jar/custom_spider.jar;md5;2d18b21a13a3e7b982f1d1642d975c9b", "lives": [ { "name": "电视直播",