diff --git a/app/src/main/java/com/github/catvod/parser/JsonBasic.java b/app/src/main/java/com/github/catvod/parser/JsonBasic.java deleted file mode 100644 index 2f5b204c..00000000 --- a/app/src/main/java/com/github/catvod/parser/JsonBasic.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.github.catvod.parser; - -import android.util.Base64; - -import com.github.catvod.crawler.SpiderDebug; -import com.github.catvod.net.OkHttp; -import com.github.catvod.utils.Utils; - -import org.json.JSONObject; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Set; - -public class JsonBasic { - - public static JSONObject parse(LinkedHashMap jx, String url) { - SpiderDebug.log("Load Json Parse Basic..."); - if (jx.isEmpty()) return new JSONObject(); - Set jxNames = jx.keySet(); - for (String jxName : jxNames) { - String parseUrl = jx.get(jxName); - HashMap reqHeaders = getReqHeader(parseUrl); - try { - String realUrl = reqHeaders.get("url"); - reqHeaders.remove("url"); - SpiderDebug.log(realUrl + url); - String json = OkHttp.string(realUrl + url, reqHeaders); - JSONObject taskResult = Utils.jsonParse(url, json); - if (taskResult == null) continue; - taskResult.put("jxFrom", jxName); - SpiderDebug.log(taskResult.toString()); - return taskResult; - } catch (Exception ignored) { - } - } - return new JSONObject(); - } - - public static HashMap getReqHeader(String url) { - HashMap reqHeaders = new HashMap<>(); - reqHeaders.put("url", url); - if (!url.contains("cat_ext")) return reqHeaders; - try { - int start = url.indexOf("cat_ext="); - int end = url.indexOf("&", start); - String ext = url.substring(start + 8, end); - ext = new String(Base64.decode(ext, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP)); - String newUrl = url.substring(0, start) + url.substring(end + 1); - JSONObject jsonObject = new JSONObject(ext); - if (jsonObject.has("header")) { - JSONObject headerJson = jsonObject.optJSONObject("header"); - Iterator keys = headerJson.keys(); - while (keys.hasNext()) { - String key = keys.next(); - reqHeaders.put(key, headerJson.optString(key, "")); - } - } - reqHeaders.put("url", newUrl); - } catch (Exception ignored) { - } - return reqHeaders; - } -} diff --git a/app/src/main/java/com/github/catvod/parser/JsonParallel.java b/app/src/main/java/com/github/catvod/parser/JsonParallel.java deleted file mode 100644 index d206e89e..00000000 --- a/app/src/main/java/com/github/catvod/parser/JsonParallel.java +++ /dev/null @@ -1,71 +0,0 @@ -package com.github.catvod.parser; - -import com.github.catvod.crawler.SpiderDebug; -import com.github.catvod.net.OkHttp; -import com.github.catvod.utils.Utils; - -import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CompletionService; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -public class JsonParallel { - - private static final String ParseOKTag = "p_json_parse"; - - public static JSONObject parse(LinkedHashMap jx, String url) { - if (jx.isEmpty()) return new JSONObject(); - ExecutorService service = Executors.newFixedThreadPool(3); - CompletionService completionService = new ExecutorCompletionService<>(service); - List> futures = new ArrayList<>(); - Set jxNames = jx.keySet(); - for (String jxName : jxNames) { - String parseUrl = jx.get(jxName); - futures.add(completionService.submit(() -> { - try { - HashMap reqHeaders = JsonBasic.getReqHeader(parseUrl); - String realUrl = reqHeaders.get("url"); - reqHeaders.remove("url"); - SpiderDebug.log(realUrl + url); - String json = OkHttp.string(realUrl + url, ParseOKTag, reqHeaders); - JSONObject taskResult = Utils.jsonParse(url, json); - taskResult.put("jxFrom", jxName); - SpiderDebug.log(taskResult.toString()); - return taskResult; - } catch (Exception ignored) { - return null; - } - })); - } - JSONObject pTaskResult = null; - for (int i = 0; i < futures.size(); ++i) { - try { - Future completed = completionService.take(); - pTaskResult = completed.get(); - if (pTaskResult != null) { - OkHttp.cancel(ParseOKTag); - for (int j = 0; j < futures.size(); j++) { - try { - futures.get(j).cancel(true); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - futures.clear(); - break; - } - } catch (Exception ignored) { - } - } - service.shutdownNow(); - return pTaskResult != null ? pTaskResult : new JSONObject(); - } -} diff --git a/app/src/main/java/com/github/catvod/parser/JsonSequence.java b/app/src/main/java/com/github/catvod/parser/JsonSequence.java deleted file mode 100644 index c82a438d..00000000 --- a/app/src/main/java/com/github/catvod/parser/JsonSequence.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.github.catvod.parser; - -import com.github.catvod.crawler.SpiderDebug; -import com.github.catvod.net.OkHttp; -import com.github.catvod.utils.Utils; - -import org.json.JSONObject; - -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Set; - -public class JsonSequence { - - public static JSONObject parse(LinkedHashMap jx, String url) { - if (jx.isEmpty()) return new JSONObject(); - Set jxNames = jx.keySet(); - for (String jxName : jxNames) { - try { - String parseUrl = jx.get(jxName); - HashMap reqHeaders = JsonBasic.getReqHeader(parseUrl); - String realUrl = reqHeaders.get("url"); - reqHeaders.remove("url"); - SpiderDebug.log(realUrl + url); - String json = OkHttp.string(realUrl + url, reqHeaders); - JSONObject taskResult = Utils.jsonParse(url, json); - if (taskResult == null) continue; - taskResult.put("jxFrom", jxName); - return taskResult; - } catch (Exception ignored) { - } - } - return new JSONObject(); - } -} diff --git a/app/src/main/java/com/github/catvod/parser/MixDemo.java b/app/src/main/java/com/github/catvod/parser/MixDemo.java deleted file mode 100644 index 2fec652d..00000000 --- a/app/src/main/java/com/github/catvod/parser/MixDemo.java +++ /dev/null @@ -1,88 +0,0 @@ -package com.github.catvod.parser; - -import android.util.Base64; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; - -public class MixDemo { - - private static final HashMap> flagWebJx = new HashMap<>(); - private static HashMap> configs = null; - - private static void setConfigs(LinkedHashMap> jx) { - configs = new HashMap<>(); - for (String key : jx.keySet()) { - HashMap parseBean = jx.get(key); - String type = parseBean.get("type"); - if (type.equals("1") || type.equals("0")) { - try { - JSONArray flags = new JSONObject(parseBean.get("ext")).getJSONArray("flag"); - for (int j = 0; j < flags.length(); j++) { - String flagKey = flags.getString(j); - ArrayList flagJx = configs.get(flagKey); - if (flagJx == null) { - flagJx = new ArrayList<>(); - configs.put(flagKey, flagJx); - } - flagJx.add(key); - } - } catch (Exception ignored) { - } - } - } - } - - public static JSONObject parse(LinkedHashMap> jx, String nameMe, String flag, String url) { - try { - if (configs == null) setConfigs(jx); - LinkedHashMap jsonJx = new LinkedHashMap<>(); - ArrayList webJx = new ArrayList<>(); - ArrayList flagJx = configs.get(flag); - if (flagJx != null && !flagJx.isEmpty()) { - for (int i = 0; i < flagJx.size(); i++) { - String key = flagJx.get(i); - HashMap parseBean = jx.get(key); - String type = parseBean.get("type"); - if (type.equals("1")) { - jsonJx.put(key, mixUrl(parseBean.get("url"), parseBean.get("ext"))); - } else if (type.equals("0")) { - webJx.add(parseBean.get("url")); - } - } - } else { - for (String key : jx.keySet()) { - HashMap parseBean = jx.get(key); - String type = parseBean.get("type"); - if (type.equals("1")) { - jsonJx.put(key, mixUrl(parseBean.get("url"), parseBean.get("ext"))); - } else if (type.equals("0")) { - webJx.add(parseBean.get("url")); - } - } - } - if (!webJx.isEmpty()) flagWebJx.put(flag, webJx); - JSONObject jsonResult = JsonParallel.parse(jsonJx, url); - if (jsonResult.has("url")) return jsonResult; - if (!webJx.isEmpty()) { - JSONObject webResult = new JSONObject(); - webResult.put("url", "proxy://do=parseMix&flag=" + flag + "&url=" + Base64.encodeToString(url.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP)); - webResult.put("parse", 1); - return webResult; - } - } catch (Exception ignored) { - } - return new JSONObject(); - } - - private static String mixUrl(String url, String ext) { - if (ext.trim().isEmpty()) return url; - int index = url.indexOf("?"); - if (index == -1) return url; - return url.substring(0, index + 1) + "cat_ext=" + Base64.encodeToString(ext.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP) + "&" + url.substring(index + 1); - } -} diff --git a/app/src/main/java/com/github/catvod/utils/Utils.java b/app/src/main/java/com/github/catvod/utils/Utils.java index 487201d5..1b886517 100644 --- a/app/src/main/java/com/github/catvod/utils/Utils.java +++ b/app/src/main/java/com/github/catvod/utils/Utils.java @@ -13,9 +13,6 @@ import android.webkit.WebViewClient; import com.github.catvod.crawler.SpiderDebug; import com.github.catvod.spider.Init; -import org.json.JSONException; -import org.json.JSONObject; - import java.math.BigInteger; import java.security.MessageDigest; import java.util.Arrays; @@ -72,39 +69,6 @@ public class Utils { return src; } - public static JSONObject fixJsonVodHeader(JSONObject headers, String input, String url) throws JSONException { - if (headers == null) headers = new JSONObject(); - if (input.contains("www.mgtv.com")) { - headers.put("Referer", ""); - headers.put("User-Agent", "Mozilla/5.0"); - } else if (url.contains("titan.mgtv")) { - headers.put("Referer", ""); - headers.put("User-Agent", "Mozilla/5.0"); - } else if (input.contains("bilibili")) { - headers.put("Referer", "https://www.bilibili.com/"); - headers.put("User-Agent", CHROME); - } - return headers; - } - - public static JSONObject jsonParse(String input, String json) throws JSONException { - JSONObject jsonPlayData = new JSONObject(json); - String url = jsonPlayData.getString("url"); - if (url.startsWith("//")) url = "https:" + url; - if (!url.startsWith("http")) return null; - if (url.equals(input)) if (isVip(url) || !isVideoFormat(url)) return null; - JSONObject headers = new JSONObject(); - String ua = jsonPlayData.optString("user-agent", ""); - if (ua.trim().length() > 0) headers.put("User-Agent", ua); - String referer = jsonPlayData.optString("referer", ""); - if (referer.trim().length() > 0) headers.put("Referer", referer); - headers = Utils.fixJsonVodHeader(headers, input, url); - JSONObject taskResult = new JSONObject(); - taskResult.put("header", headers); - taskResult.put("url", url); - return taskResult; - } - public static String substring(String text) { return substring(text, 1); } diff --git a/jar/custom_spider.jar b/jar/custom_spider.jar index 4b38a56d..fb72eb55 100644 Binary files a/jar/custom_spider.jar and b/jar/custom_spider.jar differ diff --git a/jar/custom_spider.jar.md5 b/jar/custom_spider.jar.md5 index 6457a4d9..3a964aef 100644 --- a/jar/custom_spider.jar.md5 +++ b/jar/custom_spider.jar.md5 @@ -1 +1 @@ -384628eab024a2a40a57835d2f0684d8 +216774d6bcf8c060124d3562267c7d89