dongge
/
AndroidCatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean code

This commit is contained in:
FongMi 2023-02-11 18:47:27 +08:00
parent 0ae4f4d3ca
commit f1eb933dab
7 changed files with 1 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
app/src/main/java/com/github/catvod/parser/JsonBasic.java
View File

@ -1,65 +0,0 @@
package com.github.catvod.parser;
import android.util.Base64;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils;
import org.json.JSONObject;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Set;
public class JsonBasic {
public static JSONObject parse(LinkedHashMap<String, String> jx, String url) {
SpiderDebug.log("Load Json Parse Basic...");
if (jx.isEmpty()) return new JSONObject();
Set<String> jxNames = jx.keySet();
for (String jxName : jxNames) {
String parseUrl = jx.get(jxName);
HashMap<String, String> reqHeaders = getReqHeader(parseUrl);
try {
String realUrl = reqHeaders.get("url");
reqHeaders.remove("url");
SpiderDebug.log(realUrl + url);
String json = OkHttp.string(realUrl + url, reqHeaders);
JSONObject taskResult = Utils.jsonParse(url, json);
if (taskResult == null) continue;
taskResult.put("jxFrom", jxName);
SpiderDebug.log(taskResult.toString());
return taskResult;
} catch (Exception ignored) {
}
}
return new JSONObject();
}
public static HashMap<String, String> getReqHeader(String url) {
HashMap<String, String> reqHeaders = new HashMap<>();
reqHeaders.put("url", url);
if (!url.contains("cat_ext")) return reqHeaders;
try {
int start = url.indexOf("cat_ext=");
int end = url.indexOf("&", start);
String ext = url.substring(start + 8, end);
ext = new String(Base64.decode(ext, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP));
String newUrl = url.substring(0, start) + url.substring(end + 1);
JSONObject jsonObject = new JSONObject(ext);
if (jsonObject.has("header")) {
JSONObject headerJson = jsonObject.optJSONObject("header");
Iterator<String> keys = headerJson.keys();
while (keys.hasNext()) {
String key = keys.next();
reqHeaders.put(key, headerJson.optString(key, ""));
}
}
reqHeaders.put("url", newUrl);
} catch (Exception ignored) {
}
return reqHeaders;
}
}

71
app/src/main/java/com/github/catvod/parser/JsonParallel.java
View File

@ -1,71 +0,0 @@
package com.github.catvod.parser;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
public class JsonParallel {
private static final String ParseOKTag = "p_json_parse";
public static JSONObject parse(LinkedHashMap<String, String> jx, String url) {
if (jx.isEmpty()) return new JSONObject();
ExecutorService service = Executors.newFixedThreadPool(3);
CompletionService<JSONObject> completionService = new ExecutorCompletionService<>(service);
List<Future<JSONObject>> futures = new ArrayList<>();
Set<String> jxNames = jx.keySet();
for (String jxName : jxNames) {
String parseUrl = jx.get(jxName);
futures.add(completionService.submit(() -> {
try {
HashMap<String, String> reqHeaders = JsonBasic.getReqHeader(parseUrl);
String realUrl = reqHeaders.get("url");
reqHeaders.remove("url");
SpiderDebug.log(realUrl + url);
String json = OkHttp.string(realUrl + url, ParseOKTag, reqHeaders);
JSONObject taskResult = Utils.jsonParse(url, json);
taskResult.put("jxFrom", jxName);
SpiderDebug.log(taskResult.toString());
return taskResult;
} catch (Exception ignored) {
return null;
}
}));
}
JSONObject pTaskResult = null;
for (int i = 0; i < futures.size(); ++i) {
try {
Future<JSONObject> completed = completionService.take();
pTaskResult = completed.get();
if (pTaskResult != null) {
OkHttp.cancel(ParseOKTag);
for (int j = 0; j < futures.size(); j++) {
try {
futures.get(j).cancel(true);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
futures.clear();
break;
}
} catch (Exception ignored) {
}
}
service.shutdownNow();
return pTaskResult != null ? pTaskResult : new JSONObject();
}
}

35
app/src/main/java/com/github/catvod/parser/JsonSequence.java
View File

@ -1,35 +0,0 @@
package com.github.catvod.parser;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils;
import org.json.JSONObject;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Set;
public class JsonSequence {
public static JSONObject parse(LinkedHashMap<String, String> jx, String url) {
if (jx.isEmpty()) return new JSONObject();
Set<String> jxNames = jx.keySet();
for (String jxName : jxNames) {
try {
String parseUrl = jx.get(jxName);
HashMap<String, String> reqHeaders = JsonBasic.getReqHeader(parseUrl);
String realUrl = reqHeaders.get("url");
reqHeaders.remove("url");
SpiderDebug.log(realUrl + url);
String json = OkHttp.string(realUrl + url, reqHeaders);
JSONObject taskResult = Utils.jsonParse(url, json);
if (taskResult == null) continue;
taskResult.put("jxFrom", jxName);
return taskResult;
} catch (Exception ignored) {
}
}
return new JSONObject();
}
}

88
app/src/main/java/com/github/catvod/parser/MixDemo.java
View File

@ -1,88 +0,0 @@
package com.github.catvod.parser;
import android.util.Base64;
import org.json.JSONArray;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
public class MixDemo {
private static final HashMap<String, ArrayList<String>> flagWebJx = new HashMap<>();
private static HashMap<String, ArrayList<String>> configs = null;
private static void setConfigs(LinkedHashMap<String, HashMap<String, String>> jx) {
configs = new HashMap<>();
for (String key : jx.keySet()) {
HashMap<String, String> parseBean = jx.get(key);
String type = parseBean.get("type");
if (type.equals("1") || type.equals("0")) {
try {
JSONArray flags = new JSONObject(parseBean.get("ext")).getJSONArray("flag");
for (int j = 0; j < flags.length(); j++) {
String flagKey = flags.getString(j);
ArrayList<String> flagJx = configs.get(flagKey);
if (flagJx == null) {
flagJx = new ArrayList<>();
configs.put(flagKey, flagJx);
}
flagJx.add(key);
}
} catch (Exception ignored) {
}
}
}
}
public static JSONObject parse(LinkedHashMap<String, HashMap<String, String>> jx, String nameMe, String flag, String url) {
try {
if (configs == null) setConfigs(jx);
LinkedHashMap<String, String> jsonJx = new LinkedHashMap<>();
ArrayList<String> webJx = new ArrayList<>();
ArrayList<String> flagJx = configs.get(flag);
if (flagJx != null && !flagJx.isEmpty()) {
for (int i = 0; i < flagJx.size(); i++) {
String key = flagJx.get(i);
HashMap<String, String> parseBean = jx.get(key);
String type = parseBean.get("type");
if (type.equals("1")) {
jsonJx.put(key, mixUrl(parseBean.get("url"), parseBean.get("ext")));
} else if (type.equals("0")) {
webJx.add(parseBean.get("url"));
}
}
} else {
for (String key : jx.keySet()) {
HashMap<String, String> parseBean = jx.get(key);
String type = parseBean.get("type");
if (type.equals("1")) {
jsonJx.put(key, mixUrl(parseBean.get("url"), parseBean.get("ext")));
} else if (type.equals("0")) {
webJx.add(parseBean.get("url"));
}
}
}
if (!webJx.isEmpty()) flagWebJx.put(flag, webJx);
JSONObject jsonResult = JsonParallel.parse(jsonJx, url);
if (jsonResult.has("url")) return jsonResult;
if (!webJx.isEmpty()) {
JSONObject webResult = new JSONObject();
webResult.put("url", "proxy://do=parseMix&flag=" + flag + "&url=" + Base64.encodeToString(url.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP));
webResult.put("parse", 1);
return webResult;
}
} catch (Exception ignored) {
}
return new JSONObject();
}
private static String mixUrl(String url, String ext) {
if (ext.trim().isEmpty()) return url;
int index = url.indexOf("?");
if (index == -1) return url;
return url.substring(0, index + 1) + "cat_ext=" + Base64.encodeToString(ext.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP) + "&" + url.substring(index + 1);
}
}

36
app/src/main/java/com/github/catvod/utils/Utils.java
View File

@ -13,9 +13,6 @@ import android.webkit.WebViewClient;
import com.github.catvod.crawler.SpiderDebug; import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.spider.Init; import com.github.catvod.spider.Init;
import org.json.JSONException;
import org.json.JSONObject;
import java.math.BigInteger; import java.math.BigInteger;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.util.Arrays; import java.util.Arrays;
@ -72,39 +69,6 @@ public class Utils {
return src; return src;
} }
public static JSONObject fixJsonVodHeader(JSONObject headers, String input, String url) throws JSONException {
if (headers == null) headers = new JSONObject();
if (input.contains("www.mgtv.com")) {
headers.put("Referer", "");
headers.put("User-Agent", "Mozilla/5.0");
} else if (url.contains("titan.mgtv")) {
headers.put("Referer", "");
headers.put("User-Agent", "Mozilla/5.0");
} else if (input.contains("bilibili")) {
headers.put("Referer", "https://www.bilibili.com/");
headers.put("User-Agent", CHROME);
}
return headers;
}
public static JSONObject jsonParse(String input, String json) throws JSONException {
JSONObject jsonPlayData = new JSONObject(json);
String url = jsonPlayData.getString("url");
if (url.startsWith("//")) url = "https:" + url;
if (!url.startsWith("http")) return null;
if (url.equals(input)) if (isVip(url) || !isVideoFormat(url)) return null;
JSONObject headers = new JSONObject();
String ua = jsonPlayData.optString("user-agent", "");
if (ua.trim().length() > 0) headers.put("User-Agent", ua);
String referer = jsonPlayData.optString("referer", "");
if (referer.trim().length() > 0) headers.put("Referer", referer);
headers = Utils.fixJsonVodHeader(headers, input, url);
JSONObject taskResult = new JSONObject();
taskResult.put("header", headers);
taskResult.put("url", url);
return taskResult;
}
public static String substring(String text) { public static String substring(String text) {
return substring(text, 1); return substring(text, 1);
} }

BIN
jar/custom_spider.jar
View File

Binary file not shown.

2
jar/custom_spider.jar.md5
View File

@ -1 +1 @@
384628eab024a2a40a57835d2f0684d8 216774d6bcf8c060124d3562267c7d89