From c9c5acf0607c8ab4156885b7690d0a18d55ad1f7 Mon Sep 17 00:00:00 2001 From: FongMi Date: Mon, 22 Aug 2022 17:22:01 +0800 Subject: [PATCH] Initial project - part 2 --- .../com/github/catvod/crawler/Spider.java | 40 + .../github/catvod/crawler/SpiderDebug.java | 16 + .../com/github/catvod/live/TxtSubscribe.java | 126 ++++ .../com/github/catvod/net/OKCallBack.java | 76 ++ .../java/com/github/catvod/net/OKRequest.java | 113 +++ .../com/github/catvod/net/OkHttpUtil.java | 154 ++++ .../catvod/net/SSLSocketFactoryCompat.java | 112 +++ .../com/github/catvod/parser/JsonBasic.java | 65 ++ .../github/catvod/parser/JsonParallel.java | 71 ++ .../github/catvod/parser/JsonSequence.java | 35 + .../com/github/catvod/parser/MixDemo.java | 88 +++ .../java/com/github/catvod/spider/Init.java | 31 + .../java/com/github/catvod/spider/Proxy.java | 48 ++ .../java/com/github/catvod/utils/Misc.java | 128 ++++ .../java/com/github/catvod/utils/Trans.java | 23 + .../com/github/catvod/xpath/XPathRule.java | 686 ++++++++++++++++++ 16 files changed, 1812 insertions(+) create mode 100644 app/src/main/java/com/github/catvod/crawler/Spider.java create mode 100644 app/src/main/java/com/github/catvod/crawler/SpiderDebug.java create mode 100644 app/src/main/java/com/github/catvod/live/TxtSubscribe.java create mode 100644 app/src/main/java/com/github/catvod/net/OKCallBack.java create mode 100644 app/src/main/java/com/github/catvod/net/OKRequest.java create mode 100644 app/src/main/java/com/github/catvod/net/OkHttpUtil.java create mode 100644 app/src/main/java/com/github/catvod/net/SSLSocketFactoryCompat.java create mode 100644 app/src/main/java/com/github/catvod/parser/JsonBasic.java create mode 100644 app/src/main/java/com/github/catvod/parser/JsonParallel.java create mode 100644 app/src/main/java/com/github/catvod/parser/JsonSequence.java create mode 100644 app/src/main/java/com/github/catvod/parser/MixDemo.java create mode 100644 app/src/main/java/com/github/catvod/spider/Init.java create mode 100644 app/src/main/java/com/github/catvod/spider/Proxy.java create mode 100644 app/src/main/java/com/github/catvod/utils/Misc.java create mode 100644 app/src/main/java/com/github/catvod/utils/Trans.java create mode 100644 app/src/main/java/com/github/catvod/xpath/XPathRule.java diff --git a/app/src/main/java/com/github/catvod/crawler/Spider.java b/app/src/main/java/com/github/catvod/crawler/Spider.java new file mode 100644 index 00000000..386ffd18 --- /dev/null +++ b/app/src/main/java/com/github/catvod/crawler/Spider.java @@ -0,0 +1,40 @@ +package com.github.catvod.crawler; + +import android.content.Context; + +import java.util.HashMap; +import java.util.List; + +public abstract class Spider { + + public void init(Context context) { + } + + public void init(Context context, String extend) { + init(context); + } + + public String homeContent(boolean filter) { + return ""; + } + + public String homeVideoContent() { + return ""; + } + + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + return ""; + } + + public String detailContent(List ids) { + return ""; + } + + public String searchContent(String key, boolean quick) { + return ""; + } + + public String playerContent(String flag, String id, List vipFlags) { + return ""; + } +} diff --git a/app/src/main/java/com/github/catvod/crawler/SpiderDebug.java b/app/src/main/java/com/github/catvod/crawler/SpiderDebug.java new file mode 100644 index 00000000..12b2d6d7 --- /dev/null +++ b/app/src/main/java/com/github/catvod/crawler/SpiderDebug.java @@ -0,0 +1,16 @@ +package com.github.catvod.crawler; + +import android.util.Log; + +public class SpiderDebug { + + private static final String TAG = SpiderDebug.class.getSimpleName(); + + public static void log(Throwable th) { + Log.d(TAG, th.getMessage()); + } + + public static void log(String msg) { + Log.d(TAG, msg); + } +} diff --git a/app/src/main/java/com/github/catvod/live/TxtSubscribe.java b/app/src/main/java/com/github/catvod/live/TxtSubscribe.java new file mode 100644 index 00000000..98e1e2b6 --- /dev/null +++ b/app/src/main/java/com/github/catvod/live/TxtSubscribe.java @@ -0,0 +1,126 @@ +package com.github.catvod.live; + +import com.github.catvod.net.OkHttpUtil; +import com.github.catvod.utils.Trans; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; + +public class TxtSubscribe { + + public static void subscribe(LinkedHashMap>> allLives, String url, HashMap headers) { + String content = OkHttpUtil.string(url, headers); + parse(allLives, content); + } + + public static void parse(LinkedHashMap>> allLives, String txt) { + try { + BufferedReader br = new BufferedReader(new StringReader(txt)); + String line = br.readLine(); + LinkedHashMap> noGroup = new LinkedHashMap<>(); + LinkedHashMap> groupLives = noGroup; + while (line != null) { + if (line.trim().isEmpty()) { + line = br.readLine(); + continue; + } + String[] lineInfo = line.split(","); + if (lineInfo.length < 2) { + line = br.readLine(); + continue; + } + if (line.contains("#genre#")) { + String group = Trans.get(lineInfo[0].trim()); + if (!allLives.containsKey(group)) { + groupLives = new LinkedHashMap<>(); + allLives.put(group, groupLives); + } else { + groupLives = allLives.get(group); + } + } else { + String title = Trans.get(lineInfo[0].trim()); + String[] urlMix = lineInfo[1].trim().split("#"); + for (String mix : urlMix) { + String url = mix.trim(); + if (url.isEmpty()) continue; + if (url.startsWith("http") || url.startsWith("rtsp") || url.startsWith("rtmp")) { + ArrayList urls; + if (!groupLives.containsKey(title)) { + urls = new ArrayList<>(); + groupLives.put(title, urls); + } else { + urls = groupLives.get(title); + } + if (!urls.contains(url)) urls.add(url); + } + } + } + line = br.readLine(); + } + br.close(); + if (!noGroup.isEmpty()) { + allLives.put("未分類", noGroup); + } + } catch (Throwable ignored) { + + } + } + + public static String live2Json(LinkedHashMap>> lives) { + JSONArray groups = new JSONArray(); + for (String group : lives.keySet()) { + JSONArray channels = new JSONArray(); + LinkedHashMap> allChannel = lives.get(group); + if (allChannel.isEmpty()) continue; + for (String channel : allChannel.keySet()) { + ArrayList allUrls = allChannel.get(channel); + if (allUrls.isEmpty()) continue; + JSONArray urls = new JSONArray(); + for (int i = 0; i < allUrls.size(); i++) { + urls.put(allUrls.get(i)); + } + JSONObject newChannel = new JSONObject(); + try { + newChannel.put("name", channel); + newChannel.put("urls", urls); + } catch (JSONException ignored) { + } + channels.put(newChannel); + } + JSONObject newGroup = new JSONObject(); + try { + newGroup.put("group", group); + newGroup.put("channels", channels); + } catch (JSONException ignored) { + } + groups.put(newGroup); + } + return groups.toString(); + } + + public static Object[] load(String ext) { + try { + LinkedHashMap>> allLives = new LinkedHashMap<>(); + TxtSubscribe.subscribe(allLives, ext, null); + String json = TxtSubscribe.live2Json(allLives); + Object[] result = new Object[3]; + result[0] = 200; + result[1] = "text/plain; charset=utf-8"; + ByteArrayInputStream baos = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)); + result[2] = baos; + return result; + } catch (Throwable th) { + th.printStackTrace(); + } + return null; + } +} diff --git a/app/src/main/java/com/github/catvod/net/OKCallBack.java b/app/src/main/java/com/github/catvod/net/OKCallBack.java new file mode 100644 index 00000000..83755877 --- /dev/null +++ b/app/src/main/java/com/github/catvod/net/OKCallBack.java @@ -0,0 +1,76 @@ +package com.github.catvod.net; + +import java.io.IOException; + +import okhttp3.Call; +import okhttp3.Response; + +public abstract class OKCallBack { + + private T result = null; + + public T getResult() { + return result; + } + + protected void setResult(T val) { + result = val; + } + + protected void onError(final Call call, final Exception e) { + onFailure(call, e); + } + + protected void onSuccess(Call call, Response response) { + T obj = onParseResponse(call, response); + setResult(obj); + onResponse(obj); + } + + protected abstract T onParseResponse(Call call, Response response); + + protected void onFailure(Call call, Exception e) { + } + + protected abstract void onResponse(T response); + + public static class OKCallBackDefault extends OKCallBack { + + @Override + public Response onParseResponse(Call call, Response response) { + return response; + } + + @Override + protected void onResponse(Response response) { + } + } + + public static class OKCallBackString extends OKCallBack { + + @Override + public void onError(Call call, Exception e) { + setResult(""); + super.onError(call, e); + } + + @Override + public String onParseResponse(Call call, Response response) { + try { + return response.body().string(); + } catch (IOException e) { + return ""; + } + } + + @Override + protected void onFailure(Call call, Exception e) { + setResult(""); + super.onFailure(call, e); + } + + @Override + protected void onResponse(String response) { + } + } +} diff --git a/app/src/main/java/com/github/catvod/net/OKRequest.java b/app/src/main/java/com/github/catvod/net/OKRequest.java new file mode 100644 index 00000000..425801d0 --- /dev/null +++ b/app/src/main/java/com/github/catvod/net/OKRequest.java @@ -0,0 +1,113 @@ +package com.github.catvod.net; + +import android.text.TextUtils; + +import java.io.IOException; +import java.util.Map; + +import okhttp3.Call; +import okhttp3.FormBody; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.RequestBody; +import okhttp3.Response; + +class OKRequest { + + private final String mMethodType; + private final Map mParamsMap; + private final String mJsonStr; + private final Map mHeaderMap; + private final OKCallBack mCallBack; + private String mUrl; + private Object mTag = null; + private okhttp3.Request mOkHttpRequest; + private okhttp3.Request.Builder mRequestBuilder; + + OKRequest(String methodType, String url, Map paramsMap, Map headerMap, OKCallBack callBack) { + this(methodType, url, null, paramsMap, headerMap, callBack); + } + + OKRequest(String methodType, String url, String jsonStr, Map headerMap, OKCallBack callBack) { + this(methodType, url, jsonStr, null, headerMap, callBack); + } + + private OKRequest(String methodType, String url, String jsonStr, Map paramsMap, Map headerMap, OKCallBack callBack) { + mMethodType = methodType; + mUrl = url; + mJsonStr = jsonStr; + mParamsMap = paramsMap; + mHeaderMap = headerMap; + mCallBack = callBack; + getInstance(); + } + + public void setTag(Object tag) { + mTag = tag; + } + + private void getInstance() { + mRequestBuilder = new okhttp3.Request.Builder(); + switch (mMethodType) { + case OkHttpUtil.METHOD_GET: + setGetParams(); + break; + case OkHttpUtil.METHOD_POST: + mRequestBuilder.post(getRequestBody()); + break; + } + mRequestBuilder.url(mUrl); + if (mTag != null) + mRequestBuilder.tag(mTag); + if (mHeaderMap != null) { + setHeader(); + } + mOkHttpRequest = mRequestBuilder.build(); + } + + private RequestBody getRequestBody() { + if (!TextUtils.isEmpty(mJsonStr)) { + MediaType JSON = MediaType.parse("application/json; charset=utf-8"); + return RequestBody.create(JSON, mJsonStr); + } + FormBody.Builder formBody = new FormBody.Builder(); + if (mParamsMap != null) { + for (String key : mParamsMap.keySet()) { + formBody.add(key, mParamsMap.get(key)); + } + } + return formBody.build(); + } + + private void setGetParams() { + if (mParamsMap != null) { + mUrl = mUrl + "?"; + for (String key : mParamsMap.keySet()) { + mUrl = mUrl + key + "=" + mParamsMap.get(key) + "&"; + } + mUrl = mUrl.substring(0, mUrl.length() - 1); + } + } + + private void setHeader() { + if (mHeaderMap != null) { + for (String key : mHeaderMap.keySet()) { + mRequestBuilder.addHeader(key, mHeaderMap.get(key)); + } + } + } + + void execute(OkHttpClient client) { + Call call = client.newCall(mOkHttpRequest); + try { + Response response = call.execute(); + if (mCallBack != null) { + mCallBack.onSuccess(call, response); + } + } catch (IOException e) { + if (mCallBack != null) { + mCallBack.onError(call, e); + } + } + } +} diff --git a/app/src/main/java/com/github/catvod/net/OkHttpUtil.java b/app/src/main/java/com/github/catvod/net/OkHttpUtil.java new file mode 100644 index 00000000..1bf00117 --- /dev/null +++ b/app/src/main/java/com/github/catvod/net/OkHttpUtil.java @@ -0,0 +1,154 @@ +package com.github.catvod.net; + +import com.github.catvod.crawler.Spider; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import okhttp3.Call; +import okhttp3.Dns; +import okhttp3.OkHttpClient; +import okhttp3.Response; + +public class OkHttpUtil { + + public static final String METHOD_GET = "GET"; + public static final String METHOD_POST = "POST"; + + private static final Object lockO = new Object(); + private static OkHttpClient defaultClient = null; + private static OkHttpClient noRedirectClient = null; + private static final int DEFAULT_TIMEOUT = 15; + + public static OkHttpClient defaultClient() { + synchronized (lockO) { + if (defaultClient == null) { + OkHttpClient.Builder builder = new OkHttpClient.Builder().dns(safeDns()).readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).retryOnConnectionFailure(true).sslSocketFactory(new SSLSocketFactoryCompat(SSLSocketFactoryCompat.trustAllCert), SSLSocketFactoryCompat.trustAllCert); + defaultClient = builder.build(); + } + return defaultClient; + } + } + + public static OkHttpClient noRedirectClient() { + synchronized (lockO) { + if (noRedirectClient == null) { + OkHttpClient.Builder builder = new OkHttpClient.Builder().dns(safeDns()).readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).followRedirects(false).followSslRedirects(false).retryOnConnectionFailure(true).sslSocketFactory(new SSLSocketFactoryCompat(SSLSocketFactoryCompat.trustAllCert), SSLSocketFactoryCompat.trustAllCert); + noRedirectClient = builder.build(); + } + return noRedirectClient; + } + } + + public static Dns safeDns() { + try { + return (Dns) Spider.class.getMethod("safeDns").invoke(null); + } catch (Exception e) { + return Dns.SYSTEM; + } + } + + public static String string(OkHttpClient client, String url, String tag, Map paramsMap, Map headerMap, Map> respHeaderMap) { + return string(client, url, tag, paramsMap, headerMap, respHeaderMap, OkHttpUtil.METHOD_GET); + } + + public static String string(OkHttpClient client, String url, String tag, Map paramsMap, Map headerMap, Map> respHeaderMap, String httpMethod) { + OKCallBack.OKCallBackString callback = new OKCallBack.OKCallBackString() { + @Override + public String onParseResponse(Call call, Response response) { + try { + if (respHeaderMap != null) { + respHeaderMap.clear(); + respHeaderMap.putAll(response.headers().toMultimap()); + } + return response.body().string(); + } catch (IOException e) { + return ""; + } + } + }; + OKRequest req = new OKRequest(httpMethod, url, paramsMap, headerMap, callback); + req.setTag(tag); + req.execute(client); + return callback.getResult(); + } + + public static String stringNoRedirect(String url, Map headerMap, Map> respHeaderMap) { + return string(noRedirectClient(), url, null, null, headerMap, respHeaderMap); + } + + public static String string(String url) { + return string(defaultClient(), url, null, null, null, null); + } + + public static String string(String url, Map headerMap) { + return string(defaultClient(), url, null, null, headerMap, null); + } + + public static String string(String url, Map headerMap, Map> respHeaderMap) { + return string(defaultClient(), url, null, null, headerMap, respHeaderMap); + } + + public static String string(String url, String tag, Map headerMap) { + return string(defaultClient(), url, tag, null, headerMap, null); + } + + public static void get(OkHttpClient client, String url, OKCallBack callBack) { + get(client, url, null, null, callBack); + } + + public static void get(OkHttpClient client, String url, Map paramsMap, OKCallBack callBack) { + get(client, url, paramsMap, null, callBack); + } + + public static void get(OkHttpClient client, String url, Map paramsMap, Map headerMap, OKCallBack callBack) { + new OKRequest(METHOD_GET, url, paramsMap, headerMap, callBack).execute(client); + } + + public static String post(String url) { + return post(url, null); + } + + public static String post(String url, Map paramsMap) { + return post(url, paramsMap, null); + } + + public static String post(String url, Map paramsMap, Map headerMap) { + OKCallBack.OKCallBackString callback = new OKCallBack.OKCallBackString(); + new OKRequest(METHOD_POST, url, paramsMap, headerMap, callback).execute(defaultClient()); + return callback.getResult(); + } + + public static String postJson(String url, String jsonStr, Map headerMap) { + OKCallBack.OKCallBackString callback = new OKCallBack.OKCallBackString(); + new OKRequest(METHOD_POST, url, jsonStr, headerMap, callback).execute(defaultClient()); + return callback.getResult(); + } + + public static void cancel(OkHttpClient client, Object tag) { + if (client == null || tag == null) return; + for (Call call : client.dispatcher().queuedCalls()) { + if (tag.equals(call.request().tag())) { + call.cancel(); + } + } + for (Call call : client.dispatcher().runningCalls()) { + if (tag.equals(call.request().tag())) { + call.cancel(); + } + } + } + + public static void cancel(Object tag) { + cancel(defaultClient(), tag); + } + + public static String getRedirectLocation(Map> headers) { + if (headers == null) return null; + if (headers.containsKey("location")) return headers.get("location").get(0); + if (headers.containsKey("Location")) return headers.get("Location").get(0); + return null; + } +} diff --git a/app/src/main/java/com/github/catvod/net/SSLSocketFactoryCompat.java b/app/src/main/java/com/github/catvod/net/SSLSocketFactoryCompat.java new file mode 100644 index 00000000..d84ca0ac --- /dev/null +++ b/app/src/main/java/com/github/catvod/net/SSLSocketFactoryCompat.java @@ -0,0 +1,112 @@ +package com.github.catvod.net; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.security.GeneralSecurityException; +import java.security.cert.X509Certificate; +import java.util.LinkedList; +import java.util.List; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.X509TrustManager; + +public class SSLSocketFactoryCompat extends SSLSocketFactory { + + public static final X509TrustManager trustAllCert = new X509TrustManager() { + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) { + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[]{}; + } + }; + + static String[] protocols = null; + static String[] cipherSuites = null; + + static { + try { + SSLSocket socket = (SSLSocket) SSLSocketFactory.getDefault().createSocket(); + if (socket != null) { + List protocols = new LinkedList<>(); + for (String protocol : socket.getSupportedProtocols()) if (!protocol.toUpperCase().contains("SSL")) protocols.add(protocol); + SSLSocketFactoryCompat.protocols = protocols.toArray(new String[protocols.size()]); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private final SSLSocketFactory defaultFactory; + + public SSLSocketFactoryCompat(X509TrustManager tm) { + try { + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, (tm != null) ? new X509TrustManager[]{tm} : null, null); + defaultFactory = sslContext.getSocketFactory(); + } catch (GeneralSecurityException e) { + throw new AssertionError(); // The system has no TLS. Just give up. + } + } + + private void upgradeTLS(SSLSocket ssl) { + if (protocols != null) { + ssl.setEnabledProtocols(protocols); + } + } + + @Override + public String[] getDefaultCipherSuites() { + return cipherSuites; + } + + @Override + public String[] getSupportedCipherSuites() { + return cipherSuites; + } + + @Override + public Socket createSocket(Socket s, String host, int port, boolean autoClose) throws IOException { + Socket ssl = defaultFactory.createSocket(s, host, port, autoClose); + if (ssl instanceof SSLSocket) upgradeTLS((SSLSocket) ssl); + return ssl; + } + + @Override + public Socket createSocket(String host, int port) throws IOException { + Socket ssl = defaultFactory.createSocket(host, port); + if (ssl instanceof SSLSocket) upgradeTLS((SSLSocket) ssl); + return ssl; + } + + @Override + public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException { + Socket ssl = defaultFactory.createSocket(host, port, localHost, localPort); + if (ssl instanceof SSLSocket) upgradeTLS((SSLSocket) ssl); + return ssl; + } + + @Override + public Socket createSocket(InetAddress host, int port) throws IOException { + Socket ssl = defaultFactory.createSocket(host, port); + if (ssl instanceof SSLSocket) upgradeTLS((SSLSocket) ssl); + return ssl; + } + + @Override + public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException { + Socket ssl = defaultFactory.createSocket(address, port, localAddress, localPort); + if (ssl instanceof SSLSocket) upgradeTLS((SSLSocket) ssl); + return ssl; + } +} diff --git a/app/src/main/java/com/github/catvod/parser/JsonBasic.java b/app/src/main/java/com/github/catvod/parser/JsonBasic.java new file mode 100644 index 00000000..9d2c313c --- /dev/null +++ b/app/src/main/java/com/github/catvod/parser/JsonBasic.java @@ -0,0 +1,65 @@ +package com.github.catvod.parser; + +import android.util.Base64; + +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.net.OkHttpUtil; +import com.github.catvod.utils.Misc; + +import org.json.JSONObject; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Set; + +public class JsonBasic { + + public static JSONObject parse(LinkedHashMap jx, String url) { + SpiderDebug.log("Load Json Parse Basic..."); + if (jx.isEmpty()) return new JSONObject(); + Set jxNames = jx.keySet(); + for (String jxName : jxNames) { + String parseUrl = jx.get(jxName); + HashMap reqHeaders = getReqHeader(parseUrl); + try { + String realUrl = reqHeaders.get("url"); + reqHeaders.remove("url"); + SpiderDebug.log(realUrl + url); + String json = OkHttpUtil.string(realUrl + url, reqHeaders); + JSONObject taskResult = Misc.jsonParse(url, json); + if (taskResult == null) continue; + taskResult.put("jxFrom", jxName); + SpiderDebug.log(taskResult.toString()); + return taskResult; + } catch (Exception ignored) { + } + } + return new JSONObject(); + } + + public static HashMap getReqHeader(String url) { + HashMap reqHeaders = new HashMap<>(); + reqHeaders.put("url", url); + if (!url.contains("cat_ext")) return reqHeaders; + try { + int start = url.indexOf("cat_ext="); + int end = url.indexOf("&", start); + String ext = url.substring(start + 8, end); + ext = new String(Base64.decode(ext, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP)); + String newUrl = url.substring(0, start) + url.substring(end + 1); + JSONObject jsonObject = new JSONObject(ext); + if (jsonObject.has("header")) { + JSONObject headerJson = jsonObject.optJSONObject("header"); + Iterator keys = headerJson.keys(); + while (keys.hasNext()) { + String key = keys.next(); + reqHeaders.put(key, headerJson.optString(key, "")); + } + } + reqHeaders.put("url", newUrl); + } catch (Exception ignored) { + } + return reqHeaders; + } +} diff --git a/app/src/main/java/com/github/catvod/parser/JsonParallel.java b/app/src/main/java/com/github/catvod/parser/JsonParallel.java new file mode 100644 index 00000000..ae07394c --- /dev/null +++ b/app/src/main/java/com/github/catvod/parser/JsonParallel.java @@ -0,0 +1,71 @@ +package com.github.catvod.parser; + +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.net.OkHttpUtil; +import com.github.catvod.utils.Misc; + +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +public class JsonParallel { + + private static final String ParseOKTag = "p_json_parse"; + + public static JSONObject parse(LinkedHashMap jx, String url) { + if (jx.isEmpty()) return new JSONObject(); + ExecutorService service = Executors.newFixedThreadPool(3); + CompletionService completionService = new ExecutorCompletionService<>(service); + List> futures = new ArrayList<>(); + Set jxNames = jx.keySet(); + for (String jxName : jxNames) { + String parseUrl = jx.get(jxName); + futures.add(completionService.submit(() -> { + try { + HashMap reqHeaders = JsonBasic.getReqHeader(parseUrl); + String realUrl = reqHeaders.get("url"); + reqHeaders.remove("url"); + SpiderDebug.log(realUrl + url); + String json = OkHttpUtil.string(realUrl + url, ParseOKTag, reqHeaders); + JSONObject taskResult = Misc.jsonParse(url, json); + taskResult.put("jxFrom", jxName); + SpiderDebug.log(taskResult.toString()); + return taskResult; + } catch (Exception ignored) { + return null; + } + })); + } + JSONObject pTaskResult = null; + for (int i = 0; i < futures.size(); ++i) { + try { + Future completed = completionService.take(); + pTaskResult = completed.get(); + if (pTaskResult != null) { + OkHttpUtil.cancel(ParseOKTag); + for (int j = 0; j < futures.size(); j++) { + try { + futures.get(j).cancel(true); + } catch (Throwable th) { + SpiderDebug.log(th); + } + } + futures.clear(); + break; + } + } catch (Exception ignored) { + } + } + service.shutdownNow(); + return pTaskResult != null ? pTaskResult : new JSONObject(); + } +} diff --git a/app/src/main/java/com/github/catvod/parser/JsonSequence.java b/app/src/main/java/com/github/catvod/parser/JsonSequence.java new file mode 100644 index 00000000..fdf6f320 --- /dev/null +++ b/app/src/main/java/com/github/catvod/parser/JsonSequence.java @@ -0,0 +1,35 @@ +package com.github.catvod.parser; + +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.net.OkHttpUtil; +import com.github.catvod.utils.Misc; + +import org.json.JSONObject; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Set; + +public class JsonSequence { + + public static JSONObject parse(LinkedHashMap jx, String url) { + if (jx.isEmpty()) return new JSONObject(); + Set jxNames = jx.keySet(); + for (String jxName : jxNames) { + try { + String parseUrl = jx.get(jxName); + HashMap reqHeaders = JsonBasic.getReqHeader(parseUrl); + String realUrl = reqHeaders.get("url"); + reqHeaders.remove("url"); + SpiderDebug.log(realUrl + url); + String json = OkHttpUtil.string(realUrl + url, reqHeaders); + JSONObject taskResult = Misc.jsonParse(url, json); + if (taskResult == null) continue; + taskResult.put("jxFrom", jxName); + return taskResult; + } catch (Exception ignored) { + } + } + return new JSONObject(); + } +} diff --git a/app/src/main/java/com/github/catvod/parser/MixDemo.java b/app/src/main/java/com/github/catvod/parser/MixDemo.java new file mode 100644 index 00000000..2fec652d --- /dev/null +++ b/app/src/main/java/com/github/catvod/parser/MixDemo.java @@ -0,0 +1,88 @@ +package com.github.catvod.parser; + +import android.util.Base64; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; + +public class MixDemo { + + private static final HashMap> flagWebJx = new HashMap<>(); + private static HashMap> configs = null; + + private static void setConfigs(LinkedHashMap> jx) { + configs = new HashMap<>(); + for (String key : jx.keySet()) { + HashMap parseBean = jx.get(key); + String type = parseBean.get("type"); + if (type.equals("1") || type.equals("0")) { + try { + JSONArray flags = new JSONObject(parseBean.get("ext")).getJSONArray("flag"); + for (int j = 0; j < flags.length(); j++) { + String flagKey = flags.getString(j); + ArrayList flagJx = configs.get(flagKey); + if (flagJx == null) { + flagJx = new ArrayList<>(); + configs.put(flagKey, flagJx); + } + flagJx.add(key); + } + } catch (Exception ignored) { + } + } + } + } + + public static JSONObject parse(LinkedHashMap> jx, String nameMe, String flag, String url) { + try { + if (configs == null) setConfigs(jx); + LinkedHashMap jsonJx = new LinkedHashMap<>(); + ArrayList webJx = new ArrayList<>(); + ArrayList flagJx = configs.get(flag); + if (flagJx != null && !flagJx.isEmpty()) { + for (int i = 0; i < flagJx.size(); i++) { + String key = flagJx.get(i); + HashMap parseBean = jx.get(key); + String type = parseBean.get("type"); + if (type.equals("1")) { + jsonJx.put(key, mixUrl(parseBean.get("url"), parseBean.get("ext"))); + } else if (type.equals("0")) { + webJx.add(parseBean.get("url")); + } + } + } else { + for (String key : jx.keySet()) { + HashMap parseBean = jx.get(key); + String type = parseBean.get("type"); + if (type.equals("1")) { + jsonJx.put(key, mixUrl(parseBean.get("url"), parseBean.get("ext"))); + } else if (type.equals("0")) { + webJx.add(parseBean.get("url")); + } + } + } + if (!webJx.isEmpty()) flagWebJx.put(flag, webJx); + JSONObject jsonResult = JsonParallel.parse(jsonJx, url); + if (jsonResult.has("url")) return jsonResult; + if (!webJx.isEmpty()) { + JSONObject webResult = new JSONObject(); + webResult.put("url", "proxy://do=parseMix&flag=" + flag + "&url=" + Base64.encodeToString(url.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP)); + webResult.put("parse", 1); + return webResult; + } + } catch (Exception ignored) { + } + return new JSONObject(); + } + + private static String mixUrl(String url, String ext) { + if (ext.trim().isEmpty()) return url; + int index = url.indexOf("?"); + if (index == -1) return url; + return url.substring(0, index + 1) + "cat_ext=" + Base64.encodeToString(ext.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP) + "&" + url.substring(index + 1); + } +} diff --git a/app/src/main/java/com/github/catvod/spider/Init.java b/app/src/main/java/com/github/catvod/spider/Init.java new file mode 100644 index 00000000..e98af6cf --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/Init.java @@ -0,0 +1,31 @@ +package com.github.catvod.spider; + +import android.app.Application; +import android.content.Context; +import android.os.Handler; +import android.os.Looper; + +import com.github.catvod.crawler.SpiderDebug; + +public class Init { + + private final Handler mHandler; + private Application mApp; + + private static class Loader { + static volatile Init INSTANCE = new Init(); + } + + private static Init get() { + return Loader.INSTANCE; + } + + public Init() { + this.mHandler = new Handler(Looper.getMainLooper()); + } + + public static void init(Context context) { + SpiderDebug.log("自定義爬蟲代碼載入成功！"); + get().mApp = ((Application) context); + } +} diff --git a/app/src/main/java/com/github/catvod/spider/Proxy.java b/app/src/main/java/com/github/catvod/spider/Proxy.java new file mode 100644 index 00000000..8201199f --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/Proxy.java @@ -0,0 +1,48 @@ +package com.github.catvod.spider; + +import android.util.Base64; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.live.TxtSubscribe; +import com.github.catvod.net.OkHttpUtil; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Objects; + +public class Proxy extends Spider { + + private static int port = -1; + + public static Object[] proxy(Map params) { + switch (Objects.requireNonNull(params.get("do"))) { + case "ck": + return new Object[]{200, "text/plain; charset=utf-8", new ByteArrayInputStream("ok".getBytes(StandardCharsets.UTF_8))}; + case "live": + return TxtSubscribe.load(new String(Base64.decode(params.get("ext"), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP), StandardCharsets.UTF_8)); + default: + return null; + } + } + + static void adjustPort() { + if (Proxy.port > 0) return; + int port = 9978; + while (port < 10000) { + String resp = OkHttpUtil.string("http://127.0.0.1:" + port + "/proxy?do=ck", null); + if (resp.equals("ok")) { + SpiderDebug.log("Found local server port " + port); + Proxy.port = port; + break; + } + port++; + } + } + + public static String getUrl() { + adjustPort(); + return "http://127.0.0.1:" + port + "/proxy"; + } +} diff --git a/app/src/main/java/com/github/catvod/utils/Misc.java b/app/src/main/java/com/github/catvod/utils/Misc.java new file mode 100644 index 00000000..c279f933 --- /dev/null +++ b/app/src/main/java/com/github/catvod/utils/Misc.java @@ -0,0 +1,128 @@ +package com.github.catvod.utils; + +import android.net.Uri; + +import com.github.catvod.crawler.SpiderDebug; + +import org.json.JSONException; +import org.json.JSONObject; + +import java.math.BigInteger; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.regex.Pattern; + +public class Misc { + + public static final String CHROME = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.62 Safari/537.36"; + private static final Pattern snifferMatch = Pattern.compile("http((?!http).){20,}?\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg)\\?.*|http((?!http).){20,}\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg)|http((?!http).){20,}?\\/m3u8\\?pt=m3u8.*|http((?!http).)*?default\\.ixigua\\.com\\/.*|http((?!http).)*?cdn-tos[^\\?]*|http((?!http).)*?\\/obj\\/tos[^\\?]*|http.*?\\/player\\/m3u8play\\.php\\?url=.*|http.*?\\/player\\/.*?[pP]lay\\.php\\?url=.*|http.*?\\/playlist\\/m3u8\\/\\?vid=.*|http.*?\\.php\\?type=m3u8&.*|http.*?\\/download.aspx\\?.*|http.*?\\/api\\/up_api.php\\?.*|https.*?\\.66yk\\.cn.*|http((?!http).)*?netease\\.com\\/file\\/.*"); + + public static boolean isVip(String url) { + try { + boolean isVip = false; + String host = Uri.parse(url).getHost(); + String[] vipWebsites = new String[]{"iqiyi.com", "v.qq.com", "youku.com", "le.com", "tudou.com", "mgtv.com", "sohu.com", "acfun.cn", "bilibili.com", "baofeng.com", "pptv.com"}; + for (String vipWebsite : vipWebsites) { + if (host.contains(vipWebsite)) { + if ("iqiyi.com".equals(vipWebsite)) { + if (url.contains("iqiyi.com/a_") || url.contains("iqiyi.com/w_") || url.contains("iqiyi.com/v_")) { + isVip = true; + break; + } + } else { + isVip = true; + break; + } + } + } + return isVip; + } catch (Exception ignored) { + } + return false; + } + + public static boolean isVideoFormat(String url) { + if (snifferMatch.matcher(url).find()) { + return !url.contains("cdn-tos") || !url.contains(".js"); + } + return false; + } + + public static String fixUrl(String base, String src) { + try { + if (src.startsWith("//")) { + Uri parse = Uri.parse(base); + src = parse.getScheme() + ":" + src; + } else if (!src.contains("://")) { + Uri parse = Uri.parse(base); + src = parse.getScheme() + "://" + parse.getHost() + src; + } + } catch (Exception e) { + SpiderDebug.log(e); + } + return src; + } + + public static JSONObject fixJsonVodHeader(JSONObject headers, String input, String url) throws JSONException { + if (headers == null) + headers = new JSONObject(); + if (input.contains("www.mgtv.com")) { + headers.put("Referer", " "); + headers.put("User-Agent", " Mozilla/5.0"); + } else if (url.contains("titan.mgtv")) { + headers.put("Referer", " "); + headers.put("User-Agent", " Mozilla/5.0"); + } else if (input.contains("bilibili")) { + headers.put("Referer", " https://www.bilibili.com/"); + headers.put("User-Agent", " " + Misc.CHROME); + } + return headers; + } + + public static JSONObject jsonParse(String input, String json) throws JSONException { + JSONObject jsonPlayData = new JSONObject(json); + String url = jsonPlayData.getString("url"); + if (url.startsWith("//")) { + url = "https:" + url; + } + if (!url.startsWith("http")) { + return null; + } + if (url.equals(input)) { + if (isVip(url) || !isVideoFormat(url)) return null; + } + JSONObject headers = new JSONObject(); + String ua = jsonPlayData.optString("user-agent", ""); + if (ua.trim().length() > 0) { + headers.put("User-Agent", " " + ua); + } + String referer = jsonPlayData.optString("referer", ""); + if (referer.trim().length() > 0) { + headers.put("Referer", " " + referer); + } + headers = Misc.fixJsonVodHeader(headers, input, url); + JSONObject taskResult = new JSONObject(); + taskResult.put("header", headers); + taskResult.put("url", url); + return taskResult; + } + + public static String MD5(String src) { + return MD5(src, StandardCharsets.UTF_8); + } + + public static String MD5(String src, Charset charset) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] messageDigest = md.digest(src.getBytes(charset)); + BigInteger no = new BigInteger(1, messageDigest); + StringBuilder sb = new StringBuilder(no.toString(16)); + while (sb.length() < 32) sb.insert(0, "0"); + return sb.toString().toLowerCase(); + } catch (NoSuchAlgorithmException e) { + return ""; + } + } +} diff --git a/app/src/main/java/com/github/catvod/utils/Trans.java b/app/src/main/java/com/github/catvod/utils/Trans.java new file mode 100644 index 00000000..e4198db6 --- /dev/null +++ b/app/src/main/java/com/github/catvod/utils/Trans.java @@ -0,0 +1,23 @@ +package com.github.catvod.utils; + +import java.util.HashMap; + +public class Trans { + + public static HashMap S2T = new HashMap<>(); + + public static void init() { + char[] UTF8T = "萬與醜專業叢東絲丟兩嚴喪個爿豐臨為麗舉麼義烏樂喬習鄉書買亂爭於虧雲亙亞產畝親褻嚲億僅從侖倉儀們價眾優夥會傴傘偉傳傷倀倫傖偽佇體餘傭僉俠侶僥偵側僑儈儕儂俁儔儼倆儷儉債傾傯僂僨償儻儐儲儺兒兌兗黨蘭關興茲養獸囅內岡冊寫軍農塚馮衝決況凍淨淒涼淩減湊凜幾鳳鳧憑凱擊氹鑿芻劃劉則剛創刪別剗剄劊劌剴劑剮劍剝劇勸辦務勱動勵勁勞勢勳猛勩勻匭匱區醫華協單賣盧鹵臥衛卻巹廠廳歷厲壓厭厙廁廂厴廈廚廄廝縣參靉靆雙發變敘疊葉號歎嘰籲後嚇呂嗎唚噸聽啟吳嘸囈嘔嚦唄員咼嗆嗚詠哢嚨嚀噝吒噅鹹呱響啞噠嘵嗶噦嘩噲嚌噥喲嘜嗊嘮啢嗩唕喚呼嘖嗇囀齧囉嘽嘯噴嘍嚳囁嗬噯噓嚶囑嚕劈囂謔團園囪圍圇國圖圓聖壙場阪壞塊堅壇壢壩塢墳墜壟壟壚壘墾坰堊墊埡墶壋塏堖塒塤堝墊垵塹墮壪牆壯聲殼壺壼處備複夠頭誇夾奪奩奐奮獎奧妝婦媽嫵嫗媯姍婁婭嬈嬌孌娛媧嫻嫿嬰嬋嬸媼嬡嬪嬙嬤孫學孿寧寶實寵審憲宮寬賓寢對尋導壽將爾塵堯尷屍盡層屭屜屆屬屢屨嶼歲豈嶇崗峴嶴嵐島嶺崠巋嶨嶧峽嶢嶠崢巒嶗崍嶮嶄嶸嶔崳嶁脊巔鞏巰幣帥師幃帳簾幟帶幀幫幬幘幗冪襆幹並廣莊慶廬廡庫應廟龐廢廎廩開異棄張彌弳彎彈強歸當錄彠彥徹徑徠禦憶懺憂愾懷態慫憮慪悵愴憐總懟懌戀懇惡慟懨愷惻惱惲悅愨懸慳憫驚懼慘懲憊愜慚憚慣湣慍憤憒願懾憖怵懣懶懍戇戔戲戧戰戩戶紮撲扡執擴捫掃揚擾撫拋摶摳掄搶護報擔擬攏揀擁攔擰撥擇掛摯攣掗撾撻挾撓擋撟掙擠揮撏撈損撿換搗據撚擄摑擲撣摻摜摣攬撳攙擱摟攪攜攝攄擺搖擯攤攖撐攆擷擼攛擻攢敵斂數齋斕斗斬斷無舊時曠暘曇晝曨顯晉曬曉曄暈暉暫曖劄術樸機殺雜權條來楊榪傑極構樅樞棗櫪梘棖槍楓梟櫃檸檉梔柵標棧櫛櫳棟櫨櫟欄樹棲樣欒棬椏橈楨檔榿橋樺檜槳樁夢檮棶檢欞槨櫝槧欏橢樓欖櫬櫚櫸檟檻檳櫧橫檣櫻櫫櫥櫓櫞簷檁歡歟歐殲歿殤殘殞殮殫殯毆毀轂畢斃氈毿氌氣氫氬氳彙漢汙湯洶遝溝沒灃漚瀝淪滄渢溈滬濔濘淚澩瀧瀘濼瀉潑澤涇潔灑窪浹淺漿澆湞溮濁測澮濟瀏滻渾滸濃潯濜塗湧濤澇淶漣潿渦溳渙滌潤澗漲澀澱淵淥漬瀆漸澠漁瀋滲溫遊灣濕潰濺漵漊潷滾滯灩灄滿瀅濾濫灤濱灘澦瀠瀟瀲濰潛瀦瀾瀨瀕灝滅燈靈災燦煬爐燉煒熗點煉熾爍爛烴燭煙煩燒燁燴燙燼熱煥燜燾煆糊溜愛爺牘犛牽犧犢強狀獷獁猶狽麅獮獰獨狹獅獪猙獄猻獫獵獼玀豬貓蝟獻獺璣璵瑒瑪瑋環現瑲璽瑉玨琺瓏璫琿璡璉瑣瓊瑤璦璿瓔瓚甕甌電畫暢佘疇癤療瘧癘瘍鬁瘡瘋皰屙癰痙癢瘂癆瘓癇癡癉瘮瘞瘺癟癱癮癭癩癬癲臒皚皺皸盞鹽監蓋盜盤瞘眥矓著睜睞瞼瞞矚矯磯礬礦碭碼磚硨硯碸礪礱礫礎硜矽碩硤磽磑礄確鹼礙磧磣堿镟滾禮禕禰禎禱禍稟祿禪離禿稈種積稱穢穠穭稅穌穩穡窮竊竅窯竄窩窺竇窶豎競篤筍筆筧箋籠籩築篳篩簹箏籌簽簡籙簀篋籜籮簞簫簣簍籃籬籪籟糴類秈糶糲粵糞糧糝餱緊縶糸糾紆紅紂纖紇約級紈纊紀紉緯紜紘純紕紗綱納紝縱綸紛紙紋紡紵紖紐紓線紺絏紱練組紳細織終縐絆紼絀紹繹經紿綁絨結絝繞絰絎繪給絢絳絡絕絞統綆綃絹繡綌綏絛繼綈績緒綾緓續綺緋綽緔緄繩維綿綬繃綢綯綹綣綜綻綰綠綴緇緙緗緘緬纜緹緲緝縕繢緦綞緞緶線緱縋緩締縷編緡緣縉縛縟縝縫縗縞纏縭縊縑繽縹縵縲纓縮繆繅纈繚繕繒韁繾繰繯繳纘罌網羅罰罷羆羈羥羨翹翽翬耮耬聳恥聶聾職聹聯聵聰肅腸膚膁腎腫脹脅膽勝朧腖臚脛膠脈膾髒臍腦膿臠腳脫腡臉臘醃膕齶膩靦膃騰臏臢輿艤艦艙艫艱豔艸藝節羋薌蕪蘆蓯葦藶莧萇蒼苧蘇檾蘋莖蘢蔦塋煢繭荊薦薘莢蕘蓽蕎薈薺蕩榮葷滎犖熒蕁藎蓀蔭蕒葒葤藥蒞蓧萊蓮蒔萵薟獲蕕瑩鶯蓴蘀蘿螢營縈蕭薩蔥蕆蕢蔣蔞藍薊蘺蕷鎣驀薔蘞藺藹蘄蘊藪槁蘚虜慮虛蟲虯蟣雖蝦蠆蝕蟻螞蠶蠔蜆蠱蠣蟶蠻蟄蛺蟯螄蠐蛻蝸蠟蠅蟈蟬蠍螻蠑螿蟎蠨釁銜補襯袞襖嫋褘襪襲襏裝襠褌褳襝褲襇褸襤繈襴見觀覎規覓視覘覽覺覬覡覿覥覦覯覲覷觴觸觶讋譽謄訁計訂訃認譏訐訌討讓訕訖訓議訊記訒講諱謳詎訝訥許訛論訩訟諷設訪訣證詁訶評詛識詗詐訴診詆謅詞詘詔詖譯詒誆誄試詿詩詰詼誠誅詵話誕詬詮詭詢詣諍該詳詫諢詡譸誡誣語誚誤誥誘誨誑說誦誒請諸諏諾讀諑誹課諉諛誰諗調諂諒諄誶談誼謀諶諜謊諫諧謔謁謂諤諭諼讒諮諳諺諦謎諞諝謨讜謖謝謠謗諡謙謐謹謾謫譾謬譚譖譙讕譜譎讞譴譫讖穀豶貝貞負貟貢財責賢敗賬貨質販貪貧貶購貯貫貳賤賁貰貼貴貺貸貿費賀貽賊贄賈賄貲賃賂贓資賅贐賕賑賚賒賦賭齎贖賞賜贔賙賡賠賧賴賵贅賻賺賽賾贗讚贇贈贍贏贛赬趙趕趨趲躉躍蹌蹠躒踐躂蹺蹕躚躋踴躊蹤躓躑躡蹣躕躥躪躦軀車軋軌軒軑軔轉軛輪軟轟軲軻轤軸軹軼軤軫轢軺輕軾載輊轎輈輇輅較輒輔輛輦輩輝輥輞輬輟輜輳輻輯轀輸轡轅轄輾轆轍轔辭辯辮邊遼達遷過邁運還這進遠違連遲邇逕跡適選遜遞邐邏遺遙鄧鄺鄔郵鄒鄴鄰鬱郤郟鄶鄭鄆酈鄖鄲醞醱醬釅釃釀釋里钜鑒鑾鏨釓釔針釘釗釙釕釷釺釧釤鈒釩釣鍆釹鍚釵鈃鈣鈈鈦鈍鈔鍾鈉鋇鋼鈑鈐鑰欽鈞鎢鉤鈧鈁鈥鈄鈕鈀鈺錢鉦鉗鈷缽鈳鉕鈽鈸鉞鑽鉬鉭鉀鈿鈾鐵鉑鈴鑠鉛鉚鈰鉉鉈鉍鈹鐸鉶銬銠鉺銪鋏鋣鐃銍鐺銅鋁銱銦鎧鍘銖銑鋌銩銛鏵銓鉿銚鉻銘錚銫鉸銥鏟銃鐋銨銀銣鑄鐒鋪鋙錸鋱鏈鏗銷鎖鋰鋥鋤鍋鋯鋨鏽銼鋝鋒鋅鋶鐦鐧銳銻鋃鋟鋦錒錆鍺錯錨錡錁錕錩錫錮鑼錘錐錦鍁錈錇錟錠鍵鋸錳錙鍥鍈鍇鏘鍶鍔鍤鍬鍾鍛鎪鍠鍰鎄鍍鎂鏤鎡鏌鎮鎛鎘鑷鐫鎳鎿鎦鎬鎊鎰鎔鏢鏜鏍鏰鏞鏡鏑鏃鏇鏐鐔钁鐐鏷鑥鐓鑭鐠鑹鏹鐙鑊鐳鐶鐲鐮鐿鑔鑣鑞鑲長門閂閃閆閈閉問闖閏闈閑閎間閔閌悶閘鬧閨聞闥閩閭闓閥閣閡閫鬮閱閬闍閾閹閶鬩閿閽閻閼闡闌闃闠闊闋闔闐闒闕闞闤隊陽陰陣階際陸隴陳陘陝隉隕險隨隱隸雋難雛讎靂霧霽黴靄靚靜靨韃鞽韉韝韋韌韍韓韙韞韜韻頁頂頃頇項順須頊頑顧頓頎頒頌頏預顱領頗頸頡頰頲頜潁熲頦頤頻頮頹頷頴穎顆題顒顎顓顏額顳顢顛顙顥纇顫顬顰顴風颺颭颮颯颶颸颼颻飀飄飆飆飛饗饜飣饑飥餳飩餼飪飫飭飯飲餞飾飽飼飿飴餌饒餉餄餎餃餏餅餑餖餓餘餒餕餜餛餡館餷饋餶餿饞饁饃餺餾饈饉饅饊饌饢馬馭馱馴馳驅馹駁驢駔駛駟駙駒騶駐駝駑駕驛駘驍罵駰驕驊駱駭駢驫驪騁驗騂駸駿騏騎騍騅騌驌驂騙騭騤騷騖驁騮騫騸驃騾驄驏驟驥驦驤髏髖髕鬢魘魎魚魛魢魷魨魯魴魺鮁鮃鯰鱸鮋鮓鮒鮊鮑鱟鮍鮐鮭鮚鮳鮪鮞鮦鰂鮜鱠鱭鮫鮮鮺鯗鱘鯁鱺鰱鰹鯉鰣鰷鯀鯊鯇鮶鯽鯒鯖鯪鯕鯫鯡鯤鯧鯝鯢鯰鯛鯨鯵鯴鯔鱝鰈鰏鱨鯷鰮鰃鰓鱷鰍鰒鰉鰁鱂鯿鰠鼇鰭鰨鰥鰩鰟鰜鰳鰾鱈鱉鰻鰵鱅鰼鱖鱔鱗鱒鱯鱤鱧鱣鳥鳩雞鳶鳴鳲鷗鴉鶬鴇鴆鴣鶇鸕鴨鴞鴦鴒鴟鴝鴛鴬鴕鷥鷙鴯鴰鵂鴴鵃鴿鸞鴻鵐鵓鸝鵑鵠鵝鵒鷳鵜鵡鵲鶓鵪鶤鵯鵬鵮鶉鶊鵷鷫鶘鶡鶚鶻鶿鶥鶩鷊鷂鶲鶹鶺鷁鶼鶴鷖鸚鷓鷚鷯鷦鷲鷸鷺鸇鷹鸌鸏鸛鸘鹺麥麩黃黌黶黷黲黽黿鼂鼉鞀鼴齇齊齏齒齔齕齗齟齡齙齠齜齦齬齪齲齷龍龔龕龜誌製谘範鬆冇嚐嘗鬨準鐘彆閒乾儘臟拚作".toCharArray(); + char[] UTF8S = "万与丑专业丛东丝丢两严丧个丬丰临为丽举么义乌乐乔习乡书买乱争于亏云亘亚产亩亲亵亸亿仅从仑仓仪们价众优伙会伛伞伟传伤伥伦伧伪伫体余佣佥侠侣侥侦侧侨侩侪侬俣俦俨俩俪俭债倾偬偻偾偿傥傧储傩儿兑兖党兰关兴兹养兽冁内冈册写军农冢冯冲决况冻净凄凉凌减凑凛几凤凫凭凯击凼凿刍划刘则刚创删别刬刭刽刿剀剂剐剑剥剧劝办务劢动励劲劳势勋勐勚匀匦匮区医华协单卖卢卤卧卫却卺厂厅历厉压厌厍厕厢厣厦厨厩厮县参叆叇双发变叙叠叶号叹叽吁后吓吕吗吣吨听启吴呒呓呕呖呗员呙呛呜咏咔咙咛咝咤咴咸哌响哑哒哓哔哕哗哙哜哝哟唛唝唠唡唢唣唤唿啧啬啭啮啰啴啸喷喽喾嗫呵嗳嘘嘤嘱噜噼嚣嚯团园囱围囵国图圆圣圹场坂坏块坚坛坜坝坞坟坠垄垅垆垒垦垧垩垫垭垯垱垲垴埘埙埚埝埯堑堕塆墙壮声壳壶壸处备复够头夸夹夺奁奂奋奖奥妆妇妈妩妪妫姗娄娅娆娇娈娱娲娴婳婴婵婶媪嫒嫔嫱嬷孙学孪宁宝实宠审宪宫宽宾寝对寻导寿将尔尘尧尴尸尽层屃屉届属屡屦屿岁岂岖岗岘岙岚岛岭岽岿峃峄峡峣峤峥峦崂崃崄崭嵘嵚嵛嵝嵴巅巩巯币帅师帏帐帘帜带帧帮帱帻帼幂幞干并广庄庆庐庑库应庙庞废庼廪开异弃张弥弪弯弹强归当录彟彦彻径徕御忆忏忧忾怀态怂怃怄怅怆怜总怼怿恋恳恶恸恹恺恻恼恽悦悫悬悭悯惊惧惨惩惫惬惭惮惯愍愠愤愦愿慑慭憷懑懒懔戆戋戏戗战戬户扎扑扦执扩扪扫扬扰抚抛抟抠抡抢护报担拟拢拣拥拦拧拨择挂挚挛挜挝挞挟挠挡挢挣挤挥挦捞损捡换捣据捻掳掴掷掸掺掼揸揽揿搀搁搂搅携摄摅摆摇摈摊撄撑撵撷撸撺擞攒敌敛数斋斓斗斩断无旧时旷旸昙昼昽显晋晒晓晔晕晖暂暧札术朴机杀杂权条来杨杩杰极构枞枢枣枥枧枨枪枫枭柜柠柽栀栅标栈栉栊栋栌栎栏树栖样栾桊桠桡桢档桤桥桦桧桨桩梦梼梾检棂椁椟椠椤椭楼榄榇榈榉槚槛槟槠横樯樱橥橱橹橼檐檩欢欤欧歼殁殇残殒殓殚殡殴毁毂毕毙毡毵氇气氢氩氲汇汉污汤汹沓沟没沣沤沥沦沧沨沩沪沵泞泪泶泷泸泺泻泼泽泾洁洒洼浃浅浆浇浈浉浊测浍济浏浐浑浒浓浔浕涂涌涛涝涞涟涠涡涢涣涤润涧涨涩淀渊渌渍渎渐渑渔渖渗温游湾湿溃溅溆溇滗滚滞滟滠满滢滤滥滦滨滩滪潆潇潋潍潜潴澜濑濒灏灭灯灵灾灿炀炉炖炜炝点炼炽烁烂烃烛烟烦烧烨烩烫烬热焕焖焘煅煳熘爱爷牍牦牵牺犊强状犷犸犹狈狍狝狞独狭狮狯狰狱狲猃猎猕猡猪猫猬献獭玑玙玚玛玮环现玱玺珉珏珐珑珰珲琎琏琐琼瑶瑷璇璎瓒瓮瓯电画畅畲畴疖疗疟疠疡疬疮疯疱疴痈痉痒痖痨痪痫痴瘅瘆瘗瘘瘪瘫瘾瘿癞癣癫癯皑皱皲盏盐监盖盗盘眍眦眬着睁睐睑瞒瞩矫矶矾矿砀码砖砗砚砜砺砻砾础硁硅硕硖硗硙硚确硷碍碛碜碱碹磙礼祎祢祯祷祸禀禄禅离秃秆种积称秽秾稆税稣稳穑穷窃窍窑窜窝窥窦窭竖竞笃笋笔笕笺笼笾筑筚筛筜筝筹签简箓箦箧箨箩箪箫篑篓篮篱簖籁籴类籼粜粝粤粪粮糁糇紧絷纟纠纡红纣纤纥约级纨纩纪纫纬纭纮纯纰纱纲纳纴纵纶纷纸纹纺纻纼纽纾线绀绁绂练组绅细织终绉绊绋绌绍绎经绐绑绒结绔绕绖绗绘给绚绛络绝绞统绠绡绢绣绤绥绦继绨绩绪绫绬续绮绯绰绱绲绳维绵绶绷绸绹绺绻综绽绾绿缀缁缂缃缄缅缆缇缈缉缊缋缌缍缎缏缐缑缒缓缔缕编缗缘缙缚缛缜缝缞缟缠缡缢缣缤缥缦缧缨缩缪缫缬缭缮缯缰缱缲缳缴缵罂网罗罚罢罴羁羟羡翘翙翚耢耧耸耻聂聋职聍联聩聪肃肠肤肷肾肿胀胁胆胜胧胨胪胫胶脉脍脏脐脑脓脔脚脱脶脸腊腌腘腭腻腼腽腾膑臜舆舣舰舱舻艰艳艹艺节芈芗芜芦苁苇苈苋苌苍苎苏苘苹茎茏茑茔茕茧荆荐荙荚荛荜荞荟荠荡荣荤荥荦荧荨荩荪荫荬荭荮药莅莜莱莲莳莴莶获莸莹莺莼萚萝萤营萦萧萨葱蒇蒉蒋蒌蓝蓟蓠蓣蓥蓦蔷蔹蔺蔼蕲蕴薮藁藓虏虑虚虫虬虮虽虾虿蚀蚁蚂蚕蚝蚬蛊蛎蛏蛮蛰蛱蛲蛳蛴蜕蜗蜡蝇蝈蝉蝎蝼蝾螀螨蟏衅衔补衬衮袄袅袆袜袭袯装裆裈裢裣裤裥褛褴襁襕见观觃规觅视觇览觉觊觋觌觍觎觏觐觑觞触觯詟誉誊讠计订讣认讥讦讧讨让讪讫训议讯记讱讲讳讴讵讶讷许讹论讻讼讽设访诀证诂诃评诅识诇诈诉诊诋诌词诎诏诐译诒诓诔试诖诗诘诙诚诛诜话诞诟诠诡询诣诤该详诧诨诩诪诫诬语诮误诰诱诲诳说诵诶请诸诹诺读诼诽课诿谀谁谂调谄谅谆谇谈谊谋谌谍谎谏谐谑谒谓谔谕谖谗谘谙谚谛谜谝谞谟谠谡谢谣谤谥谦谧谨谩谪谫谬谭谮谯谰谱谲谳谴谵谶谷豮贝贞负贠贡财责贤败账货质贩贪贫贬购贮贯贰贱贲贳贴贵贶贷贸费贺贻贼贽贾贿赀赁赂赃资赅赆赇赈赉赊赋赌赍赎赏赐赑赒赓赔赕赖赗赘赙赚赛赜赝赞赟赠赡赢赣赪赵赶趋趱趸跃跄跖跞践跶跷跸跹跻踊踌踪踬踯蹑蹒蹰蹿躏躜躯车轧轨轩轪轫转轭轮软轰轱轲轳轴轵轶轷轸轹轺轻轼载轾轿辀辁辂较辄辅辆辇辈辉辊辋辌辍辎辏辐辑辒输辔辕辖辗辘辙辚辞辩辫边辽达迁过迈运还这进远违连迟迩迳迹适选逊递逦逻遗遥邓邝邬邮邹邺邻郁郄郏郐郑郓郦郧郸酝酦酱酽酾酿释里鉅鉴銮錾钆钇针钉钊钋钌钍钎钏钐钑钒钓钔钕钖钗钘钙钚钛钝钞钟钠钡钢钣钤钥钦钧钨钩钪钫钬钭钮钯钰钱钲钳钴钵钶钷钸钹钺钻钼钽钾钿铀铁铂铃铄铅铆铈铉铊铋铍铎铏铐铑铒铕铗铘铙铚铛铜铝铞铟铠铡铢铣铤铥铦铧铨铪铫铬铭铮铯铰铱铲铳铴铵银铷铸铹铺铻铼铽链铿销锁锂锃锄锅锆锇锈锉锊锋锌锍锎锏锐锑锒锓锔锕锖锗错锚锜锞锟锠锡锢锣锤锥锦锨锩锫锬锭键锯锰锱锲锳锴锵锶锷锸锹锺锻锼锽锾锿镀镁镂镃镆镇镈镉镊镌镍镎镏镐镑镒镕镖镗镙镚镛镜镝镞镟镠镡镢镣镤镥镦镧镨镩镪镫镬镭镮镯镰镱镲镳镴镶长门闩闪闫闬闭问闯闰闱闲闳间闵闶闷闸闹闺闻闼闽闾闿阀阁阂阃阄阅阆阇阈阉阊阋阌阍阎阏阐阑阒阓阔阕阖阗阘阙阚阛队阳阴阵阶际陆陇陈陉陕陧陨险随隐隶隽难雏雠雳雾霁霉霭靓静靥鞑鞒鞯鞴韦韧韨韩韪韫韬韵页顶顷顸项顺须顼顽顾顿颀颁颂颃预颅领颇颈颉颊颋颌颍颎颏颐频颒颓颔颕颖颗题颙颚颛颜额颞颟颠颡颢颣颤颥颦颧风飏飐飑飒飓飔飕飖飗飘飙飚飞飨餍饤饥饦饧饨饩饪饫饬饭饮饯饰饱饲饳饴饵饶饷饸饹饺饻饼饽饾饿馀馁馂馃馄馅馆馇馈馉馊馋馌馍馎馏馐馑馒馓馔馕马驭驮驯驰驱驲驳驴驵驶驷驸驹驺驻驼驽驾驿骀骁骂骃骄骅骆骇骈骉骊骋验骍骎骏骐骑骒骓骔骕骖骗骘骙骚骛骜骝骞骟骠骡骢骣骤骥骦骧髅髋髌鬓魇魉鱼鱽鱾鱿鲀鲁鲂鲄鲅鲆鲇鲈鲉鲊鲋鲌鲍鲎鲏鲐鲑鲒鲓鲔鲕鲖鲗鲘鲙鲚鲛鲜鲝鲞鲟鲠鲡鲢鲣鲤鲥鲦鲧鲨鲩鲪鲫鲬鲭鲮鲯鲰鲱鲲鲳鲴鲵鲶鲷鲸鲹鲺鲻鲼鲽鲾鲿鳀鳁鳂鳃鳄鳅鳆鳇鳈鳉鳊鳋鳌鳍鳎鳏鳐鳑鳒鳓鳔鳕鳖鳗鳘鳙鳛鳜鳝鳞鳟鳠鳡鳢鳣鸟鸠鸡鸢鸣鸤鸥鸦鸧鸨鸩鸪鸫鸬鸭鸮鸯鸰鸱鸲鸳鸴鸵鸶鸷鸸鸹鸺鸻鸼鸽鸾鸿鹀鹁鹂鹃鹄鹅鹆鹇鹈鹉鹊鹋鹌鹍鹎鹏鹐鹑鹒鹓鹔鹕鹖鹗鹘鹚鹛鹜鹝鹞鹟鹠鹡鹢鹣鹤鹥鹦鹧鹨鹩鹪鹫鹬鹭鹯鹰鹱鹲鹳鹴鹾麦麸黄黉黡黩黪黾鼋鼌鼍鼗鼹齄齐齑齿龀龁龂龃龄龅龆龇龈龉龊龋龌龙龚龛龟志制咨范松冇尝尝闹准钟彆闲干尽脏拼作".toCharArray(); + for (int i = 0, n = UTF8T.length; i < n; ++i) Trans.S2T.put(UTF8S[i], UTF8T[i]); + } + + public static String get(String text) { + char[] chars = text.toCharArray(); + for (int i = 0, n = chars.length; i < n; ++i) { + Character found = S2T.get(chars[i]); + if (null != found) chars[i] = found; + } + return String.valueOf(chars); + } +} diff --git a/app/src/main/java/com/github/catvod/xpath/XPathRule.java b/app/src/main/java/com/github/catvod/xpath/XPathRule.java new file mode 100644 index 00000000..44d7d1fb --- /dev/null +++ b/app/src/main/java/com/github/catvod/xpath/XPathRule.java @@ -0,0 +1,686 @@ +package com.github.catvod.xpath; + +import com.github.catvod.crawler.SpiderDebug; + +import org.json.JSONObject; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class XPathRule { + /** + * user-agent + */ + private String ua; + /** + * 取得分類和首頁推薦的Url + */ + private String homeUrl; + /** + * 分類節點 xpath + */ + private String cateNode; + /** + * 分類節點名 xpath + */ + private String cateName; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern cateNameR; + /** + * 分類節點 id xpath + */ + private String cateId; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern cateIdR; + /** + * 手動指定分類如果有則不從 homeUrl 中獲取分類 + */ + private final LinkedHashMap cateManual = new LinkedHashMap<>(); + + /** + * 篩選 + */ + private JSONObject filter; + + /** + * 更新推薦影片節點 xpath + */ + private String homeVodNode; + /** + * 更新推薦影片名稱 xpath + */ + private String homeVodName; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern homeVodNameR; + /** + * 更新推薦影片 id xpath + */ + private String homeVodId; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern homeVodIdR; + /** + * 更新推薦影片圖片 xpath + */ + private String homeVodImg; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern homeVodImgR; + /** + * 更新推薦影片簡介 xpath + */ + private String homeVodMark; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern homeVodMarkR; + /** + * 分類頁地址 + */ + private String cateUrl; + /** + * 分類頁影片節點 xpath + */ + private String cateVodNode; + /** + * 分類頁影片名稱 xpath + */ + private String cateVodName; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern cateVodNameR; + /** + * 分類頁影片影片id xpath + */ + private String cateVodId; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern cateVodIdR; + /** + * 分類頁影片影片圖片 xpath + */ + private String cateVodImg; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern cateVodImgR; + /** + * 分類頁影片影片簡介 xpath + */ + private String cateVodMark; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern cateVodMarkR; + + /** + * 詳情頁面 + */ + private String dtUrl; + /** + * 詳情節點 xpath + */ + private String dtNode; + /** + * 詳情影片 xpath + */ + private String dtName; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern dtNameR; + /** + * 詳情影片圖片 xpath + */ + private String dtImg; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern dtImgR; + /** + * 詳情影片分類 xpath + */ + private String dtCate; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern dtCateR; + /** + * 詳情影片年份 xpath + */ + private String dtYear; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern dtYearR; + /** + * 詳情影片地區 xpath + */ + private String dtArea; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern dtAreaR; + /** + * 詳情影片簡介 xpath + */ + private String dtMark; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern dtMarkR; + /** + * 詳情演員 xpath + */ + private String dtActor; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern dtActorR; + /** + * 詳情導演 xpath + */ + private String dtDirector; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern dtDirectorR; + /** + * 詳情說明 xpath + */ + private String dtDesc; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern dtDescR; + + /** + * 詳情播放來源節點 + */ + private String dtFromNode; + /** + * 詳情播放來源名稱 xpath + */ + private String dtFromName; + /** + * 詳情 + */ + private Pattern dtFromNameR; + /** + * 詳情播放地址列表節點 xpath + */ + private String dtUrlNode; + /** + * 詳情播放地址節點 xpath + */ + private String dtUrlSubNode; + /** + * 詳情播放地址id xpath + */ + private String dtUrlId; + /** + * 詳情 + */ + private Pattern dtUrlIdR; + /** + * 詳情播放地址名稱 xpath + */ + private String dtUrlName; + /** + * 詳情 + */ + private Pattern dtUrlNameR; + /** + * 播放頁面url + */ + private String playUrl; + /** + * 播放解析調用ua + */ + private String playUa; + + /** + * 搜尋頁地址 + */ + private String searchUrl; + + /** + * 搜尋頁影片節點 xpath + */ + private String scVodNode; + /** + * 搜尋頁影片名稱 xpath + */ + private String scVodName; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern scVodNameR; + /** + * 搜尋頁影片id xpath + */ + private String scVodId; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern scVodIdR; + /** + * 搜尋頁影片圖片 xpath + */ + private String scVodImg; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern scVodImgR; + /** + * 搜尋頁影片簡介 xpath + */ + private String scVodMark; + /** + * 正則對取到的數據進行二次處理 + */ + private Pattern scVodMarkR; + + private static Pattern getPattern(JSONObject json, String key) { + String v = json.optString(key).trim(); + if (v.isEmpty()) + return null; + else { + try { + return Pattern.compile(v); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + return null; + } + + private static String doReplaceRegex(Pattern pattern, String src) { + if (pattern == null) + return src; + try { + Matcher matcher = pattern.matcher(src); + if (matcher.find()) { + return matcher.group(1).trim(); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + return src; + } + + public static XPathRule fromJson(String json) { + try { + JSONObject jsonObj = new JSONObject(json); + XPathRule rule = new XPathRule(); + rule.ua = jsonObj.optString("ua"); + rule.homeUrl = jsonObj.optString("homeUrl").trim(); + rule.cateNode = jsonObj.optString("cateNode").trim(); + rule.cateName = jsonObj.optString("cateName").trim(); + rule.cateNameR = getPattern(jsonObj, "cateNameR"); + rule.cateId = jsonObj.optString("cateId").trim(); + rule.cateIdR = getPattern(jsonObj, "cateIdR"); + JSONObject navs = jsonObj.optJSONObject("cateManual"); + if (navs != null) { + Iterator keys = navs.keys(); + while (keys.hasNext()) { + String name = keys.next(); + rule.cateManual.put(name.trim(), navs.getString(name).trim()); + } + } + rule.filter = jsonObj.optJSONObject("filter"); + rule.homeVodNode = jsonObj.optString("homeVodNode").trim(); + rule.homeVodName = jsonObj.optString("homeVodName").trim(); + rule.homeVodNameR = getPattern(jsonObj, "homeVodNameR"); + rule.homeVodId = jsonObj.optString("homeVodId").trim(); + rule.homeVodIdR = getPattern(jsonObj, "homeVodIdR"); + rule.homeVodImg = jsonObj.optString("homeVodImg").trim(); + rule.homeVodImgR = getPattern(jsonObj, "homeVodImgR"); + rule.homeVodMark = jsonObj.optString("homeVodMark").trim(); + rule.homeVodMarkR = getPattern(jsonObj, "homeVodMarkR"); + rule.cateUrl = jsonObj.optString("cateUrl").trim(); + rule.cateVodNode = jsonObj.optString("cateVodNode").trim(); + rule.cateVodName = jsonObj.optString("cateVodName").trim(); + rule.cateVodNameR = getPattern(jsonObj, "cateVodNameR"); + rule.cateVodId = jsonObj.optString("cateVodId").trim(); + rule.cateVodIdR = getPattern(jsonObj, "cateVodIdR"); + rule.cateVodImg = jsonObj.optString("cateVodImg").trim(); + rule.cateVodImgR = getPattern(jsonObj, "cateVodImgR"); + rule.cateVodMark = jsonObj.optString("cateVodMark").trim(); + rule.cateVodMarkR = getPattern(jsonObj, "cateVodMarkR"); + rule.dtUrl = jsonObj.optString("dtUrl"); + rule.dtNode = jsonObj.optString("dtNode"); + rule.dtName = jsonObj.optString("dtName"); + rule.dtNameR = getPattern(jsonObj, "dtNameR"); + rule.dtImg = jsonObj.optString("dtImg"); + rule.dtImgR = getPattern(jsonObj, "dtImgR"); + rule.dtCate = jsonObj.optString("dtCate"); + rule.dtCateR = getPattern(jsonObj, "dtCateR"); + rule.dtYear = jsonObj.optString("dtYear"); + rule.dtYearR = getPattern(jsonObj, "dtYearR"); + rule.dtArea = jsonObj.optString("dtArea"); + rule.dtAreaR = getPattern(jsonObj, "dtAreaR"); + rule.dtMark = jsonObj.optString("dtMark"); + rule.dtMarkR = getPattern(jsonObj, "dtMarkR"); + rule.dtActor = jsonObj.optString("dtActor"); + rule.dtActorR = getPattern(jsonObj, "dtActorR"); + rule.dtDirector = jsonObj.optString("dtDirector"); + rule.dtDirectorR = getPattern(jsonObj, "dtDirectorR"); + rule.dtDesc = jsonObj.optString("dtDesc"); + rule.dtDescR = getPattern(jsonObj, "dtDescR"); + rule.dtFromNode = jsonObj.optString("dtFromNode"); + rule.dtFromName = jsonObj.optString("dtFromName"); + rule.dtFromNameR = getPattern(jsonObj, "dtFromNameR"); + rule.dtUrlNode = jsonObj.optString("dtUrlNode"); + rule.dtUrlSubNode = jsonObj.optString("dtUrlSubNode"); + rule.dtUrlId = jsonObj.optString("dtUrlId"); + rule.dtUrlIdR = getPattern(jsonObj, "dtUrlIdR"); + rule.dtUrlName = jsonObj.optString("dtUrlName"); + rule.dtUrlNameR = getPattern(jsonObj, "dtUrlNameR"); + rule.playUrl = jsonObj.optString("playUrl"); + rule.playUa = jsonObj.optString("playUa"); + rule.searchUrl = jsonObj.optString("searchUrl"); + rule.scVodNode = jsonObj.optString("scVodNode").trim(); + rule.scVodName = jsonObj.optString("scVodName").trim(); + rule.scVodNameR = getPattern(jsonObj, "scVodNameR"); + rule.scVodId = jsonObj.optString("scVodId").trim(); + rule.scVodIdR = getPattern(jsonObj, "scVodIdR"); + rule.scVodImg = jsonObj.optString("scVodImg").trim(); + rule.scVodImgR = getPattern(jsonObj, "scVodImgR"); + rule.scVodMark = jsonObj.optString("scVodMark").trim(); + rule.scVodMarkR = getPattern(jsonObj, "scVodMarkR"); + return rule; + } catch (Exception e) { + SpiderDebug.log(e); + } + return null; + } + + public String getUa() { + return ua; + } + + public String getHomeUrl() { + return homeUrl; + } + + public String getCateNode() { + return cateNode; + } + + public String getCateName() { + return cateName; + } + + public String getCateNameR(String src) { + return doReplaceRegex(cateNameR, src); + } + + public String getCateId() { + return cateId; + } + + public String getCateIdR(String src) { + return doReplaceRegex(cateIdR, src); + } + + public LinkedHashMap getCateManual() { + return cateManual; + } + + public JSONObject getFilter() { + return filter; + } + + public String getHomeVodNode() { + return homeVodNode; + } + + public String getHomeVodName() { + return homeVodName; + } + + public String getHomeVodNameR(String src) { + return doReplaceRegex(homeVodNameR, src); + } + + public String getHomeVodId() { + return homeVodId; + } + + public String getHomeVodIdR(String src) { + return doReplaceRegex(homeVodIdR, src); + } + + public String getHomeVodImg() { + return homeVodImg; + } + + public String getHomeVodImgR(String src) { + return doReplaceRegex(homeVodImgR, src); + } + + public String getHomeVodMark() { + return homeVodMark; + } + + public String getHomeVodMarkR(String src) { + return doReplaceRegex(homeVodMarkR, src); + } + + public String getCateUrl() { + return cateUrl; + } + + public String getCateVodNode() { + return cateVodNode; + } + + public String getCateVodName() { + return cateVodName; + } + + public String getCateVodNameR(String src) { + return doReplaceRegex(cateVodNameR, src); + } + + public String getCateVodId() { + return cateVodId; + } + + public String getCateVodIdR(String src) { + return doReplaceRegex(cateVodIdR, src); + } + + public String getCateVodImg() { + return cateVodImg; + } + + public String getCateVodImgR(String src) { + return doReplaceRegex(cateVodImgR, src); + } + + public String getCateVodMark() { + return cateVodMark; + } + + public String getCateVodMarkR(String src) { + return doReplaceRegex(cateVodNameR, src); + } + + public String getDetailUrl() { + return dtUrl; + } + + public String getDetailNode() { + return dtNode; + } + + public String getDetailName() { + return dtName; + } + + public String getDetailNameR(String src) { + return doReplaceRegex(dtNameR, src); + } + + public String getDetailImg() { + return dtImg; + } + + public String getDetailImgR(String src) { + return doReplaceRegex(dtImgR, src); + } + + public String getDetailCate() { + return dtCate; + } + + public String getDetailCateR(String src) { + return doReplaceRegex(dtCateR, src); + } + + public String getDetailYear() { + return dtYear; + } + + public String getDetailYearR(String src) { + return doReplaceRegex(dtYearR, src); + } + + public String getDetailArea() { + return dtArea; + } + + public String getDetailAreaR(String src) { + return doReplaceRegex(dtAreaR, src); + } + + public String getDetailMark() { + return dtMark; + } + + public String getDetailMarkR(String src) { + return doReplaceRegex(dtMarkR, src); + } + + public String getDetailActor() { + return dtActor; + } + + public String getDetailActorR(String src) { + return doReplaceRegex(dtActorR, src); + } + + public String getDetailDirector() { + return dtDirector; + } + + public String getDetailDirectorR(String src) { + return doReplaceRegex(dtDirectorR, src); + } + + public String getDetailDesc() { + return dtDesc; + } + + public String getDetailDescR(String src) { + return doReplaceRegex(dtDescR, src); + } + + public String getDetailFromNode() { + return dtFromNode; + } + + public String getDetailFromName() { + return dtFromName; + } + + public String getDetailFromNameR(String src) { + return doReplaceRegex(dtFromNameR, src); + } + + public String getDetailUrlNode() { + return dtUrlNode; + } + + public String getDetailUrlSubNode() { + return dtUrlSubNode; + } + + public String getDetailUrlId() { + return dtUrlId; + } + + public String getDetailUrlIdR(String src) { + return doReplaceRegex(dtUrlIdR, src); + } + + public String getDetailUrlName() { + return dtUrlName; + } + + public String getDetailUrlNameR(String src) { + return doReplaceRegex(dtUrlNameR, src); + } + + public String getPlayUrl() { + return playUrl; + } + + public String getPlayUa() { + return playUa; + } + + public String getSearchUrl() { + return searchUrl; + } + + public String getSearchVodNode() { + return scVodNode; + } + + public String getSearchVodName() { + return scVodName; + } + + public String getSearchVodNameR(String src) { + return doReplaceRegex(scVodNameR, src); + } + + public String getSearchVodId() { + return scVodId; + } + + public String getSearchVodIdR(String src) { + return doReplaceRegex(scVodIdR, src); + } + + public String getSearchVodImg() { + return scVodImg; + } + + public String getSearchVodImgR(String src) { + return doReplaceRegex(scVodImgR, src); + } + + public String getSearchVodMark() { + return scVodMark; + } + + public String getSearchVodMarkR(String src) { + return doReplaceRegex(scVodMarkR, src); + } +} \ No newline at end of file