Initial project - part 2
This commit is contained in:
FongMi
parent
81e4e9d0ee
commit
c9c5acf060
|
|
@ -0,0 +1,40 @@
|
|||
package com.github.catvod.crawler;
|
||||
|
||||
import android.content.Context;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
public abstract class Spider {
|
||||
|
||||
public void init(Context context) {
|
||||
}
|
||||
|
||||
public void init(Context context, String extend) {
|
||||
init(context);
|
||||
}
|
||||
|
||||
public String homeContent(boolean filter) {
|
||||
return "";
|
||||
}
|
||||
|
||||
public String homeVideoContent() {
|
||||
return "";
|
||||
}
|
||||
|
||||
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
|
||||
return "";
|
||||
}
|
||||
|
||||
public String detailContent(List<String> ids) {
|
||||
return "";
|
||||
}
|
||||
|
||||
public String searchContent(String key, boolean quick) {
|
||||
return "";
|
||||
}
|
||||
|
||||
public String playerContent(String flag, String id, List<String> vipFlags) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package com.github.catvod.crawler;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
public class SpiderDebug {
|
||||
|
||||
private static final String TAG = SpiderDebug.class.getSimpleName();
|
||||
|
||||
public static void log(Throwable th) {
|
||||
Log.d(TAG, th.getMessage());
|
||||
}
|
||||
|
||||
public static void log(String msg) {
|
||||
Log.d(TAG, msg);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
package com.github.catvod.live;
|
||||
|
||||
import com.github.catvod.net.OkHttpUtil;
|
||||
import com.github.catvod.utils.Trans;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.StringReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
|
||||
public class TxtSubscribe {
|
||||
|
||||
public static void subscribe(LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> allLives, String url, HashMap<String, String> headers) {
|
||||
String content = OkHttpUtil.string(url, headers);
|
||||
parse(allLives, content);
|
||||
}
|
||||
|
||||
public static void parse(LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> allLives, String txt) {
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new StringReader(txt));
|
||||
String line = br.readLine();
|
||||
LinkedHashMap<String, ArrayList<String>> noGroup = new LinkedHashMap<>();
|
||||
LinkedHashMap<String, ArrayList<String>> groupLives = noGroup;
|
||||
while (line != null) {
|
||||
if (line.trim().isEmpty()) {
|
||||
line = br.readLine();
|
||||
continue;
|
||||
}
|
||||
String[] lineInfo = line.split(",");
|
||||
if (lineInfo.length < 2) {
|
||||
line = br.readLine();
|
||||
continue;
|
||||
}
|
||||
if (line.contains("#genre#")) {
|
||||
String group = Trans.get(lineInfo[0].trim());
|
||||
if (!allLives.containsKey(group)) {
|
||||
groupLives = new LinkedHashMap<>();
|
||||
allLives.put(group, groupLives);
|
||||
} else {
|
||||
groupLives = allLives.get(group);
|
||||
}
|
||||
} else {
|
||||
String title = Trans.get(lineInfo[0].trim());
|
||||
String[] urlMix = lineInfo[1].trim().split("#");
|
||||
for (String mix : urlMix) {
|
||||
String url = mix.trim();
|
||||
if (url.isEmpty()) continue;
|
||||
if (url.startsWith("http") || url.startsWith("rtsp") || url.startsWith("rtmp")) {
|
||||
ArrayList<String> urls;
|
||||
if (!groupLives.containsKey(title)) {
|
||||
urls = new ArrayList<>();
|
||||
groupLives.put(title, urls);
|
||||
} else {
|
||||
urls = groupLives.get(title);
|
||||
}
|
||||
if (!urls.contains(url)) urls.add(url);
|
||||
}
|
||||
}
|
||||
}
|
||||
line = br.readLine();
|
||||
}
|
||||
br.close();
|
||||
if (!noGroup.isEmpty()) {
|
||||
allLives.put("未分類", noGroup);
|
||||
}
|
||||
} catch (Throwable ignored) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public static String live2Json(LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> lives) {
|
||||
JSONArray groups = new JSONArray();
|
||||
for (String group : lives.keySet()) {
|
||||
JSONArray channels = new JSONArray();
|
||||
LinkedHashMap<String, ArrayList<String>> allChannel = lives.get(group);
|
||||
if (allChannel.isEmpty()) continue;
|
||||
for (String channel : allChannel.keySet()) {
|
||||
ArrayList<String> allUrls = allChannel.get(channel);
|
||||
if (allUrls.isEmpty()) continue;
|
||||
JSONArray urls = new JSONArray();
|
||||
for (int i = 0; i < allUrls.size(); i++) {
|
||||
urls.put(allUrls.get(i));
|
||||
}
|
||||
JSONObject newChannel = new JSONObject();
|
||||
try {
|
||||
newChannel.put("name", channel);
|
||||
newChannel.put("urls", urls);
|
||||
} catch (JSONException ignored) {
|
||||
}
|
||||
channels.put(newChannel);
|
||||
}
|
||||
JSONObject newGroup = new JSONObject();
|
||||
try {
|
||||
newGroup.put("group", group);
|
||||
newGroup.put("channels", channels);
|
||||
} catch (JSONException ignored) {
|
||||
}
|
||||
groups.put(newGroup);
|
||||
}
|
||||
return groups.toString();
|
||||
}
|
||||
|
||||
public static Object[] load(String ext) {
|
||||
try {
|
||||
LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> allLives = new LinkedHashMap<>();
|
||||
TxtSubscribe.subscribe(allLives, ext, null);
|
||||
String json = TxtSubscribe.live2Json(allLives);
|
||||
Object[] result = new Object[3];
|
||||
result[0] = 200;
|
||||
result[1] = "text/plain; charset=utf-8";
|
||||
ByteArrayInputStream baos = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
|
||||
result[2] = baos;
|
||||
return result;
|
||||
} catch (Throwable th) {
|
||||
th.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
package com.github.catvod.net;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import okhttp3.Call;
|
||||
import okhttp3.Response;
|
||||
|
||||
public abstract class OKCallBack<T> {
|
||||
|
||||
private T result = null;
|
||||
|
||||
public T getResult() {
|
||||
return result;
|
||||
}
|
||||
|
||||
protected void setResult(T val) {
|
||||
result = val;
|
||||
}
|
||||
|
||||
protected void onError(final Call call, final Exception e) {
|
||||
onFailure(call, e);
|
||||
}
|
||||
|
||||
protected void onSuccess(Call call, Response response) {
|
||||
T obj = onParseResponse(call, response);
|
||||
setResult(obj);
|
||||
onResponse(obj);
|
||||
}
|
||||
|
||||
protected abstract T onParseResponse(Call call, Response response);
|
||||
|
||||
protected void onFailure(Call call, Exception e) {
|
||||
}
|
||||
|
||||
protected abstract void onResponse(T response);
|
||||
|
||||
public static class OKCallBackDefault extends OKCallBack<Response> {
|
||||
|
||||
@Override
|
||||
public Response onParseResponse(Call call, Response response) {
|
||||
return response;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onResponse(Response response) {
|
||||
}
|
||||
}
|
||||
|
||||
public static class OKCallBackString extends OKCallBack<String> {
|
||||
|
||||
@Override
|
||||
public void onError(Call call, Exception e) {
|
||||
setResult("");
|
||||
super.onError(call, e);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String onParseResponse(Call call, Response response) {
|
||||
try {
|
||||
return response.body().string();
|
||||
} catch (IOException e) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onFailure(Call call, Exception e) {
|
||||
setResult("");
|
||||
super.onFailure(call, e);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onResponse(String response) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
package com.github.catvod.net;
|
||||
|
||||
import android.text.TextUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
import okhttp3.Call;
|
||||
import okhttp3.FormBody;
|
||||
import okhttp3.MediaType;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.RequestBody;
|
||||
import okhttp3.Response;
|
||||
|
||||
class OKRequest {
|
||||
|
||||
private final String mMethodType;
|
||||
private final Map<String, String> mParamsMap;
|
||||
private final String mJsonStr;
|
||||
private final Map<String, String> mHeaderMap;
|
||||
private final OKCallBack mCallBack;
|
||||
private String mUrl;
|
||||
private Object mTag = null;
|
||||
private okhttp3.Request mOkHttpRequest;
|
||||
private okhttp3.Request.Builder mRequestBuilder;
|
||||
|
||||
OKRequest(String methodType, String url, Map<String, String> paramsMap, Map<String, String> headerMap, OKCallBack callBack) {
|
||||
this(methodType, url, null, paramsMap, headerMap, callBack);
|
||||
}
|
||||
|
||||
OKRequest(String methodType, String url, String jsonStr, Map<String, String> headerMap, OKCallBack callBack) {
|
||||
this(methodType, url, jsonStr, null, headerMap, callBack);
|
||||
}
|
||||
|
||||
private OKRequest(String methodType, String url, String jsonStr, Map<String, String> paramsMap, Map<String, String> headerMap, OKCallBack callBack) {
|
||||
mMethodType = methodType;
|
||||
mUrl = url;
|
||||
mJsonStr = jsonStr;
|
||||
mParamsMap = paramsMap;
|
||||
mHeaderMap = headerMap;
|
||||
mCallBack = callBack;
|
||||
getInstance();
|
||||
}
|
||||
|
||||
public void setTag(Object tag) {
|
||||
mTag = tag;
|
||||
}
|
||||
|
||||
private void getInstance() {
|
||||
mRequestBuilder = new okhttp3.Request.Builder();
|
||||
switch (mMethodType) {
|
||||
case OkHttpUtil.METHOD_GET:
|
||||
setGetParams();
|
||||
break;
|
||||
case OkHttpUtil.METHOD_POST:
|
||||
mRequestBuilder.post(getRequestBody());
|
||||
break;
|
||||
}
|
||||
mRequestBuilder.url(mUrl);
|
||||
if (mTag != null)
|
||||
mRequestBuilder.tag(mTag);
|
||||
if (mHeaderMap != null) {
|
||||
setHeader();
|
||||
}
|
||||
mOkHttpRequest = mRequestBuilder.build();
|
||||
}
|
||||
|
||||
private RequestBody getRequestBody() {
|
||||
if (!TextUtils.isEmpty(mJsonStr)) {
|
||||
MediaType JSON = MediaType.parse("application/json; charset=utf-8");
|
||||
return RequestBody.create(JSON, mJsonStr);
|
||||
}
|
||||
FormBody.Builder formBody = new FormBody.Builder();
|
||||
if (mParamsMap != null) {
|
||||
for (String key : mParamsMap.keySet()) {
|
||||
formBody.add(key, mParamsMap.get(key));
|
||||
}
|
||||
}
|
||||
return formBody.build();
|
||||
}
|
||||
|
||||
private void setGetParams() {
|
||||
if (mParamsMap != null) {
|
||||
mUrl = mUrl + "?";
|
||||
for (String key : mParamsMap.keySet()) {
|
||||
mUrl = mUrl + key + "=" + mParamsMap.get(key) + "&";
|
||||
}
|
||||
mUrl = mUrl.substring(0, mUrl.length() - 1);
|
||||
}
|
||||
}
|
||||
|
||||
private void setHeader() {
|
||||
if (mHeaderMap != null) {
|
||||
for (String key : mHeaderMap.keySet()) {
|
||||
mRequestBuilder.addHeader(key, mHeaderMap.get(key));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void execute(OkHttpClient client) {
|
||||
Call call = client.newCall(mOkHttpRequest);
|
||||
try {
|
||||
Response response = call.execute();
|
||||
if (mCallBack != null) {
|
||||
mCallBack.onSuccess(call, response);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
if (mCallBack != null) {
|
||||
mCallBack.onError(call, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,154 @@
|
|||
package com.github.catvod.net;
|
||||
|
||||
import com.github.catvod.crawler.Spider;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import okhttp3.Call;
|
||||
import okhttp3.Dns;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Response;
|
||||
|
||||
public class OkHttpUtil {
|
||||
|
||||
public static final String METHOD_GET = "GET";
|
||||
public static final String METHOD_POST = "POST";
|
||||
|
||||
private static final Object lockO = new Object();
|
||||
private static OkHttpClient defaultClient = null;
|
||||
private static OkHttpClient noRedirectClient = null;
|
||||
private static final int DEFAULT_TIMEOUT = 15;
|
||||
|
||||
public static OkHttpClient defaultClient() {
|
||||
synchronized (lockO) {
|
||||
if (defaultClient == null) {
|
||||
OkHttpClient.Builder builder = new OkHttpClient.Builder().dns(safeDns()).readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).retryOnConnectionFailure(true).sslSocketFactory(new SSLSocketFactoryCompat(SSLSocketFactoryCompat.trustAllCert), SSLSocketFactoryCompat.trustAllCert);
|
||||
defaultClient = builder.build();
|
||||
}
|
||||
return defaultClient;
|
||||
}
|
||||
}
|
||||
|
||||
public static OkHttpClient noRedirectClient() {
|
||||
synchronized (lockO) {
|
||||
if (noRedirectClient == null) {
|
||||
OkHttpClient.Builder builder = new OkHttpClient.Builder().dns(safeDns()).readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS).followRedirects(false).followSslRedirects(false).retryOnConnectionFailure(true).sslSocketFactory(new SSLSocketFactoryCompat(SSLSocketFactoryCompat.trustAllCert), SSLSocketFactoryCompat.trustAllCert);
|
||||
noRedirectClient = builder.build();
|
||||
}
|
||||
return noRedirectClient;
|
||||
}
|
||||
}
|
||||
|
||||
public static Dns safeDns() {
|
||||
try {
|
||||
return (Dns) Spider.class.getMethod("safeDns").invoke(null);
|
||||
} catch (Exception e) {
|
||||
return Dns.SYSTEM;
|
||||
}
|
||||
}
|
||||
|
||||
public static String string(OkHttpClient client, String url, String tag, Map<String, String> paramsMap, Map<String, String> headerMap, Map<String, List<String>> respHeaderMap) {
|
||||
return string(client, url, tag, paramsMap, headerMap, respHeaderMap, OkHttpUtil.METHOD_GET);
|
||||
}
|
||||
|
||||
public static String string(OkHttpClient client, String url, String tag, Map<String, String> paramsMap, Map<String, String> headerMap, Map<String, List<String>> respHeaderMap, String httpMethod) {
|
||||
OKCallBack.OKCallBackString callback = new OKCallBack.OKCallBackString() {
|
||||
@Override
|
||||
public String onParseResponse(Call call, Response response) {
|
||||
try {
|
||||
if (respHeaderMap != null) {
|
||||
respHeaderMap.clear();
|
||||
respHeaderMap.putAll(response.headers().toMultimap());
|
||||
}
|
||||
return response.body().string();
|
||||
} catch (IOException e) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
};
|
||||
OKRequest req = new OKRequest(httpMethod, url, paramsMap, headerMap, callback);
|
||||
req.setTag(tag);
|
||||
req.execute(client);
|
||||
return callback.getResult();
|
||||
}
|
||||
|
||||
public static String stringNoRedirect(String url, Map<String, String> headerMap, Map<String, List<String>> respHeaderMap) {
|
||||
return string(noRedirectClient(), url, null, null, headerMap, respHeaderMap);
|
||||
}
|
||||
|
||||
public static String string(String url) {
|
||||
return string(defaultClient(), url, null, null, null, null);
|
||||
}
|
||||
|
||||
public static String string(String url, Map<String, String> headerMap) {
|
||||
return string(defaultClient(), url, null, null, headerMap, null);
|
||||
}
|
||||
|
||||
public static String string(String url, Map<String, String> headerMap, Map<String, List<String>> respHeaderMap) {
|
||||
return string(defaultClient(), url, null, null, headerMap, respHeaderMap);
|
||||
}
|
||||
|
||||
public static String string(String url, String tag, Map<String, String> headerMap) {
|
||||
return string(defaultClient(), url, tag, null, headerMap, null);
|
||||
}
|
||||
|
||||
public static void get(OkHttpClient client, String url, OKCallBack callBack) {
|
||||
get(client, url, null, null, callBack);
|
||||
}
|
||||
|
||||
public static void get(OkHttpClient client, String url, Map<String, String> paramsMap, OKCallBack callBack) {
|
||||
get(client, url, paramsMap, null, callBack);
|
||||
}
|
||||
|
||||
public static void get(OkHttpClient client, String url, Map<String, String> paramsMap, Map<String, String> headerMap, OKCallBack callBack) {
|
||||
new OKRequest(METHOD_GET, url, paramsMap, headerMap, callBack).execute(client);
|
||||
}
|
||||
|
||||
public static String post(String url) {
|
||||
return post(url, null);
|
||||
}
|
||||
|
||||
public static String post(String url, Map<String, String> paramsMap) {
|
||||
return post(url, paramsMap, null);
|
||||
}
|
||||
|
||||
public static String post(String url, Map<String, String> paramsMap, Map<String, String> headerMap) {
|
||||
OKCallBack.OKCallBackString callback = new OKCallBack.OKCallBackString();
|
||||
new OKRequest(METHOD_POST, url, paramsMap, headerMap, callback).execute(defaultClient());
|
||||
return callback.getResult();
|
||||
}
|
||||
|
||||
public static String postJson(String url, String jsonStr, Map<String, String> headerMap) {
|
||||
OKCallBack.OKCallBackString callback = new OKCallBack.OKCallBackString();
|
||||
new OKRequest(METHOD_POST, url, jsonStr, headerMap, callback).execute(defaultClient());
|
||||
return callback.getResult();
|
||||
}
|
||||
|
||||
public static void cancel(OkHttpClient client, Object tag) {
|
||||
if (client == null || tag == null) return;
|
||||
for (Call call : client.dispatcher().queuedCalls()) {
|
||||
if (tag.equals(call.request().tag())) {
|
||||
call.cancel();
|
||||
}
|
||||
}
|
||||
for (Call call : client.dispatcher().runningCalls()) {
|
||||
if (tag.equals(call.request().tag())) {
|
||||
call.cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void cancel(Object tag) {
|
||||
cancel(defaultClient(), tag);
|
||||
}
|
||||
|
||||
public static String getRedirectLocation(Map<String, List<String>> headers) {
|
||||
if (headers == null) return null;
|
||||
if (headers.containsKey("location")) return headers.get("location").get(0);
|
||||
if (headers.containsKey("Location")) return headers.get("Location").get(0);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
package com.github.catvod.net;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.net.Socket;
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.SSLSocket;
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import javax.net.ssl.X509TrustManager;
|
||||
|
||||
public class SSLSocketFactoryCompat extends SSLSocketFactory {
|
||||
|
||||
public static final X509TrustManager trustAllCert = new X509TrustManager() {
|
||||
|
||||
@Override
|
||||
public void checkClientTrusted(X509Certificate[] chain, String authType) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkServerTrusted(X509Certificate[] chain, String authType) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public X509Certificate[] getAcceptedIssuers() {
|
||||
return new X509Certificate[]{};
|
||||
}
|
||||
};
|
||||
|
||||
static String[] protocols = null;
|
||||
static String[] cipherSuites = null;
|
||||
|
||||
static {
|
||||
try {
|
||||
SSLSocket socket = (SSLSocket) SSLSocketFactory.getDefault().createSocket();
|
||||
if (socket != null) {
|
||||
List<String> protocols = new LinkedList<>();
|
||||
for (String protocol : socket.getSupportedProtocols()) if (!protocol.toUpperCase().contains("SSL")) protocols.add(protocol);
|
||||
SSLSocketFactoryCompat.protocols = protocols.toArray(new String[protocols.size()]);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private final SSLSocketFactory defaultFactory;
|
||||
|
||||
public SSLSocketFactoryCompat(X509TrustManager tm) {
|
||||
try {
|
||||
SSLContext sslContext = SSLContext.getInstance("TLS");
|
||||
sslContext.init(null, (tm != null) ? new X509TrustManager[]{tm} : null, null);
|
||||
defaultFactory = sslContext.getSocketFactory();
|
||||
} catch (GeneralSecurityException e) {
|
||||
throw new AssertionError(); // The system has no TLS. Just give up.
|
||||
}
|
||||
}
|
||||
|
||||
private void upgradeTLS(SSLSocket ssl) {
|
||||
if (protocols != null) {
|
||||
ssl.setEnabledProtocols(protocols);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] getDefaultCipherSuites() {
|
||||
return cipherSuites;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] getSupportedCipherSuites() {
|
||||
return cipherSuites;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket(Socket s, String host, int port, boolean autoClose) throws IOException {
|
||||
Socket ssl = defaultFactory.createSocket(s, host, port, autoClose);
|
||||
if (ssl instanceof SSLSocket) upgradeTLS((SSLSocket) ssl);
|
||||
return ssl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket(String host, int port) throws IOException {
|
||||
Socket ssl = defaultFactory.createSocket(host, port);
|
||||
if (ssl instanceof SSLSocket) upgradeTLS((SSLSocket) ssl);
|
||||
return ssl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException {
|
||||
Socket ssl = defaultFactory.createSocket(host, port, localHost, localPort);
|
||||
if (ssl instanceof SSLSocket) upgradeTLS((SSLSocket) ssl);
|
||||
return ssl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket(InetAddress host, int port) throws IOException {
|
||||
Socket ssl = defaultFactory.createSocket(host, port);
|
||||
if (ssl instanceof SSLSocket) upgradeTLS((SSLSocket) ssl);
|
||||
return ssl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException {
|
||||
Socket ssl = defaultFactory.createSocket(address, port, localAddress, localPort);
|
||||
if (ssl instanceof SSLSocket) upgradeTLS((SSLSocket) ssl);
|
||||
return ssl;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
package com.github.catvod.parser;
|
||||
|
||||
import android.util.Base64;
|
||||
|
||||
import com.github.catvod.crawler.SpiderDebug;
|
||||
import com.github.catvod.net.OkHttpUtil;
|
||||
import com.github.catvod.utils.Misc;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Set;
|
||||
|
||||
public class JsonBasic {
|
||||
|
||||
public static JSONObject parse(LinkedHashMap<String, String> jx, String url) {
|
||||
SpiderDebug.log("Load Json Parse Basic...");
|
||||
if (jx.isEmpty()) return new JSONObject();
|
||||
Set<String> jxNames = jx.keySet();
|
||||
for (String jxName : jxNames) {
|
||||
String parseUrl = jx.get(jxName);
|
||||
HashMap<String, String> reqHeaders = getReqHeader(parseUrl);
|
||||
try {
|
||||
String realUrl = reqHeaders.get("url");
|
||||
reqHeaders.remove("url");
|
||||
SpiderDebug.log(realUrl + url);
|
||||
String json = OkHttpUtil.string(realUrl + url, reqHeaders);
|
||||
JSONObject taskResult = Misc.jsonParse(url, json);
|
||||
if (taskResult == null) continue;
|
||||
taskResult.put("jxFrom", jxName);
|
||||
SpiderDebug.log(taskResult.toString());
|
||||
return taskResult;
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
return new JSONObject();
|
||||
}
|
||||
|
||||
public static HashMap<String, String> getReqHeader(String url) {
|
||||
HashMap<String, String> reqHeaders = new HashMap<>();
|
||||
reqHeaders.put("url", url);
|
||||
if (!url.contains("cat_ext")) return reqHeaders;
|
||||
try {
|
||||
int start = url.indexOf("cat_ext=");
|
||||
int end = url.indexOf("&", start);
|
||||
String ext = url.substring(start + 8, end);
|
||||
ext = new String(Base64.decode(ext, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP));
|
||||
String newUrl = url.substring(0, start) + url.substring(end + 1);
|
||||
JSONObject jsonObject = new JSONObject(ext);
|
||||
if (jsonObject.has("header")) {
|
||||
JSONObject headerJson = jsonObject.optJSONObject("header");
|
||||
Iterator<String> keys = headerJson.keys();
|
||||
while (keys.hasNext()) {
|
||||
String key = keys.next();
|
||||
reqHeaders.put(key, headerJson.optString(key, ""));
|
||||
}
|
||||
}
|
||||
reqHeaders.put("url", newUrl);
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
return reqHeaders;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
package com.github.catvod.parser;
|
||||
|
||||
import com.github.catvod.crawler.SpiderDebug;
|
||||
import com.github.catvod.net.OkHttpUtil;
|
||||
import com.github.catvod.utils.Misc;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletionService;
|
||||
import java.util.concurrent.ExecutorCompletionService;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
public class JsonParallel {
|
||||
|
||||
private static final String ParseOKTag = "p_json_parse";
|
||||
|
||||
public static JSONObject parse(LinkedHashMap<String, String> jx, String url) {
|
||||
if (jx.isEmpty()) return new JSONObject();
|
||||
ExecutorService service = Executors.newFixedThreadPool(3);
|
||||
CompletionService<JSONObject> completionService = new ExecutorCompletionService<>(service);
|
||||
List<Future<JSONObject>> futures = new ArrayList<>();
|
||||
Set<String> jxNames = jx.keySet();
|
||||
for (String jxName : jxNames) {
|
||||
String parseUrl = jx.get(jxName);
|
||||
futures.add(completionService.submit(() -> {
|
||||
try {
|
||||
HashMap<String, String> reqHeaders = JsonBasic.getReqHeader(parseUrl);
|
||||
String realUrl = reqHeaders.get("url");
|
||||
reqHeaders.remove("url");
|
||||
SpiderDebug.log(realUrl + url);
|
||||
String json = OkHttpUtil.string(realUrl + url, ParseOKTag, reqHeaders);
|
||||
JSONObject taskResult = Misc.jsonParse(url, json);
|
||||
taskResult.put("jxFrom", jxName);
|
||||
SpiderDebug.log(taskResult.toString());
|
||||
return taskResult;
|
||||
} catch (Exception ignored) {
|
||||
return null;
|
||||
}
|
||||
}));
|
||||
}
|
||||
JSONObject pTaskResult = null;
|
||||
for (int i = 0; i < futures.size(); ++i) {
|
||||
try {
|
||||
Future<JSONObject> completed = completionService.take();
|
||||
pTaskResult = completed.get();
|
||||
if (pTaskResult != null) {
|
||||
OkHttpUtil.cancel(ParseOKTag);
|
||||
for (int j = 0; j < futures.size(); j++) {
|
||||
try {
|
||||
futures.get(j).cancel(true);
|
||||
} catch (Throwable th) {
|
||||
SpiderDebug.log(th);
|
||||
}
|
||||
}
|
||||
futures.clear();
|
||||
break;
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
service.shutdownNow();
|
||||
return pTaskResult != null ? pTaskResult : new JSONObject();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package com.github.catvod.parser;
|
||||
|
||||
import com.github.catvod.crawler.SpiderDebug;
|
||||
import com.github.catvod.net.OkHttpUtil;
|
||||
import com.github.catvod.utils.Misc;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Set;
|
||||
|
||||
public class JsonSequence {
|
||||
|
||||
public static JSONObject parse(LinkedHashMap<String, String> jx, String url) {
|
||||
if (jx.isEmpty()) return new JSONObject();
|
||||
Set<String> jxNames = jx.keySet();
|
||||
for (String jxName : jxNames) {
|
||||
try {
|
||||
String parseUrl = jx.get(jxName);
|
||||
HashMap<String, String> reqHeaders = JsonBasic.getReqHeader(parseUrl);
|
||||
String realUrl = reqHeaders.get("url");
|
||||
reqHeaders.remove("url");
|
||||
SpiderDebug.log(realUrl + url);
|
||||
String json = OkHttpUtil.string(realUrl + url, reqHeaders);
|
||||
JSONObject taskResult = Misc.jsonParse(url, json);
|
||||
if (taskResult == null) continue;
|
||||
taskResult.put("jxFrom", jxName);
|
||||
return taskResult;
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
return new JSONObject();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
package com.github.catvod.parser;
|
||||
|
||||
import android.util.Base64;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
|
||||
public class MixDemo {
|
||||
|
||||
private static final HashMap<String, ArrayList<String>> flagWebJx = new HashMap<>();
|
||||
private static HashMap<String, ArrayList<String>> configs = null;
|
||||
|
||||
private static void setConfigs(LinkedHashMap<String, HashMap<String, String>> jx) {
|
||||
configs = new HashMap<>();
|
||||
for (String key : jx.keySet()) {
|
||||
HashMap<String, String> parseBean = jx.get(key);
|
||||
String type = parseBean.get("type");
|
||||
if (type.equals("1") || type.equals("0")) {
|
||||
try {
|
||||
JSONArray flags = new JSONObject(parseBean.get("ext")).getJSONArray("flag");
|
||||
for (int j = 0; j < flags.length(); j++) {
|
||||
String flagKey = flags.getString(j);
|
||||
ArrayList<String> flagJx = configs.get(flagKey);
|
||||
if (flagJx == null) {
|
||||
flagJx = new ArrayList<>();
|
||||
configs.put(flagKey, flagJx);
|
||||
}
|
||||
flagJx.add(key);
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static JSONObject parse(LinkedHashMap<String, HashMap<String, String>> jx, String nameMe, String flag, String url) {
|
||||
try {
|
||||
if (configs == null) setConfigs(jx);
|
||||
LinkedHashMap<String, String> jsonJx = new LinkedHashMap<>();
|
||||
ArrayList<String> webJx = new ArrayList<>();
|
||||
ArrayList<String> flagJx = configs.get(flag);
|
||||
if (flagJx != null && !flagJx.isEmpty()) {
|
||||
for (int i = 0; i < flagJx.size(); i++) {
|
||||
String key = flagJx.get(i);
|
||||
HashMap<String, String> parseBean = jx.get(key);
|
||||
String type = parseBean.get("type");
|
||||
if (type.equals("1")) {
|
||||
jsonJx.put(key, mixUrl(parseBean.get("url"), parseBean.get("ext")));
|
||||
} else if (type.equals("0")) {
|
||||
webJx.add(parseBean.get("url"));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (String key : jx.keySet()) {
|
||||
HashMap<String, String> parseBean = jx.get(key);
|
||||
String type = parseBean.get("type");
|
||||
if (type.equals("1")) {
|
||||
jsonJx.put(key, mixUrl(parseBean.get("url"), parseBean.get("ext")));
|
||||
} else if (type.equals("0")) {
|
||||
webJx.add(parseBean.get("url"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!webJx.isEmpty()) flagWebJx.put(flag, webJx);
|
||||
JSONObject jsonResult = JsonParallel.parse(jsonJx, url);
|
||||
if (jsonResult.has("url")) return jsonResult;
|
||||
if (!webJx.isEmpty()) {
|
||||
JSONObject webResult = new JSONObject();
|
||||
webResult.put("url", "proxy://do=parseMix&flag=" + flag + "&url=" + Base64.encodeToString(url.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP));
|
||||
webResult.put("parse", 1);
|
||||
return webResult;
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
return new JSONObject();
|
||||
}
|
||||
|
||||
private static String mixUrl(String url, String ext) {
|
||||
if (ext.trim().isEmpty()) return url;
|
||||
int index = url.indexOf("?");
|
||||
if (index == -1) return url;
|
||||
return url.substring(0, index + 1) + "cat_ext=" + Base64.encodeToString(ext.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP) + "&" + url.substring(index + 1);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package com.github.catvod.spider;
|
||||
|
||||
import android.app.Application;
|
||||
import android.content.Context;
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
|
||||
import com.github.catvod.crawler.SpiderDebug;
|
||||
|
||||
public class Init {
|
||||
|
||||
private final Handler mHandler;
|
||||
private Application mApp;
|
||||
|
||||
private static class Loader {
|
||||
static volatile Init INSTANCE = new Init();
|
||||
}
|
||||
|
||||
private static Init get() {
|
||||
return Loader.INSTANCE;
|
||||
}
|
||||
|
||||
public Init() {
|
||||
this.mHandler = new Handler(Looper.getMainLooper());
|
||||
}
|
||||
|
||||
public static void init(Context context) {
|
||||
SpiderDebug.log("自定義爬蟲代碼載入成功！");
|
||||
get().mApp = ((Application) context);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
package com.github.catvod.spider;
|
||||
|
||||
import android.util.Base64;
|
||||
|
||||
import com.github.catvod.crawler.Spider;
|
||||
import com.github.catvod.crawler.SpiderDebug;
|
||||
import com.github.catvod.live.TxtSubscribe;
|
||||
import com.github.catvod.net.OkHttpUtil;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
public class Proxy extends Spider {
|
||||
|
||||
private static int port = -1;
|
||||
|
||||
public static Object[] proxy(Map<String, String> params) {
|
||||
switch (Objects.requireNonNull(params.get("do"))) {
|
||||
case "ck":
|
||||
return new Object[]{200, "text/plain; charset=utf-8", new ByteArrayInputStream("ok".getBytes(StandardCharsets.UTF_8))};
|
||||
case "live":
|
||||
return TxtSubscribe.load(new String(Base64.decode(params.get("ext"), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP), StandardCharsets.UTF_8));
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
static void adjustPort() {
|
||||
if (Proxy.port > 0) return;
|
||||
int port = 9978;
|
||||
while (port < 10000) {
|
||||
String resp = OkHttpUtil.string("http://127.0.0.1:" + port + "/proxy?do=ck", null);
|
||||
if (resp.equals("ok")) {
|
||||
SpiderDebug.log("Found local server port " + port);
|
||||
Proxy.port = port;
|
||||
break;
|
||||
}
|
||||
port++;
|
||||
}
|
||||
}
|
||||
|
||||
public static String getUrl() {
|
||||
adjustPort();
|
||||
return "http://127.0.0.1:" + port + "/proxy";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
package com.github.catvod.utils;
|
||||
|
||||
import android.net.Uri;
|
||||
|
||||
import com.github.catvod.crawler.SpiderDebug;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class Misc {
|
||||
|
||||
public static final String CHROME = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.62 Safari/537.36";
|
||||
private static final Pattern snifferMatch = Pattern.compile("http((?!http).){20,}?\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg)\\?.*|http((?!http).){20,}\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg)|http((?!http).){20,}?\\/m3u8\\?pt=m3u8.*|http((?!http).)*?default\\.ixigua\\.com\\/.*|http((?!http).)*?cdn-tos[^\\?]*|http((?!http).)*?\\/obj\\/tos[^\\?]*|http.*?\\/player\\/m3u8play\\.php\\?url=.*|http.*?\\/player\\/.*?[pP]lay\\.php\\?url=.*|http.*?\\/playlist\\/m3u8\\/\\?vid=.*|http.*?\\.php\\?type=m3u8&.*|http.*?\\/download.aspx\\?.*|http.*?\\/api\\/up_api.php\\?.*|https.*?\\.66yk\\.cn.*|http((?!http).)*?netease\\.com\\/file\\/.*");
|
||||
|
||||
public static boolean isVip(String url) {
|
||||
try {
|
||||
boolean isVip = false;
|
||||
String host = Uri.parse(url).getHost();
|
||||
String[] vipWebsites = new String[]{"iqiyi.com", "v.qq.com", "youku.com", "le.com", "tudou.com", "mgtv.com", "sohu.com", "acfun.cn", "bilibili.com", "baofeng.com", "pptv.com"};
|
||||
for (String vipWebsite : vipWebsites) {
|
||||
if (host.contains(vipWebsite)) {
|
||||
if ("iqiyi.com".equals(vipWebsite)) {
|
||||
if (url.contains("iqiyi.com/a_") || url.contains("iqiyi.com/w_") || url.contains("iqiyi.com/v_")) {
|
||||
isVip = true;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
isVip = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return isVip;
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static boolean isVideoFormat(String url) {
|
||||
if (snifferMatch.matcher(url).find()) {
|
||||
return !url.contains("cdn-tos") || !url.contains(".js");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static String fixUrl(String base, String src) {
|
||||
try {
|
||||
if (src.startsWith("//")) {
|
||||
Uri parse = Uri.parse(base);
|
||||
src = parse.getScheme() + ":" + src;
|
||||
} else if (!src.contains("://")) {
|
||||
Uri parse = Uri.parse(base);
|
||||
src = parse.getScheme() + "://" + parse.getHost() + src;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
SpiderDebug.log(e);
|
||||
}
|
||||
return src;
|
||||
}
|
||||
|
||||
public static JSONObject fixJsonVodHeader(JSONObject headers, String input, String url) throws JSONException {
|
||||
if (headers == null)
|
||||
headers = new JSONObject();
|
||||
if (input.contains("www.mgtv.com")) {
|
||||
headers.put("Referer", " ");
|
||||
headers.put("User-Agent", " Mozilla/5.0");
|
||||
} else if (url.contains("titan.mgtv")) {
|
||||
headers.put("Referer", " ");
|
||||
headers.put("User-Agent", " Mozilla/5.0");
|
||||
} else if (input.contains("bilibili")) {
|
||||
headers.put("Referer", " https://www.bilibili.com/");
|
||||
headers.put("User-Agent", " " + Misc.CHROME);
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
public static JSONObject jsonParse(String input, String json) throws JSONException {
|
||||
JSONObject jsonPlayData = new JSONObject(json);
|
||||
String url = jsonPlayData.getString("url");
|
||||
if (url.startsWith("//")) {
|
||||
url = "https:" + url;
|
||||
}
|
||||
if (!url.startsWith("http")) {
|
||||
return null;
|
||||
}
|
||||
if (url.equals(input)) {
|
||||
if (isVip(url) || !isVideoFormat(url)) return null;
|
||||
}
|
||||
JSONObject headers = new JSONObject();
|
||||
String ua = jsonPlayData.optString("user-agent", "");
|
||||
if (ua.trim().length() > 0) {
|
||||
headers.put("User-Agent", " " + ua);
|
||||
}
|
||||
String referer = jsonPlayData.optString("referer", "");
|
||||
if (referer.trim().length() > 0) {
|
||||
headers.put("Referer", " " + referer);
|
||||
}
|
||||
headers = Misc.fixJsonVodHeader(headers, input, url);
|
||||
JSONObject taskResult = new JSONObject();
|
||||
taskResult.put("header", headers);
|
||||
taskResult.put("url", url);
|
||||
return taskResult;
|
||||
}
|
||||
|
||||
public static String MD5(String src) {
|
||||
return MD5(src, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public static String MD5(String src, Charset charset) {
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance("MD5");
|
||||
byte[] messageDigest = md.digest(src.getBytes(charset));
|
||||
BigInteger no = new BigInteger(1, messageDigest);
|
||||
StringBuilder sb = new StringBuilder(no.toString(16));
|
||||
while (sb.length() < 32) sb.insert(0, "0");
|
||||
return sb.toString().toLowerCase();
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,686 @@
|
|||
package com.github.catvod.xpath;
|
||||
|
||||
import com.github.catvod.crawler.SpiderDebug;
|
||||
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class XPathRule {
|
||||
/**
|
||||
* user-agent
|
||||
*/
|
||||
private String ua;
|
||||
/**
|
||||
* 取得分類和首頁推薦的Url
|
||||
*/
|
||||
private String homeUrl;
|
||||
/**
|
||||
* 分類節點 xpath
|
||||
*/
|
||||
private String cateNode;
|
||||
/**
|
||||
* 分類節點名 xpath
|
||||
*/
|
||||
private String cateName;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern cateNameR;
|
||||
/**
|
||||
* 分類節點 id xpath
|
||||
*/
|
||||
private String cateId;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern cateIdR;
|
||||
/**
|
||||
* 手動指定分類如果有則不從 homeUrl 中獲取分類
|
||||
*/
|
||||
private final LinkedHashMap<String, String> cateManual = new LinkedHashMap<>();
|
||||
|
||||
/**
|
||||
* 篩選
|
||||
*/
|
||||
private JSONObject filter;
|
||||
|
||||
/**
|
||||
* 更新推薦影片節點 xpath
|
||||
*/
|
||||
private String homeVodNode;
|
||||
/**
|
||||
* 更新推薦影片名稱 xpath
|
||||
*/
|
||||
private String homeVodName;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern homeVodNameR;
|
||||
/**
|
||||
* 更新推薦影片 id xpath
|
||||
*/
|
||||
private String homeVodId;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern homeVodIdR;
|
||||
/**
|
||||
* 更新推薦影片圖片 xpath
|
||||
*/
|
||||
private String homeVodImg;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern homeVodImgR;
|
||||
/**
|
||||
* 更新推薦影片簡介 xpath
|
||||
*/
|
||||
private String homeVodMark;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern homeVodMarkR;
|
||||
/**
|
||||
* 分類頁地址
|
||||
*/
|
||||
private String cateUrl;
|
||||
/**
|
||||
* 分類頁影片節點 xpath
|
||||
*/
|
||||
private String cateVodNode;
|
||||
/**
|
||||
* 分類頁影片名稱 xpath
|
||||
*/
|
||||
private String cateVodName;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern cateVodNameR;
|
||||
/**
|
||||
* 分類頁影片影片id xpath
|
||||
*/
|
||||
private String cateVodId;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern cateVodIdR;
|
||||
/**
|
||||
* 分類頁影片影片圖片 xpath
|
||||
*/
|
||||
private String cateVodImg;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern cateVodImgR;
|
||||
/**
|
||||
* 分類頁影片影片簡介 xpath
|
||||
*/
|
||||
private String cateVodMark;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern cateVodMarkR;
|
||||
|
||||
/**
|
||||
* 詳情頁面
|
||||
*/
|
||||
private String dtUrl;
|
||||
/**
|
||||
* 詳情節點 xpath
|
||||
*/
|
||||
private String dtNode;
|
||||
/**
|
||||
* 詳情影片 xpath
|
||||
*/
|
||||
private String dtName;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern dtNameR;
|
||||
/**
|
||||
* 詳情影片圖片 xpath
|
||||
*/
|
||||
private String dtImg;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern dtImgR;
|
||||
/**
|
||||
* 詳情影片分類 xpath
|
||||
*/
|
||||
private String dtCate;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern dtCateR;
|
||||
/**
|
||||
* 詳情影片年份 xpath
|
||||
*/
|
||||
private String dtYear;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern dtYearR;
|
||||
/**
|
||||
* 詳情影片地區 xpath
|
||||
*/
|
||||
private String dtArea;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern dtAreaR;
|
||||
/**
|
||||
* 詳情影片簡介 xpath
|
||||
*/
|
||||
private String dtMark;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern dtMarkR;
|
||||
/**
|
||||
* 詳情演員 xpath
|
||||
*/
|
||||
private String dtActor;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern dtActorR;
|
||||
/**
|
||||
* 詳情導演 xpath
|
||||
*/
|
||||
private String dtDirector;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern dtDirectorR;
|
||||
/**
|
||||
* 詳情說明 xpath
|
||||
*/
|
||||
private String dtDesc;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern dtDescR;
|
||||
|
||||
/**
|
||||
* 詳情播放來源節點
|
||||
*/
|
||||
private String dtFromNode;
|
||||
/**
|
||||
* 詳情播放來源名稱 xpath
|
||||
*/
|
||||
private String dtFromName;
|
||||
/**
|
||||
* 詳情
|
||||
*/
|
||||
private Pattern dtFromNameR;
|
||||
/**
|
||||
* 詳情播放地址列表節點 xpath
|
||||
*/
|
||||
private String dtUrlNode;
|
||||
/**
|
||||
* 詳情播放地址節點 xpath
|
||||
*/
|
||||
private String dtUrlSubNode;
|
||||
/**
|
||||
* 詳情播放地址id xpath
|
||||
*/
|
||||
private String dtUrlId;
|
||||
/**
|
||||
* 詳情
|
||||
*/
|
||||
private Pattern dtUrlIdR;
|
||||
/**
|
||||
* 詳情播放地址名稱 xpath
|
||||
*/
|
||||
private String dtUrlName;
|
||||
/**
|
||||
* 詳情
|
||||
*/
|
||||
private Pattern dtUrlNameR;
|
||||
/**
|
||||
* 播放頁面url
|
||||
*/
|
||||
private String playUrl;
|
||||
/**
|
||||
* 播放解析調用ua
|
||||
*/
|
||||
private String playUa;
|
||||
|
||||
/**
|
||||
* 搜尋頁地址
|
||||
*/
|
||||
private String searchUrl;
|
||||
|
||||
/**
|
||||
* 搜尋頁影片節點 xpath
|
||||
*/
|
||||
private String scVodNode;
|
||||
/**
|
||||
* 搜尋頁影片名稱 xpath
|
||||
*/
|
||||
private String scVodName;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern scVodNameR;
|
||||
/**
|
||||
* 搜尋頁影片id xpath
|
||||
*/
|
||||
private String scVodId;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern scVodIdR;
|
||||
/**
|
||||
* 搜尋頁影片圖片 xpath
|
||||
*/
|
||||
private String scVodImg;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern scVodImgR;
|
||||
/**
|
||||
* 搜尋頁影片簡介 xpath
|
||||
*/
|
||||
private String scVodMark;
|
||||
/**
|
||||
* 正則對取到的數據進行二次處理
|
||||
*/
|
||||
private Pattern scVodMarkR;
|
||||
|
||||
private static Pattern getPattern(JSONObject json, String key) {
|
||||
String v = json.optString(key).trim();
|
||||
if (v.isEmpty())
|
||||
return null;
|
||||
else {
|
||||
try {
|
||||
return Pattern.compile(v);
|
||||
} catch (Exception e) {
|
||||
SpiderDebug.log(e);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static String doReplaceRegex(Pattern pattern, String src) {
|
||||
if (pattern == null)
|
||||
return src;
|
||||
try {
|
||||
Matcher matcher = pattern.matcher(src);
|
||||
if (matcher.find()) {
|
||||
return matcher.group(1).trim();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
SpiderDebug.log(e);
|
||||
}
|
||||
return src;
|
||||
}
|
||||
|
||||
public static XPathRule fromJson(String json) {
|
||||
try {
|
||||
JSONObject jsonObj = new JSONObject(json);
|
||||
XPathRule rule = new XPathRule();
|
||||
rule.ua = jsonObj.optString("ua");
|
||||
rule.homeUrl = jsonObj.optString("homeUrl").trim();
|
||||
rule.cateNode = jsonObj.optString("cateNode").trim();
|
||||
rule.cateName = jsonObj.optString("cateName").trim();
|
||||
rule.cateNameR = getPattern(jsonObj, "cateNameR");
|
||||
rule.cateId = jsonObj.optString("cateId").trim();
|
||||
rule.cateIdR = getPattern(jsonObj, "cateIdR");
|
||||
JSONObject navs = jsonObj.optJSONObject("cateManual");
|
||||
if (navs != null) {
|
||||
Iterator<String> keys = navs.keys();
|
||||
while (keys.hasNext()) {
|
||||
String name = keys.next();
|
||||
rule.cateManual.put(name.trim(), navs.getString(name).trim());
|
||||
}
|
||||
}
|
||||
rule.filter = jsonObj.optJSONObject("filter");
|
||||
rule.homeVodNode = jsonObj.optString("homeVodNode").trim();
|
||||
rule.homeVodName = jsonObj.optString("homeVodName").trim();
|
||||
rule.homeVodNameR = getPattern(jsonObj, "homeVodNameR");
|
||||
rule.homeVodId = jsonObj.optString("homeVodId").trim();
|
||||
rule.homeVodIdR = getPattern(jsonObj, "homeVodIdR");
|
||||
rule.homeVodImg = jsonObj.optString("homeVodImg").trim();
|
||||
rule.homeVodImgR = getPattern(jsonObj, "homeVodImgR");
|
||||
rule.homeVodMark = jsonObj.optString("homeVodMark").trim();
|
||||
rule.homeVodMarkR = getPattern(jsonObj, "homeVodMarkR");
|
||||
rule.cateUrl = jsonObj.optString("cateUrl").trim();
|
||||
rule.cateVodNode = jsonObj.optString("cateVodNode").trim();
|
||||
rule.cateVodName = jsonObj.optString("cateVodName").trim();
|
||||
rule.cateVodNameR = getPattern(jsonObj, "cateVodNameR");
|
||||
rule.cateVodId = jsonObj.optString("cateVodId").trim();
|
||||
rule.cateVodIdR = getPattern(jsonObj, "cateVodIdR");
|
||||
rule.cateVodImg = jsonObj.optString("cateVodImg").trim();
|
||||
rule.cateVodImgR = getPattern(jsonObj, "cateVodImgR");
|
||||
rule.cateVodMark = jsonObj.optString("cateVodMark").trim();
|
||||
rule.cateVodMarkR = getPattern(jsonObj, "cateVodMarkR");
|
||||
rule.dtUrl = jsonObj.optString("dtUrl");
|
||||
rule.dtNode = jsonObj.optString("dtNode");
|
||||
rule.dtName = jsonObj.optString("dtName");
|
||||
rule.dtNameR = getPattern(jsonObj, "dtNameR");
|
||||
rule.dtImg = jsonObj.optString("dtImg");
|
||||
rule.dtImgR = getPattern(jsonObj, "dtImgR");
|
||||
rule.dtCate = jsonObj.optString("dtCate");
|
||||
rule.dtCateR = getPattern(jsonObj, "dtCateR");
|
||||
rule.dtYear = jsonObj.optString("dtYear");
|
||||
rule.dtYearR = getPattern(jsonObj, "dtYearR");
|
||||
rule.dtArea = jsonObj.optString("dtArea");
|
||||
rule.dtAreaR = getPattern(jsonObj, "dtAreaR");
|
||||
rule.dtMark = jsonObj.optString("dtMark");
|
||||
rule.dtMarkR = getPattern(jsonObj, "dtMarkR");
|
||||
rule.dtActor = jsonObj.optString("dtActor");
|
||||
rule.dtActorR = getPattern(jsonObj, "dtActorR");
|
||||
rule.dtDirector = jsonObj.optString("dtDirector");
|
||||
rule.dtDirectorR = getPattern(jsonObj, "dtDirectorR");
|
||||
rule.dtDesc = jsonObj.optString("dtDesc");
|
||||
rule.dtDescR = getPattern(jsonObj, "dtDescR");
|
||||
rule.dtFromNode = jsonObj.optString("dtFromNode");
|
||||
rule.dtFromName = jsonObj.optString("dtFromName");
|
||||
rule.dtFromNameR = getPattern(jsonObj, "dtFromNameR");
|
||||
rule.dtUrlNode = jsonObj.optString("dtUrlNode");
|
||||
rule.dtUrlSubNode = jsonObj.optString("dtUrlSubNode");
|
||||
rule.dtUrlId = jsonObj.optString("dtUrlId");
|
||||
rule.dtUrlIdR = getPattern(jsonObj, "dtUrlIdR");
|
||||
rule.dtUrlName = jsonObj.optString("dtUrlName");
|
||||
rule.dtUrlNameR = getPattern(jsonObj, "dtUrlNameR");
|
||||
rule.playUrl = jsonObj.optString("playUrl");
|
||||
rule.playUa = jsonObj.optString("playUa");
|
||||
rule.searchUrl = jsonObj.optString("searchUrl");
|
||||
rule.scVodNode = jsonObj.optString("scVodNode").trim();
|
||||
rule.scVodName = jsonObj.optString("scVodName").trim();
|
||||
rule.scVodNameR = getPattern(jsonObj, "scVodNameR");
|
||||
rule.scVodId = jsonObj.optString("scVodId").trim();
|
||||
rule.scVodIdR = getPattern(jsonObj, "scVodIdR");
|
||||
rule.scVodImg = jsonObj.optString("scVodImg").trim();
|
||||
rule.scVodImgR = getPattern(jsonObj, "scVodImgR");
|
||||
rule.scVodMark = jsonObj.optString("scVodMark").trim();
|
||||
rule.scVodMarkR = getPattern(jsonObj, "scVodMarkR");
|
||||
return rule;
|
||||
} catch (Exception e) {
|
||||
SpiderDebug.log(e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public String getUa() {
|
||||
return ua;
|
||||
}
|
||||
|
||||
public String getHomeUrl() {
|
||||
return homeUrl;
|
||||
}
|
||||
|
||||
public String getCateNode() {
|
||||
return cateNode;
|
||||
}
|
||||
|
||||
public String getCateName() {
|
||||
return cateName;
|
||||
}
|
||||
|
||||
public String getCateNameR(String src) {
|
||||
return doReplaceRegex(cateNameR, src);
|
||||
}
|
||||
|
||||
public String getCateId() {
|
||||
return cateId;
|
||||
}
|
||||
|
||||
public String getCateIdR(String src) {
|
||||
return doReplaceRegex(cateIdR, src);
|
||||
}
|
||||
|
||||
public LinkedHashMap<String, String> getCateManual() {
|
||||
return cateManual;
|
||||
}
|
||||
|
||||
public JSONObject getFilter() {
|
||||
return filter;
|
||||
}
|
||||
|
||||
public String getHomeVodNode() {
|
||||
return homeVodNode;
|
||||
}
|
||||
|
||||
public String getHomeVodName() {
|
||||
return homeVodName;
|
||||
}
|
||||
|
||||
public String getHomeVodNameR(String src) {
|
||||
return doReplaceRegex(homeVodNameR, src);
|
||||
}
|
||||
|
||||
public String getHomeVodId() {
|
||||
return homeVodId;
|
||||
}
|
||||
|
||||
public String getHomeVodIdR(String src) {
|
||||
return doReplaceRegex(homeVodIdR, src);
|
||||
}
|
||||
|
||||
public String getHomeVodImg() {
|
||||
return homeVodImg;
|
||||
}
|
||||
|
||||
public String getHomeVodImgR(String src) {
|
||||
return doReplaceRegex(homeVodImgR, src);
|
||||
}
|
||||
|
||||
public String getHomeVodMark() {
|
||||
return homeVodMark;
|
||||
}
|
||||
|
||||
public String getHomeVodMarkR(String src) {
|
||||
return doReplaceRegex(homeVodMarkR, src);
|
||||
}
|
||||
|
||||
public String getCateUrl() {
|
||||
return cateUrl;
|
||||
}
|
||||
|
||||
public String getCateVodNode() {
|
||||
return cateVodNode;
|
||||
}
|
||||
|
||||
public String getCateVodName() {
|
||||
return cateVodName;
|
||||
}
|
||||
|
||||
public String getCateVodNameR(String src) {
|
||||
return doReplaceRegex(cateVodNameR, src);
|
||||
}
|
||||
|
||||
public String getCateVodId() {
|
||||
return cateVodId;
|
||||
}
|
||||
|
||||
public String getCateVodIdR(String src) {
|
||||
return doReplaceRegex(cateVodIdR, src);
|
||||
}
|
||||
|
||||
public String getCateVodImg() {
|
||||
return cateVodImg;
|
||||
}
|
||||
|
||||
public String getCateVodImgR(String src) {
|
||||
return doReplaceRegex(cateVodImgR, src);
|
||||
}
|
||||
|
||||
public String getCateVodMark() {
|
||||
return cateVodMark;
|
||||
}
|
||||
|
||||
public String getCateVodMarkR(String src) {
|
||||
return doReplaceRegex(cateVodNameR, src);
|
||||
}
|
||||
|
||||
public String getDetailUrl() {
|
||||
return dtUrl;
|
||||
}
|
||||
|
||||
public String getDetailNode() {
|
||||
return dtNode;
|
||||
}
|
||||
|
||||
public String getDetailName() {
|
||||
return dtName;
|
||||
}
|
||||
|
||||
public String getDetailNameR(String src) {
|
||||
return doReplaceRegex(dtNameR, src);
|
||||
}
|
||||
|
||||
public String getDetailImg() {
|
||||
return dtImg;
|
||||
}
|
||||
|
||||
public String getDetailImgR(String src) {
|
||||
return doReplaceRegex(dtImgR, src);
|
||||
}
|
||||
|
||||
public String getDetailCate() {
|
||||
return dtCate;
|
||||
}
|
||||
|
||||
public String getDetailCateR(String src) {
|
||||
return doReplaceRegex(dtCateR, src);
|
||||
}
|
||||
|
||||
public String getDetailYear() {
|
||||
return dtYear;
|
||||
}
|
||||
|
||||
public String getDetailYearR(String src) {
|
||||
return doReplaceRegex(dtYearR, src);
|
||||
}
|
||||
|
||||
public String getDetailArea() {
|
||||
return dtArea;
|
||||
}
|
||||
|
||||
public String getDetailAreaR(String src) {
|
||||
return doReplaceRegex(dtAreaR, src);
|
||||
}
|
||||
|
||||
public String getDetailMark() {
|
||||
return dtMark;
|
||||
}
|
||||
|
||||
public String getDetailMarkR(String src) {
|
||||
return doReplaceRegex(dtMarkR, src);
|
||||
}
|
||||
|
||||
public String getDetailActor() {
|
||||
return dtActor;
|
||||
}
|
||||
|
||||
public String getDetailActorR(String src) {
|
||||
return doReplaceRegex(dtActorR, src);
|
||||
}
|
||||
|
||||
public String getDetailDirector() {
|
||||
return dtDirector;
|
||||
}
|
||||
|
||||
public String getDetailDirectorR(String src) {
|
||||
return doReplaceRegex(dtDirectorR, src);
|
||||
}
|
||||
|
||||
public String getDetailDesc() {
|
||||
return dtDesc;
|
||||
}
|
||||
|
||||
public String getDetailDescR(String src) {
|
||||
return doReplaceRegex(dtDescR, src);
|
||||
}
|
||||
|
||||
public String getDetailFromNode() {
|
||||
return dtFromNode;
|
||||
}
|
||||
|
||||
public String getDetailFromName() {
|
||||
return dtFromName;
|
||||
}
|
||||
|
||||
public String getDetailFromNameR(String src) {
|
||||
return doReplaceRegex(dtFromNameR, src);
|
||||
}
|
||||
|
||||
public String getDetailUrlNode() {
|
||||
return dtUrlNode;
|
||||
}
|
||||
|
||||
public String getDetailUrlSubNode() {
|
||||
return dtUrlSubNode;
|
||||
}
|
||||
|
||||
public String getDetailUrlId() {
|
||||
return dtUrlId;
|
||||
}
|
||||
|
||||
public String getDetailUrlIdR(String src) {
|
||||
return doReplaceRegex(dtUrlIdR, src);
|
||||
}
|
||||
|
||||
public String getDetailUrlName() {
|
||||
return dtUrlName;
|
||||
}
|
||||
|
||||
public String getDetailUrlNameR(String src) {
|
||||
return doReplaceRegex(dtUrlNameR, src);
|
||||
}
|
||||
|
||||
public String getPlayUrl() {
|
||||
return playUrl;
|
||||
}
|
||||
|
||||
public String getPlayUa() {
|
||||
return playUa;
|
||||
}
|
||||
|
||||
public String getSearchUrl() {
|
||||
return searchUrl;
|
||||
}
|
||||
|
||||
public String getSearchVodNode() {
|
||||
return scVodNode;
|
||||
}
|
||||
|
||||
public String getSearchVodName() {
|
||||
return scVodName;
|
||||
}
|
||||
|
||||
public String getSearchVodNameR(String src) {
|
||||
return doReplaceRegex(scVodNameR, src);
|
||||
}
|
||||
|
||||
public String getSearchVodId() {
|
||||
return scVodId;
|
||||
}
|
||||
|
||||
public String getSearchVodIdR(String src) {
|
||||
return doReplaceRegex(scVodIdR, src);
|
||||
}
|
||||
|
||||
public String getSearchVodImg() {
|
||||
return scVodImg;
|
||||
}
|
||||
|
||||
public String getSearchVodImgR(String src) {
|
||||
return doReplaceRegex(scVodImgR, src);
|
||||
}
|
||||
|
||||
public String getSearchVodMark() {
|
||||
return scVodMark;
|
||||
}
|
||||
|
||||
public String getSearchVodMarkR(String src) {
|
||||
return doReplaceRegex(scVodMarkR, src);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue