dongge
/
AndroidCatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean code

This commit is contained in:
FongMi 2023-09-24 14:53:44 +08:00
parent a2e9ca7074
commit 9ca8f8efa0
10 changed files with 38 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/src/main/java/com/github/catvod/crawler/Spider.java
View File

@ -2,11 +2,14 @@ package com.github.catvod.crawler;
import android.content.Context; import android.content.Context;
import com.github.catvod.net.OkHttp;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import okhttp3.Dns; import okhttp3.Dns;
import okhttp3.OkHttpClient;
public abstract class Spider { public abstract class Spider {
@ -74,4 +77,8 @@ public abstract class Spider {
public static Dns safeDns() { public static Dns safeDns() {
return Dns.SYSTEM; return Dns.SYSTEM;
} }
public OkHttpClient client() {
return OkHttp.client();
}
} }

76
app/src/main/java/com/github/catvod/net/Cronet.java
View File

@ -1,7 +1,6 @@
package com.github.catvod.net; package com.github.catvod.net;
import com.github.catvod.spider.Init; import com.github.catvod.spider.Init;
import com.github.catvod.utils.Prefers;
import com.google.net.cronet.okhttptransport.CronetInterceptor; import com.google.net.cronet.okhttptransport.CronetInterceptor;
import org.chromium.net.CronetEngine; import org.chromium.net.CronetEngine;
@ -12,7 +11,6 @@ import okhttp3.OkHttpClient;
public class Cronet { public class Cronet {
private OkHttpClient proxy;
private OkHttpClient client; private OkHttpClient client;
private static class Loader { private static class Loader {
@ -25,92 +23,42 @@ public class Cronet {
public static OkHttpClient client() { public static OkHttpClient client() {
if (get().client != null) return get().client; if (get().client != null) return get().client;
return get().client = getBuilder(OkHttp.getBuilder()).build(); return get().client = getBuilder().build();
} }
public static OkHttpClient proxy() { public static OkHttpClient noRedirect() {
if (get().proxy != null) return get().proxy; return client().newBuilder().followRedirects(false).followSslRedirects(false).build();
return get().proxy = getBuilder(OkHttp.getBuilder(Prefers.getString("proxy"))).build();
}
public static OkHttpClient client(boolean proxy) {
return proxy ? proxy() : client();
}
public static OkHttpClient noRedirect(boolean proxy) {
return client(proxy).newBuilder().followRedirects(false).followSslRedirects(false).build();
} }
public static String string(String url) { public static String string(String url) {
return string(false, url); return string(url, null);
}
public static String string(boolean proxy, String url) {
return string(proxy, url, null);
} }
public static String string(String url, Map<String, String> header) { public static String string(String url, Map<String, String> header) {
return string(false, url, header); return string(url, null, header);
}
public static String string(boolean proxy, String url, Map<String, String> header) {
return string(proxy, url, null, header);
} }
public static String string(String url, Map<String, String> params, Map<String, String> header) { public static String string(String url, Map<String, String> params, Map<String, String> header) {
return string(false, url, params, header); return new OkRequest(OkHttp.GET, url, params, header).execute(client()).getBody();
}
public static String string(boolean proxy, String url, Map<String, String> params, Map<String, String> header) {
return string(client(proxy), url, params, header);
}
public static String string(OkHttpClient client, String url, Map<String, String> params, Map<String, String> header) {
return new OkRequest(OkHttp.GET, url, params, header).execute(client).getBody();
} }
public static String post(String url, Map<String, String> params) { public static String post(String url, Map<String, String> params) {
return post(false, url, params); return post(url, params, null).getBody();
}
public static String post(boolean proxy, String url, Map<String, String> params) {
return post(proxy, url, params, null).getBody();
} }
public static OkResult post(String url, Map<String, String> params, Map<String, String> header) { public static OkResult post(String url, Map<String, String> params, Map<String, String> header) {
return post(false, url, params, header); return new OkRequest(OkHttp.POST, url, params, header).execute(client());
}
public static OkResult post(boolean proxy, String url, Map<String, String> params, Map<String, String> header) {
return new OkRequest(OkHttp.POST, url, params, header).execute(client(proxy));
} }
public static String post(String url, String json) { public static String post(String url, String json) {
return post(false, url, json); return post(url, json, null).getBody();
}
public static String post(boolean proxy, String url, String json) {
return post(proxy, url, json, null).getBody();
} }
public static OkResult post(String url, String json, Map<String, String> header) { public static OkResult post(String url, String json, Map<String, String> header) {
return post(false, url, json, header); return new OkRequest(OkHttp.POST, url, json, header).execute(client());
} }
public static OkResult post(boolean proxy, String url, String json, Map<String, String> header) { private static OkHttpClient.Builder getBuilder() {
return new OkRequest(OkHttp.POST, url, json, header).execute(client(proxy)); return OkHttp.getBuilder().addInterceptor(CronetInterceptor.newBuilder(new CronetEngine.Builder(Init.context()).build()).build());
}
private static OkHttpClient.Builder getBuilder(OkHttpClient.Builder builder) {
addInterceptor(builder);
return builder;
}
private static void addInterceptor(OkHttpClient.Builder builder) {
try {
builder.addInterceptor(CronetInterceptor.newBuilder(new CronetEngine.Builder(Init.context()).build()).build());
} catch (Exception e) {
e.printStackTrace();
}
} }
} }

101
app/src/main/java/com/github/catvod/net/OkHttp.java
View File

@ -1,20 +1,12 @@
package com.github.catvod.net; package com.github.catvod.net;
import android.net.Uri;
import com.github.catvod.crawler.Spider; import com.github.catvod.crawler.Spider;
import com.github.catvod.utils.Prefers;
import java.io.IOException; import java.io.IOException;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import okhttp3.Credentials;
import okhttp3.Dns; import okhttp3.Dns;
import okhttp3.Headers; import okhttp3.Headers;
import okhttp3.OkHttpClient; import okhttp3.OkHttpClient;
@ -26,7 +18,6 @@ public class OkHttp {
public static final String POST = "POST"; public static final String POST = "POST";
public static final String GET = "GET"; public static final String GET = "GET";
private OkHttpClient proxy;
private OkHttpClient client; private OkHttpClient client;
private static class Loader { private static class Loader {
@ -41,95 +32,49 @@ public class OkHttp {
return Spider.safeDns(); return Spider.safeDns();
} }
public void resetProxy() {
Authenticator.setDefault(null);
proxy = null;
}
public static OkHttpClient client() { public static OkHttpClient client() {
if (get().client != null) return get().client; if (get().client != null) return get().client;
return get().client = getBuilder().build(); return get().client = getBuilder().build();
} }
public static OkHttpClient proxy() { public static OkHttpClient noRedirect() {
if (get().proxy != null) return get().proxy; return client().newBuilder().followRedirects(false).followSslRedirects(false).build();
return get().proxy = getBuilder(Prefers.getString("proxy")).build();
}
public static OkHttpClient client(boolean proxy) {
return proxy ? proxy() : client();
}
public static OkHttpClient noRedirect(boolean proxy) {
return client(proxy).newBuilder().followRedirects(false).followSslRedirects(false).build();
} }
public static Response newCall(String url, Map<String, String> header) throws IOException { public static Response newCall(String url, Map<String, String> header) throws IOException {
return client(false).newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute(); return client().newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute();
} }
public static String string(String url) { public static String string(String url) {
return string(false, url); return string(url, null);
}
public static String string(boolean proxy, String url) {
return string(proxy, url, null);
} }
public static String string(String url, Map<String, String> header) { public static String string(String url, Map<String, String> header) {
return string(false, url, header); return string(url, null, header);
}
public static String string(boolean proxy, String url, Map<String, String> header) {
return string(proxy, url, null, header);
} }
public static String string(String url, Map<String, String> params, Map<String, String> header) { public static String string(String url, Map<String, String> params, Map<String, String> header) {
return string(false, url, params, header); return new OkRequest(GET, url, params, header).execute(client()).getBody();
}
public static String string(boolean proxy, String url, Map<String, String> params, Map<String, String> header) {
return string(client(proxy), url, params, header);
}
public static String string(OkHttpClient client, String url, Map<String, String> params, Map<String, String> header) {
return new OkRequest(GET, url, params, header).execute(client).getBody();
} }
public static String post(String url, Map<String, String> params) { public static String post(String url, Map<String, String> params) {
return post(false, url, params); return post(url, params, null).getBody();
}
public static String post(boolean proxy, String url, Map<String, String> params) {
return post(proxy, url, params, null).getBody();
} }
public static OkResult post(String url, Map<String, String> params, Map<String, String> header) { public static OkResult post(String url, Map<String, String> params, Map<String, String> header) {
return post(false, url, params, header); return new OkRequest(POST, url, params, header).execute(client());
}
public static OkResult post(boolean proxy, String url, Map<String, String> params, Map<String, String> header) {
return new OkRequest(POST, url, params, header).execute(client(proxy));
} }
public static String post(String url, String json) { public static String post(String url, String json) {
return post(false, url, json); return post(url, json, null).getBody();
}
public static String post(boolean proxy, String url, String json) {
return post(proxy, url, json, null).getBody();
} }
public static OkResult post(String url, String json, Map<String, String> header) { public static OkResult post(String url, String json, Map<String, String> header) {
return post(false, url, json, header); return new OkRequest(POST, url, json, header).execute(client());
}
public static OkResult post(boolean proxy, String url, String json, Map<String, String> header) {
return new OkRequest(POST, url, json, header).execute(client(proxy));
} }
public static String getLocation(String url, Map<String, String> header) throws IOException { public static String getLocation(String url, Map<String, String> header) throws IOException {
return getLocation(noRedirect(false).newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute().headers().toMultimap()); return getLocation(noRedirect().newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute().headers().toMultimap());
} }
public static String getLocation(Map<String, List<String>> headers) { public static String getLocation(Map<String, List<String>> headers) {
@ -142,28 +87,4 @@ public class OkHttp {
public static OkHttpClient.Builder getBuilder() { public static OkHttpClient.Builder getBuilder() {
return new OkHttpClient.Builder().dns(dns()).connectTimeout(30, TimeUnit.SECONDS).readTimeout(30, TimeUnit.SECONDS).writeTimeout(30, TimeUnit.SECONDS).hostnameVerifier(SSLCompat.VERIFIER).sslSocketFactory(new SSLCompat(), SSLCompat.TM); return new OkHttpClient.Builder().dns(dns()).connectTimeout(30, TimeUnit.SECONDS).readTimeout(30, TimeUnit.SECONDS).writeTimeout(30, TimeUnit.SECONDS).hostnameVerifier(SSLCompat.VERIFIER).sslSocketFactory(new SSLCompat(), SSLCompat.TM);
} }
public static OkHttpClient.Builder getBuilder(String proxy) {
Uri uri = Uri.parse(proxy);
String userInfo = uri.getUserInfo();
OkHttpClient.Builder builder = client().newBuilder();
if (userInfo != null && userInfo.contains(":")) setAuthenticator(builder, userInfo);
if (uri.getScheme() == null || uri.getHost() == null || uri.getPort() <= 0) return builder;
if (uri.getScheme().startsWith("http")) builder.proxy(new Proxy(Proxy.Type.HTTP, InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort())));
if (uri.getScheme().startsWith("socks")) builder.proxy(new Proxy(Proxy.Type.SOCKS, InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort())));
return builder;
}
private static void setAuthenticator(OkHttpClient.Builder builder, String userInfo) {
builder.proxyAuthenticator((route, response) -> {
String credential = Credentials.basic(userInfo.split(":")[0], userInfo.split(":")[1]);
return response.request().newBuilder().header("Proxy-Authorization", credential).build();
});
Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(userInfo.split(":")[0], userInfo.split(":")[1].toCharArray());
}
});
}
} }

17
app/src/main/java/com/github/catvod/spider/AppYsV2.java
View File

@ -42,7 +42,7 @@ public class AppYsV2 extends Spider {
JSONArray jsonArray = null; JSONArray jsonArray = null;
if (!url.isEmpty()) { if (!url.isEmpty()) {
SpiderDebug.log(url); SpiderDebug.log(url);
String json = OkHttp.string(proxy(), url, getHeaders(url)); String json = OkHttp.string(url, getHeaders(url));
JSONObject obj = new JSONObject(json); JSONObject obj = new JSONObject(json);
if (obj.has("list") && obj.get("list") instanceof JSONArray) { if (obj.has("list") && obj.get("list") instanceof JSONArray) {
jsonArray = obj.getJSONArray("list"); jsonArray = obj.getJSONArray("list");
@ -145,7 +145,7 @@ public class AppYsV2 extends Spider {
isTV = true; isTV = true;
} }
SpiderDebug.log(url); SpiderDebug.log(url);
String json = OkHttp.string(proxy(), url, getHeaders(url)); String json = OkHttp.string(url, getHeaders(url));
JSONObject obj = new JSONObject(json); JSONObject obj = new JSONObject(json);
JSONArray videos = new JSONArray(); JSONArray videos = new JSONArray();
if (isTV) { if (isTV) {
@ -197,7 +197,7 @@ public class AppYsV2 extends Spider {
url = url.replace("筛选year", (extend != null && extend.containsKey("year")) ? extend.get("year") : ""); url = url.replace("筛选year", (extend != null && extend.containsKey("year")) ? extend.get("year") : "");
url = url.replace("排序", (extend != null && extend.containsKey("排序")) ? extend.get("排序") : ""); url = url.replace("排序", (extend != null && extend.containsKey("排序")) ? extend.get("排序") : "");
SpiderDebug.log(url); SpiderDebug.log(url);
String json = OkHttp.string(proxy(), url, getHeaders(url)); String json = OkHttp.string(url, getHeaders(url));
JSONObject obj = new JSONObject(json); JSONObject obj = new JSONObject(json);
int totalPg = Integer.MAX_VALUE; int totalPg = Integer.MAX_VALUE;
try { try {
@ -257,7 +257,7 @@ public class AppYsV2 extends Spider {
String apiUrl = getApiUrl(); String apiUrl = getApiUrl();
String url = getPlayUrlPrefix(apiUrl) + ids.get(0); String url = getPlayUrlPrefix(apiUrl) + ids.get(0);
SpiderDebug.log(url); SpiderDebug.log(url);
String json = OkHttp.string(proxy(), url, getHeaders(url)); String json = OkHttp.string(url, getHeaders(url));
JSONObject obj = new JSONObject(json); JSONObject obj = new JSONObject(json);
JSONObject result = new JSONObject(); JSONObject result = new JSONObject();
JSONObject vod = new JSONObject(); JSONObject vod = new JSONObject();
@ -272,7 +272,7 @@ public class AppYsV2 extends Spider {
public String searchContent(String key, boolean quick) throws Exception { public String searchContent(String key, boolean quick) throws Exception {
String apiUrl = getApiUrl(); String apiUrl = getApiUrl();
String url = getSearchUrl(apiUrl, URLEncoder.encode(key)); String url = getSearchUrl(apiUrl, URLEncoder.encode(key));
String json = OkHttp.string(proxy(), url, getHeaders(url)); String json = OkHttp.string(url, getHeaders(url));
JSONObject obj = new JSONObject(json); JSONObject obj = new JSONObject(json);
JSONArray jsonArray = null; JSONArray jsonArray = null;
JSONArray videos = new JSONArray(); JSONArray videos = new JSONArray();
@ -694,7 +694,7 @@ public class AppYsV2 extends Spider {
for (String parseUrl : parseUrls) { for (String parseUrl : parseUrls) {
if (parseUrl.isEmpty() || parseUrl.equals("null")) continue; if (parseUrl.isEmpty() || parseUrl.equals("null")) continue;
String playUrl = parseUrl + url; String playUrl = parseUrl + url;
String content = OkHttp.string(proxy(), playUrl); String content = OkHttp.string(playUrl);
JSONObject tryJson = null; JSONObject tryJson = null;
try { try {
tryJson = jsonParse(url, content); tryJson = jsonParse(url, content);
@ -738,11 +738,6 @@ public class AppYsV2 extends Spider {
return Utils.isVideoFormat(url); return Utils.isVideoFormat(url);
} }
@Override
public void destroy() {
OkHttp.get().resetProxy();
}
private String getApiUrl() { private String getApiUrl() {
if (extInfos == null || extInfos.length < 1) return ""; if (extInfos == null || extInfos.length < 1) return "";
return extInfos[0].trim(); return extInfos[0].trim();

5
app/src/main/java/com/github/catvod/spider/Eighteen.java
View File

@ -94,11 +94,6 @@ public class Eighteen extends Spider {
return Result.get().url(result.get("url")).string(); return Result.get().url(result.get("url")).string();
} }
@Override
public void destroy() {
OkHttp.get().resetProxy();
}
private String searchContent(String key, String pg) { private String searchContent(String key, String pg) {
HashMap<String, String> params = new HashMap<>(); HashMap<String, String> params = new HashMap<>();
params.put("search_keyword", key); params.put("search_keyword", key);

5
app/src/main/java/com/github/catvod/spider/Hanime.java
View File

@ -135,9 +135,4 @@ public class Hanime extends Spider {
public String playerContent(String flag, String id, List<String> vipFlags) throws Exception { public String playerContent(String flag, String id, List<String> vipFlags) throws Exception {
return Result.get().url(id).header(getHeaders()).string(); return Result.get().url(id).header(getHeaders()).string();
} }
@Override
public void destroy() {
OkHttp.get().resetProxy();
}
} }

5
app/src/main/java/com/github/catvod/spider/Jable.java
View File

@ -101,9 +101,4 @@ public class Jable extends Spider {
public String playerContent(String flag, String id, List<String> vipFlags) throws Exception { public String playerContent(String flag, String id, List<String> vipFlags) throws Exception {
return Result.get().url(id).header(getHeaders()).string(); return Result.get().url(id).header(getHeaders()).string();
} }
@Override
public void destroy() {
OkHttp.get().resetProxy();
}
} }

5
app/src/main/java/com/github/catvod/spider/Miss.java
View File

@ -97,11 +97,6 @@ public class Miss extends Spider {
return Result.get().parse().url(url + id).string(); return Result.get().parse().url(url + id).string();
} }
@Override
public void destroy() {
OkHttp.get().resetProxy();
}
private String searchContent(String key, String pg) { private String searchContent(String key, String pg) {
List<Vod> list = new ArrayList<>(); List<Vod> list = new ArrayList<>();
Document doc = Jsoup.parse(OkHttp.string(proxy(), url + "search/" + key + "?page=" + pg)); Document doc = Jsoup.parse(OkHttp.string(proxy(), url + "search/" + key + "?page=" + pg));

5
app/src/main/java/com/github/catvod/spider/Star.java
View File

@ -136,10 +136,5 @@ public class Star extends Spider {
public String playerContent(String flag, String id, List<String> vipFlags) throws Exception { public String playerContent(String flag, String id, List<String> vipFlags) throws Exception {
return Result.get().url(id).string(); return Result.get().url(id).string();
} }
@Override
public void destroy() {
OkHttp.get().resetProxy();
}
} }

9
app/src/main/java/com/github/catvod/spider/XPath.java
View File

@ -317,11 +317,6 @@ public class XPath extends Spider {
return Utils.isVideoFormat(url); return Utils.isVideoFormat(url);
} }
@Override
public void destroy() {
OkHttp.get().resetProxy();
}
protected String ext = null; protected String ext = null;
protected Rule rule = null; protected Rule rule = null;
@ -329,7 +324,7 @@ public class XPath extends Spider {
if (rule == null) { if (rule == null) {
if (ext != null) { if (ext != null) {
if (ext.startsWith("http")) { if (ext.startsWith("http")) {
String json = OkHttp.string(proxy(), ext, null); String json = OkHttp.string(ext, null);
rule = Rule.fromJson(json); rule = Rule.fromJson(json);
loadRuleExt(json); loadRuleExt(json);
} else { } else {
@ -345,6 +340,6 @@ public class XPath extends Spider {
protected String fetch(String webUrl) { protected String fetch(String webUrl) {
SpiderDebug.log(webUrl); SpiderDebug.log(webUrl);
return OkHttp.string(proxy(), webUrl, getHeaders()); return OkHttp.string(webUrl, getHeaders());
} }
} }