From 2a6cbc0c7c7afbe50709858715186aab4812cd1a Mon Sep 17 00:00:00 2001 From: FongMi Date: Sun, 24 Sep 2023 01:52:02 +0800 Subject: [PATCH] Clean code --- .../java/com/github/catvod/api/AliYun.java | 8 +- .../com/github/catvod/crawler/Spider.java | 22 +++ .../java/com/github/catvod/net/Cronet.java | 43 ++--- .../java/com/github/catvod/net/OkHttp.java | 172 +++++++++--------- .../java/com/github/catvod/net/OkRequest.java | 15 +- .../java/com/github/catvod/net/OkResult.java | 13 +- .../java/com/github/catvod/spider/AList.java | 4 +- .../com/github/catvod/spider/AppYsV2.java | 18 +- .../com/github/catvod/spider/Eighteen.java | 8 +- .../java/com/github/catvod/spider/Hanime.java | 10 +- .../java/com/github/catvod/spider/Jable.java | 10 +- .../java/com/github/catvod/spider/Miss.java | 8 +- .../java/com/github/catvod/spider/PanSou.java | 4 +- .../java/com/github/catvod/spider/Star.java | 12 +- .../java/com/github/catvod/spider/XPath.java | 4 +- .../com/github/catvod/spider/XiaoZhiTiao.java | 14 +- .../java/com/github/catvod/spider/Zhaozy.java | 3 +- 17 files changed, 186 insertions(+), 182 deletions(-) diff --git a/app/src/main/java/com/github/catvod/api/AliYun.java b/app/src/main/java/com/github/catvod/api/AliYun.java index ed019186..10903202 100644 --- a/app/src/main/java/com/github/catvod/api/AliYun.java +++ b/app/src/main/java/com/github/catvod/api/AliYun.java @@ -125,7 +125,7 @@ public class AliYun { private boolean alist(String url, JsonObject param) { String api = "https://aliapi.ewwe.gq/alist/ali_open/" + url; - OkResult result = OkHttp.postJson(api, param.toString(), getHeader()); + OkResult result = OkHttp.post(api, param.toString(), getHeader()); SpiderDebug.log(result.getCode() + "," + api + "," + result.getBody()); if (isManyRequest(result.getBody())) return false; oauth = OAuth.objectFrom(result.getBody()).save(); @@ -134,14 +134,14 @@ public class AliYun { private String post(String url, JsonObject param) { url = url.startsWith("https") ? url : "https://api.aliyundrive.com/" + url; - OkResult result = OkHttp.postJson(url, param.toString(), getHeader()); + OkResult result = OkHttp.post(url, param.toString(), getHeader()); SpiderDebug.log(result.getCode() + "," + url + "," + result.getBody()); return result.getBody(); } private String auth(String url, String json, boolean retry) { url = url.startsWith("https") ? url : "https://api.aliyundrive.com/" + url; - OkResult result = OkHttp.postJson(url, json, getHeaderAuth()); + OkResult result = OkHttp.post(url, json, getHeaderAuth()); SpiderDebug.log(result.getCode() + "," + url + "," + result.getBody()); if (retry && result.getCode() == 401 && refreshAccessToken()) return auth(url, json, false); if (retry && result.getCode() == 429) return auth(url, json, false); @@ -150,7 +150,7 @@ public class AliYun { private String oauth(String url, String json, boolean retry) { url = url.startsWith("https") ? url : "https://open.aliyundrive.com/adrive/v1.0/" + url; - OkResult result = OkHttp.postJson(url, json, getHeaderOpen()); + OkResult result = OkHttp.post(url, json, getHeaderOpen()); SpiderDebug.log(result.getCode() + "," + url + "," + result.getBody()); if (retry && (result.getCode() == 400 || result.getCode() == 401) && refreshOpenToken()) return oauth(url, json, false); return result.getBody(); diff --git a/app/src/main/java/com/github/catvod/crawler/Spider.java b/app/src/main/java/com/github/catvod/crawler/Spider.java index 5105fb19..833619bd 100644 --- a/app/src/main/java/com/github/catvod/crawler/Spider.java +++ b/app/src/main/java/com/github/catvod/crawler/Spider.java @@ -4,9 +4,19 @@ import android.content.Context; import java.util.HashMap; import java.util.List; +import java.util.Map; + +import okhttp3.Dns; public abstract class Spider { + private boolean proxy; + + public Spider proxy(boolean proxy) { + this.proxy = proxy; + return this; + } + public void init(Context context) throws Exception { } @@ -50,6 +60,18 @@ public abstract class Spider { return false; } + public boolean proxy() { + return proxy; + } + + public Object[] proxyLocal(Map params) throws Exception { + return null; + } + public void destroy() { } + + public static Dns safeDns() { + return Dns.SYSTEM; + } } diff --git a/app/src/main/java/com/github/catvod/net/Cronet.java b/app/src/main/java/com/github/catvod/net/Cronet.java index bf03e80a..d169695a 100644 --- a/app/src/main/java/com/github/catvod/net/Cronet.java +++ b/app/src/main/java/com/github/catvod/net/Cronet.java @@ -5,10 +5,12 @@ import com.google.net.cronet.okhttptransport.CronetInterceptor; import org.chromium.net.CronetEngine; -import java.util.List; +import java.io.IOException; import java.util.Map; +import okhttp3.Headers; import okhttp3.OkHttpClient; +import okhttp3.Request; public class Cronet { @@ -53,36 +55,20 @@ public class Cronet { return get().noRedirect; } - public static void stringNoRedirect(String url, Map header, Map> respHeader) { - string(noRedirect(), GET, url, null, null, header, respHeader); - } - - public static String string(OkHttpClient client, String method, String url, String tag, Map params, Map header, Map> respHeader) { - return new OkRequest(method, url, params, header, respHeader).tag(tag).execute(client).getBody(); - } - public static String string(String url) { return string(url, null); } public static String string(String url, Map header) { - return string(url, header, null); + return string(url, null, header); } - public static String string(String url, Map header, Map> respHeader) { - return string(url, null, header, respHeader); + public static String string(String url, Map params, Map header) { + return string(client(), GET, url, params, header); } - public static String string(String url, Map params, Map header, Map> respHeader) { - return string(url, null, params, header, respHeader); - } - - public static String string(String url, String tag, Map header) { - return string(url, tag, null, header, null); - } - - public static String string(String url, String tag, Map params, Map header, Map> respHeader) { - return string(client(), GET, url, tag, params, header, respHeader); + public static String string(OkHttpClient client, String method, String url, Map params, Map header) { + return new OkRequest(method, url, params, header).execute(client).getBody(); } public static String post(String url, Map params) { @@ -90,11 +76,7 @@ public class Cronet { } public static String post(String url, Map params, Map header) { - return post(url, params, header, null); - } - - public static String post(String url, Map params, Map header, Map> respHeader) { - return string(client(), POST, url, null, params, header, respHeader); + return string(client(), POST, url, params, header); } public static OkResult postJson(String url, String json) { @@ -105,10 +87,7 @@ public class Cronet { return new OkRequest(POST, url, json, header).execute(client()); } - public static String getRedirectLocation(Map> headers) { - if (headers == null) return null; - if (headers.containsKey("location")) return headers.get("location").get(0); - if (headers.containsKey("Location")) return headers.get("Location").get(0); - return null; + public static String getLocation(String url, Map header) throws IOException { + return OkHttp.getLocation(noRedirect().newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute().headers().toMultimap()); } } diff --git a/app/src/main/java/com/github/catvod/net/OkHttp.java b/app/src/main/java/com/github/catvod/net/OkHttp.java index f52ded98..a3294144 100644 --- a/app/src/main/java/com/github/catvod/net/OkHttp.java +++ b/app/src/main/java/com/github/catvod/net/OkHttp.java @@ -3,15 +3,17 @@ package com.github.catvod.net; import android.net.Uri; import com.github.catvod.crawler.Spider; +import com.github.catvod.utils.Prefers; import java.io.IOException; +import java.net.Authenticator; import java.net.InetSocketAddress; +import java.net.PasswordAuthentication; import java.net.Proxy; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import okhttp3.Call; import okhttp3.Credentials; import okhttp3.Dns; import okhttp3.Headers; @@ -24,8 +26,8 @@ public class OkHttp { public static final String POST = "POST"; public static final String GET = "GET"; - private final OkHttpClient noRedirect; - private final OkHttpClient client; + private OkHttpClient proxy; + private OkHttpClient client; private static class Loader { static volatile OkHttp INSTANCE = new OkHttp(); @@ -35,139 +37,129 @@ public class OkHttp { return Loader.INSTANCE; } - public OkHttp() { - client = getBuilder().build(); - noRedirect = client.newBuilder().followRedirects(false).followSslRedirects(false).build(); + public static Dns dns() { + return Spider.safeDns(); } - public static OkHttpClient.Builder getBuilder() { - return getBuilder(proxy()); - } - - public static OkHttpClient.Builder getBuilder(Uri proxy) { - OkHttpClient.Builder builder = new OkHttpClient.Builder().dns(safeDns()).readTimeout(30, TimeUnit.SECONDS).writeTimeout(30, TimeUnit.SECONDS).connectTimeout(30, TimeUnit.SECONDS).hostnameVerifier(SSLCompat.VERIFIER).sslSocketFactory(new SSLCompat(), SSLCompat.TM); - if (proxy != null && proxy.getScheme() != null && proxy.getHost() != null && proxy.getPort() > 0) setProxy(builder, proxy); - return builder; + public void resetProxy() { + Authenticator.setDefault(null); + proxy = null; } public static OkHttpClient client() { - return get().client; + if (get().client != null) return get().client; + return get().client = getBuilder().build(); } - private static OkHttpClient noRedirect() { - return get().noRedirect; + public static OkHttpClient proxy() { + if (get().proxy != null) return get().proxy; + return get().proxy = getBuilder(Prefers.getString("proxy")).build(); } - public static Dns safeDns() { - try { - return (Dns) Spider.class.getMethod("safeDns").invoke(null); - } catch (Exception e) { - return Dns.SYSTEM; - } + public static OkHttpClient client(boolean proxy) { + return proxy ? proxy() : client(); } - public static Uri proxy() { - try { - return (Uri) Spider.class.getMethod("proxy").invoke(null); - } catch (Exception e) { - return null; - } - } - - public static Response newCall(String url) throws IOException { - return client().newCall(new Request.Builder().url(url).build()).execute(); + public static OkHttpClient noRedirect(boolean proxy) { + return client(proxy).newBuilder().followRedirects(false).followSslRedirects(false).build(); } public static Response newCall(String url, Map header) throws IOException { - return client().newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute(); - } - - public static void stringNoRedirect(String url, Map header, Map> respHeader) { - string(noRedirect(), GET, url, null, null, header, respHeader); - } - - public static String string(OkHttpClient client, String method, String url, String tag, Map params, Map header, Map> respHeader) { - return new OkRequest(method, url, params, header, respHeader).tag(tag).execute(client).getBody(); + return client(false).newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute(); } public static String string(String url) { - return string(url, null); + return string(false, url); + } + + public static String string(boolean proxy, String url) { + return string(proxy, url, null); } public static String string(String url, Map header) { - return string(url, header, null); + return string(false, url, header); } - public static String string(String url, Map header, Map> respHeader) { - return string(url, null, header, respHeader); + public static String string(boolean proxy, String url, Map header) { + return string(proxy, url, null, header); } - public static String get(String url, Map params, Map header) { - return string(url, params, header, null); + public static String string(boolean proxy, String url, Map params, Map header) { + return string(client(proxy), url, params, header); } - public static String string(String url, Map params, Map header, Map> respHeader) { - return string(url, null, params, header, respHeader); - } - - public static String string(String url, String tag, Map header) { - return string(url, tag, null, header, null); - } - - public static String string(String url, String tag, Map params, Map header, Map> respHeader) { - return string(client(), GET, url, tag, params, header, respHeader); + public static String string(OkHttpClient client, String url, Map params, Map header) { + return new OkRequest(GET, url, params, header).execute(client).getBody(); } public static String post(String url, Map params) { - return post(url, params, null); + return post(false, url, params); } - public static String post(String url, Map params, Map header) { - return post(url, params, header, null); + public static String post(boolean proxy, String url, Map params) { + return post(proxy, url, params, null).getBody(); } - public static String post(String url, Map params, Map header, Map> respHeader) { - return string(client(), POST, url, null, params, header, respHeader); + public static String post(String url, String json) { + return post(false, url, json); } - public static OkResult postJson(String url, String json) { - return postJson(url, json, null); + public static String post(boolean proxy, String url, String json) { + return post(proxy, url, json, null).getBody(); } - public static OkResult postJson(String url, String json, Map header) { - return new OkRequest(POST, url, json, header).execute(client()); + public static OkResult post(String url, String json, Map header) { + return post(false, url, json, header); } - public static void cancel(Object tag) { - for (Call call : client().dispatcher().queuedCalls()) { - if (tag.equals(call.request().tag())) { - call.cancel(); - } - } - for (Call call : client().dispatcher().runningCalls()) { - if (tag.equals(call.request().tag())) { - call.cancel(); - } - } + public static OkResult post(boolean proxy, String url, String json, Map header) { + return new OkRequest(POST, url, json, header).execute(client(proxy)); } - public static String getRedirectLocation(Map> headers) { + public static OkResult post(String url, Map params, Map header) { + return post(false, url, params, header); + } + + public static OkResult post(boolean proxy, String url, Map params, Map header) { + return new OkRequest(POST, url, params, header).execute(client(proxy)); + } + + public static String getLocation(String url, Map header) throws IOException { + return getLocation(noRedirect(false).newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute().headers().toMultimap()); + } + + public static String getLocation(Map> headers) { if (headers == null) return null; if (headers.containsKey("location")) return headers.get("location").get(0); if (headers.containsKey("Location")) return headers.get("Location").get(0); return null; } - private static void setProxy(OkHttpClient.Builder builder, Uri proxy) { - String userInfo = proxy.getUserInfo(); - if (proxy.getScheme() == null || proxy.getScheme().startsWith("socks")) { - builder.proxy(new Proxy(Proxy.Type.SOCKS, InetSocketAddress.createUnresolved(proxy.getHost(), proxy.getPort()))); - } else if (proxy.getScheme().startsWith("http")) { - builder.proxy(new Proxy(Proxy.Type.HTTP, InetSocketAddress.createUnresolved(proxy.getHost(), proxy.getPort()))); - if (userInfo != null && userInfo.contains(":")) builder.proxyAuthenticator((route, response) -> { - String credential = Credentials.basic(userInfo.split(":")[0], userInfo.split(":")[1]); - return response.request().newBuilder().header("Proxy-Authorization", credential).build(); - }); - } + public static OkHttpClient.Builder getBuilder() { + return new OkHttpClient.Builder().dns(dns()).connectTimeout(30, TimeUnit.SECONDS).readTimeout(30, TimeUnit.SECONDS).writeTimeout(30, TimeUnit.SECONDS).hostnameVerifier(SSLCompat.VERIFIER).sslSocketFactory(new SSLCompat(), SSLCompat.TM); + } + + private static OkHttpClient.Builder getBuilder(String proxy) { + Uri uri = Uri.parse(proxy); + String userInfo = uri.getUserInfo(); + OkHttpClient.Builder builder = client().newBuilder(); + if (userInfo != null && userInfo.contains(":")) setAuthenticator(builder, userInfo); + if (uri.getScheme() == null || uri.getHost() == null || uri.getPort() <= 0) return builder; + if (uri.getScheme().startsWith("http")) builder.proxy(new Proxy(Proxy.Type.HTTP, InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort()))); + if (uri.getScheme().startsWith("socks")) builder.proxy(new Proxy(Proxy.Type.SOCKS, InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort()))); + return builder; + } + + private static void setAuthenticator(OkHttpClient.Builder builder, String userInfo) { + builder.proxyAuthenticator((route, response) -> { + String credential = Credentials.basic(userInfo.split(":")[0], userInfo.split(":")[1]); + return response.request().newBuilder().header("Proxy-Authorization", credential).build(); + }); + Authenticator.setDefault(new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(userInfo.split(":")[0], userInfo.split(":")[1].toCharArray()); + } + }); } } diff --git a/app/src/main/java/com/github/catvod/net/OkRequest.java b/app/src/main/java/com/github/catvod/net/OkRequest.java index 4e080443..485010c1 100644 --- a/app/src/main/java/com/github/catvod/net/OkRequest.java +++ b/app/src/main/java/com/github/catvod/net/OkRequest.java @@ -5,7 +5,6 @@ import android.text.TextUtils; import com.github.catvod.utils.Utils; import java.io.IOException; -import java.util.List; import java.util.Map; import okhttp3.FormBody; @@ -17,7 +16,6 @@ import okhttp3.Response; class OkRequest { - private final Map> respHeader; private final Map header; private final Map params; private final String method; @@ -26,21 +24,20 @@ class OkRequest { private String url; private Object tag; - OkRequest(String method, String url, Map params, Map header, Map> respHeader) { - this(method, url, null, params, header, respHeader); + OkRequest(String method, String url, Map params, Map header) { + this(method, url, null, params, header); } OkRequest(String method, String url, String json, Map header) { - this(method, url, json, null, header, null); + this(method, url, json, null, header); } - private OkRequest(String method, String url, String json, Map params, Map header, Map> respHeader) { + private OkRequest(String method, String url, String json, Map params, Map header) { this.url = url; this.json = json; this.method = method; this.params = params; this.header = header; - this.respHeader = respHeader; getInstance(); } @@ -74,9 +71,7 @@ class OkRequest { public OkResult execute(OkHttpClient client) { try { Response response = client.newCall(request).execute(); - if (respHeader != null) respHeader.clear(); - if (respHeader != null) respHeader.putAll(response.headers().toMultimap()); - return new OkResult(response.code(), response.body().string()); + return new OkResult(response.code(), response.body().string(), response.headers().toMultimap()); } catch (IOException e) { return new OkResult(); } diff --git a/app/src/main/java/com/github/catvod/net/OkResult.java b/app/src/main/java/com/github/catvod/net/OkResult.java index 8e26a5af..06d3f03c 100644 --- a/app/src/main/java/com/github/catvod/net/OkResult.java +++ b/app/src/main/java/com/github/catvod/net/OkResult.java @@ -2,19 +2,26 @@ package com.github.catvod.net; import android.text.TextUtils; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + public class OkResult { private final int code; private final String body; + private final Map> resp; public OkResult() { this.code = 500; this.body = ""; + this.resp = new HashMap<>(); } - public OkResult(int code, String body) { + public OkResult(int code, String body, Map> resp) { this.code = code; this.body = body; + this.resp = resp; } public int getCode() { @@ -24,4 +31,8 @@ public class OkResult { public String getBody() { return TextUtils.isEmpty(body) ? "" : body; } + + public Map> getResp() { + return resp; + } } diff --git a/app/src/main/java/com/github/catvod/spider/AList.java b/app/src/main/java/com/github/catvod/spider/AList.java index 1ef80a97..b9abb826 100644 --- a/app/src/main/java/com/github/catvod/spider/AList.java +++ b/app/src/main/java/com/github/catvod/spider/AList.java @@ -61,7 +61,7 @@ public class AList extends Spider { } private String post(Drive drive, String url, String param, boolean retry) { - String response = OkHttp.postJson(url, param, drive.getHeader()).getBody(); + String response = OkHttp.post(url, param, drive.getHeader()).getBody(); SpiderDebug.log(response); if (retry && response.contains("Guest user is disabled") && login(drive)) return post(drive, url, param, false); return response; @@ -152,7 +152,7 @@ public class AList extends Spider { JSONObject params = new JSONObject(); params.put("username", drive.getLogin().getUsername()); params.put("password", drive.getLogin().getPassword()); - String response = OkHttp.postJson(drive.loginApi(), params.toString()).getBody(); + String response = OkHttp.post(drive.loginApi(), params.toString()); drive.setToken(new JSONObject(response).getJSONObject("data").getString("token")); return true; } catch (Exception e) { diff --git a/app/src/main/java/com/github/catvod/spider/AppYsV2.java b/app/src/main/java/com/github/catvod/spider/AppYsV2.java index 1ccf12f4..1c3eca65 100644 --- a/app/src/main/java/com/github/catvod/spider/AppYsV2.java +++ b/app/src/main/java/com/github/catvod/spider/AppYsV2.java @@ -13,7 +13,11 @@ import org.json.JSONException; import org.json.JSONObject; import java.net.URLEncoder; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -38,7 +42,7 @@ public class AppYsV2 extends Spider { JSONArray jsonArray = null; if (!url.isEmpty()) { SpiderDebug.log(url); - String json = OkHttp.string(url, getHeaders(url)); + String json = OkHttp.string(proxy(), url, getHeaders(url)); JSONObject obj = new JSONObject(json); if (obj.has("list") && obj.get("list") instanceof JSONArray) { jsonArray = obj.getJSONArray("list"); @@ -141,7 +145,7 @@ public class AppYsV2 extends Spider { isTV = true; } SpiderDebug.log(url); - String json = OkHttp.string(url, getHeaders(url)); + String json = OkHttp.string(proxy(), url, getHeaders(url)); JSONObject obj = new JSONObject(json); JSONArray videos = new JSONArray(); if (isTV) { @@ -193,7 +197,7 @@ public class AppYsV2 extends Spider { url = url.replace("筛选year", (extend != null && extend.containsKey("year")) ? extend.get("year") : ""); url = url.replace("排序", (extend != null && extend.containsKey("排序")) ? extend.get("排序") : ""); SpiderDebug.log(url); - String json = OkHttp.string(url, getHeaders(url)); + String json = OkHttp.string(proxy(), url, getHeaders(url)); JSONObject obj = new JSONObject(json); int totalPg = Integer.MAX_VALUE; try { @@ -253,7 +257,7 @@ public class AppYsV2 extends Spider { String apiUrl = getApiUrl(); String url = getPlayUrlPrefix(apiUrl) + ids.get(0); SpiderDebug.log(url); - String json = OkHttp.string(url, getHeaders(url)); + String json = OkHttp.string(proxy(), url, getHeaders(url)); JSONObject obj = new JSONObject(json); JSONObject result = new JSONObject(); JSONObject vod = new JSONObject(); @@ -268,7 +272,7 @@ public class AppYsV2 extends Spider { public String searchContent(String key, boolean quick) throws Exception { String apiUrl = getApiUrl(); String url = getSearchUrl(apiUrl, URLEncoder.encode(key)); - String json = OkHttp.string(url, getHeaders(url)); + String json = OkHttp.string(proxy(), url, getHeaders(url)); JSONObject obj = new JSONObject(json); JSONArray jsonArray = null; JSONArray videos = new JSONArray(); @@ -690,7 +694,7 @@ public class AppYsV2 extends Spider { for (String parseUrl : parseUrls) { if (parseUrl.isEmpty() || parseUrl.equals("null")) continue; String playUrl = parseUrl + url; - String content = OkHttp.string(playUrl); + String content = OkHttp.string(proxy(), playUrl); JSONObject tryJson = null; try { tryJson = jsonParse(url, content); diff --git a/app/src/main/java/com/github/catvod/spider/Eighteen.java b/app/src/main/java/com/github/catvod/spider/Eighteen.java index 45645770..d62605c5 100644 --- a/app/src/main/java/com/github/catvod/spider/Eighteen.java +++ b/app/src/main/java/com/github/catvod/spider/Eighteen.java @@ -27,7 +27,7 @@ public class Eighteen extends Spider { public String homeContent(boolean filter) throws Exception { List classes = new ArrayList<>(); List list = new ArrayList<>(); - Document doc = Jsoup.parse(OkHttp.string(url)); + Document doc = Jsoup.parse(OkHttp.string(proxy(), url)); for (Element a : doc.select("ul.animenu__nav > li > a")) { String typeName = a.text(); String typeId = a.attr("href").replace(url, ""); @@ -50,7 +50,7 @@ public class Eighteen extends Spider { List list = new ArrayList<>(); tid = tid.replace("random", "list"); tid = tid.replace("index", pg); - Document doc = Jsoup.parse(OkHttp.string(url + tid)); + Document doc = Jsoup.parse(OkHttp.string(proxy(), url + tid)); for (Element div : doc.select("div.post")) { String id = div.select("a").attr("href").replace(url, ""); String name = div.select("h3").text(); @@ -63,7 +63,7 @@ public class Eighteen extends Spider { @Override public String detailContent(List ids) throws Exception { - Document doc = Jsoup.parse(OkHttp.string(url + ids.get(0))); + Document doc = Jsoup.parse(OkHttp.string(proxy(), url + ids.get(0))); Element wrap = doc.select("div.video-wrap").get(0); String name = wrap.select("div.archive-title > h1").text(); String pic = wrap.select("div.player-wrap > img").attr("src"); @@ -99,7 +99,7 @@ public class Eighteen extends Spider { params.put("search_keyword", key); params.put("search_type", "fc"); params.put("op", "search"); - String res = OkHttp.post(url + "searchform_search/all/" + pg + ".html", params); + String res = OkHttp.post(proxy(), url + "searchform_search/all/" + pg + ".html", params); List list = new ArrayList<>(); for (Element div : Jsoup.parse(res).select("div.post")) { String id = div.select("a").attr("href").replace(url, ""); diff --git a/app/src/main/java/com/github/catvod/spider/Hanime.java b/app/src/main/java/com/github/catvod/spider/Hanime.java index 13665868..d1a740a4 100644 --- a/app/src/main/java/com/github/catvod/spider/Hanime.java +++ b/app/src/main/java/com/github/catvod/spider/Hanime.java @@ -43,10 +43,10 @@ public class Hanime extends Spider { List list = new ArrayList<>(); List classes = new ArrayList<>(); LinkedHashMap> filters = new LinkedHashMap<>(); - Document doc1 = Jsoup.parse(OkHttp.string(siteUrl.concat("/search?genre=裏番"), getHeaders())); + Document doc1 = Jsoup.parse(OkHttp.string(proxy(), siteUrl.concat("/search?genre=裏番"), getHeaders())); List sorts = doc1.select("div.hentai-sort-options-wrapper").eachText(); List years = doc1.getElementById("year").select("option").eachAttr("value"); - Document doc2 = Jsoup.parse(OkHttp.string(siteUrl, getHeaders())); + Document doc2 = Jsoup.parse(OkHttp.string(proxy(), siteUrl, getHeaders())); for (Element element : doc2.select("a.nav-item")) { String text = element.text(); if (text.equals("新番預告") || text.equals("H漫畫")) continue; @@ -75,7 +75,7 @@ public class Hanime extends Spider { if (extend.get("by") == null) extend.put("by", "最新上市"); if (extend.get("year") == null) extend.put("year", ""); String target = siteUrl.concat("/search?genre=").concat(tid).concat("&page=").concat(pg).concat("&sort=").concat(extend.get("by")).concat("&year=").concat(extend.get("year")); - Document doc = Jsoup.parse(OkHttp.string(target, getHeaders())); + Document doc = Jsoup.parse(OkHttp.string(proxy(), target, getHeaders())); for (Element element : doc.select("div.col-xs-6")) { String pic = element.select("img").get(1).attr("src"); String url = element.select("a.overlay").attr("href"); @@ -100,7 +100,7 @@ public class Hanime extends Spider { @Override public String detailContent(List ids) throws Exception { - Document doc = Jsoup.parse(OkHttp.string(siteUrl.concat("/watch?v=").concat(ids.get(0)), getHeaders())); + Document doc = Jsoup.parse(OkHttp.string(proxy(), siteUrl.concat("/watch?v=").concat(ids.get(0)), getHeaders())); String name = doc.getElementById("shareBtn-title").text(); JSONObject json = new JSONObject(doc.select("script[type=application/ld+json]").html().trim()); String content = json.optString("description"); @@ -120,7 +120,7 @@ public class Hanime extends Spider { public String searchContent(String key, boolean quick) throws Exception { List list = new ArrayList<>(); String target = siteUrl.concat("/search?query=").concat(key).concat("&genre=&sort=&year=&month=&duration="); - Document doc = Jsoup.parse(OkHttp.string(target, getHeaders())); + Document doc = Jsoup.parse(OkHttp.string(proxy(), target, getHeaders())); for (Element element : doc.select("div.col-xs-6")) { String pic = element.select("img").get(1).attr("src"); String url = element.select("a.overlay").attr("href"); diff --git a/app/src/main/java/com/github/catvod/spider/Jable.java b/app/src/main/java/com/github/catvod/spider/Jable.java index f26cd73f..68b33a92 100644 --- a/app/src/main/java/com/github/catvod/spider/Jable.java +++ b/app/src/main/java/com/github/catvod/spider/Jable.java @@ -34,13 +34,13 @@ public class Jable extends Spider { public String homeContent(boolean filter) throws Exception { List list = new ArrayList<>(); List classes = new ArrayList<>(); - Document doc = Jsoup.parse(OkHttp.string(cateUrl, getHeaders())); + Document doc = Jsoup.parse(OkHttp.string(proxy(), cateUrl, getHeaders())); for (Element element : doc.select("div.img-box > a")) { String typeId = element.attr("href").split("/")[4]; String typeName = element.select("div.absolute-center > h4").text(); classes.add(new Class(typeId, typeName)); } - doc = Jsoup.parse(OkHttp.string(siteUrl, getHeaders())); + doc = Jsoup.parse(OkHttp.string(proxy(), siteUrl, getHeaders())); for (Element element : doc.select("div.video-img-box")) { String pic = element.select("img").attr("data-src"); String url = element.select("a").attr("href"); @@ -56,7 +56,7 @@ public class Jable extends Spider { public String categoryContent(String tid, String pg, boolean filter, HashMap extend) throws Exception { List list = new ArrayList<>(); String target = cateUrl + tid + "/?mode=async&function=get_block&block_id=list_videos_common_videos_list&sort_by=post_date&from=" + String.format(Locale.getDefault(), "%02d", Integer.parseInt(pg)) + "&_=" + System.currentTimeMillis(); - Document doc = Jsoup.parse(OkHttp.string(target, getHeaders())); + Document doc = Jsoup.parse(OkHttp.string(proxy(), target, getHeaders())); for (Element element : doc.select("div.video-img-box")) { String pic = element.select("img").attr("data-src"); String url = element.select("a").attr("href"); @@ -69,7 +69,7 @@ public class Jable extends Spider { @Override public String detailContent(List ids) throws Exception { - Document doc = Jsoup.parse(OkHttp.string(detailUrl.concat(ids.get(0)).concat("/"), getHeaders())); + Document doc = Jsoup.parse(OkHttp.string(proxy(), detailUrl.concat(ids.get(0)).concat("/"), getHeaders())); String name = doc.select("meta[property=og:title]").attr("content"); String pic = doc.select("meta[property=og:image]").attr("content"); String year = doc.select("span.inactive-color").get(0).text(); @@ -86,7 +86,7 @@ public class Jable extends Spider { @Override public String searchContent(String key, boolean quick) throws Exception { List list = new ArrayList<>(); - Document doc = Jsoup.parse(OkHttp.string(searchUrl.concat(URLEncoder.encode(key)).concat("/"), getHeaders())); + Document doc = Jsoup.parse(OkHttp.string(proxy(), searchUrl.concat(URLEncoder.encode(key)).concat("/"), getHeaders())); for (Element element : doc.select("div.video-img-box")) { String pic = element.select("img").attr("data-src"); String url = element.select("a").attr("href"); diff --git a/app/src/main/java/com/github/catvod/spider/Miss.java b/app/src/main/java/com/github/catvod/spider/Miss.java index e63a8f85..87c2087c 100644 --- a/app/src/main/java/com/github/catvod/spider/Miss.java +++ b/app/src/main/java/com/github/catvod/spider/Miss.java @@ -28,7 +28,7 @@ public class Miss extends Spider { List list = new ArrayList<>(); List classes = new ArrayList<>(); LinkedHashMap> filters = new LinkedHashMap<>(); - Document doc = Jsoup.parse(OkHttp.string(url)); + Document doc = Jsoup.parse(OkHttp.string(proxy(), url)); List typeIds = Arrays.asList("chinese-subtitle", "new", "release", "uncensored-leak", "genres/VR", "today-hot", "weekly-hot", "monthly-hot", "siro", "luxu", "gana", "maan", "scute", "ara", "uncensored-leak", "fc2", "heyzo", "tokyohot", "1pondo", "caribbeancom", "caribbeancompr", "10musume", "pacopacomama", "gachinco", "xxxav", "marriedslash", "naughty4610", "naughty0930", "madou", "twav", "furuke"); for (Element a : doc.select("nav a")) { String typeId = a.attr("href").replace(url, ""); @@ -55,7 +55,7 @@ public class Miss extends Spider { String filters = extend.get("filters"); if (TextUtils.isEmpty(filters)) target += "?page=" + pg; else target += "?filters=" + extend.get("filters") + "&page=" + pg; - Document doc = Jsoup.parse(OkHttp.string(target)); + Document doc = Jsoup.parse(OkHttp.string(proxy(), target)); for (Element div : doc.select("div.thumbnail")) { String id = div.select("a.text-secondary").attr("href").replace(url, ""); String name = div.select("a.text-secondary").text(); @@ -70,7 +70,7 @@ public class Miss extends Spider { @Override public String detailContent(List ids) throws Exception { - Document doc = Jsoup.parse(OkHttp.string(url + ids.get(0))); + Document doc = Jsoup.parse(OkHttp.string(proxy(), url + ids.get(0))); String name = doc.select("meta[property=og:title]").attr("content"); String pic = doc.select("meta[property=og:image]").attr("content"); Vod vod = new Vod(); @@ -99,7 +99,7 @@ public class Miss extends Spider { private String searchContent(String key, String pg) { List list = new ArrayList<>(); - Document doc = Jsoup.parse(OkHttp.string(url + "search/" + key + "?page=" + pg)); + Document doc = Jsoup.parse(OkHttp.string(proxy(), url + "search/" + key + "?page=" + pg)); for (Element div : doc.select("div.thumbnail")) { String id = div.select("a.text-secondary").attr("href").replace(url, ""); String name = div.select("a.text-secondary").text(); diff --git a/app/src/main/java/com/github/catvod/spider/PanSou.java b/app/src/main/java/com/github/catvod/spider/PanSou.java index 7b777bbb..71b702be 100644 --- a/app/src/main/java/com/github/catvod/spider/PanSou.java +++ b/app/src/main/java/com/github/catvod/spider/PanSou.java @@ -35,9 +35,7 @@ public class PanSou extends Ali { public String detailContent(List ids) throws Exception { if (pattern.matcher(ids.get(0)).find()) return super.detailContent(ids); String url = siteUrl + ids.get(0).replace("/s/", "/cv/"); - Map> respHeaders = new HashMap<>(); - OkHttp.stringNoRedirect(url, getHeaders(ids.get(0)), respHeaders); - url = OkHttp.getRedirectLocation(respHeaders); + url = OkHttp.getLocation(url, getHeaders(ids.get(0))); return super.detailContent(Arrays.asList(url)); } diff --git a/app/src/main/java/com/github/catvod/spider/Star.java b/app/src/main/java/com/github/catvod/spider/Star.java index ba783ba6..58514390 100644 --- a/app/src/main/java/com/github/catvod/spider/Star.java +++ b/app/src/main/java/com/github/catvod/spider/Star.java @@ -44,7 +44,7 @@ public class Star extends Spider { } private String getVer() { - for (Element script : Jsoup.parse(OkHttp.string(siteUrl, getHeader())).select("script")) if (script.attr("src").contains("buildManifest.js")) return script.attr("src").split("/")[3]; + for (Element script : Jsoup.parse(OkHttp.string(proxy(), siteUrl, getHeader())).select("script")) if (script.attr("src").contains("buildManifest.js")) return script.attr("src").split("/")[3]; return ""; } @@ -65,7 +65,7 @@ public class Star extends Spider { LinkedHashMap> filters = new LinkedHashMap<>(); for (Map.Entry entry : map.entrySet()) classes.add(new Class(entry.getKey(), entry.getValue())); for (Class type : classes) { - Element script = Jsoup.parse(OkHttp.string(siteUrl + type.getTypeId() + "/all/all/all", getHeader())).select("#__NEXT_DATA__").get(0); + Element script = Jsoup.parse(OkHttp.string(proxy(), siteUrl + type.getTypeId() + "/all/all/all", getHeader())).select("#__NEXT_DATA__").get(0); JSONObject obj = new JSONObject(script.data()).getJSONObject("props").getJSONObject("pageProps").getJSONObject("filterCondition"); Condition item = Condition.objectFrom(obj.toString()); filters.put(type.getTypeId(), item.getFilter()); @@ -76,7 +76,7 @@ public class Star extends Spider { @Override public String homeVideoContent() throws Exception { List list = new ArrayList<>(); - Element script = Jsoup.parse(OkHttp.string(siteUrl)).select("#__NEXT_DATA__").get(0); + Element script = Jsoup.parse(OkHttp.string(proxy(), siteUrl)).select("#__NEXT_DATA__").get(0); List cards = Card.arrayFrom(new JSONObject(script.data()).getJSONObject("props").getJSONObject("pageProps").getJSONArray("cards").toString()); for (Card card : cards) if (!card.getName().equals("电视直播")) for (Card item : card.getCards()) list.add(item.vod()); return Result.string(list); @@ -94,7 +94,7 @@ public class Star extends Spider { if (year.length() > 0) query.setYear(year); if (type.length() > 0) query.setLabel(type); if (area.length() > 0) query.setCountry(area); - String body = OkHttp.postJson(apiUrl, query.toString()).getBody(); + String body = OkHttp.post(proxy(), apiUrl, query.toString()); List cards = Card.arrayFrom(new JSONObject(body).getJSONObject("data").getJSONArray("list").toString()); List list = new ArrayList<>(); for (Card card : cards) list.add(card.vod()); @@ -103,7 +103,7 @@ public class Star extends Spider { @Override public String detailContent(List ids) throws Exception { - Element script = Jsoup.parse(OkHttp.string(detail.concat(ids.get(0)), getHeader())).select("#__NEXT_DATA__").get(0); + Element script = Jsoup.parse(OkHttp.string(proxy(), detail.concat(ids.get(0)), getHeader())).select("#__NEXT_DATA__").get(0); Detail detail = Detail.objectFrom(new JSONObject(script.data()).getJSONObject("props").getJSONObject("pageProps").getJSONObject("pageData").toString()); Vod vod = new Vod(); vod.setVodId(ids.get(0)); @@ -126,7 +126,7 @@ public class Star extends Spider { @Override public String searchContent(String key, boolean quick) throws Exception { List list = new ArrayList<>(); - String json = OkHttp.string(siteUrl + data + ver + "/search.json?word=" + URLEncoder.encode(key), getHeader()); + String json = OkHttp.string(proxy(), siteUrl + data + ver + "/search.json?word=" + URLEncoder.encode(key), getHeader()); List items = Card.arrayFrom(new JSONObject(json).getJSONObject("pageProps").getJSONArray("initList").toString()); for (Card item : items) list.add(item.vod()); return Result.string(list); diff --git a/app/src/main/java/com/github/catvod/spider/XPath.java b/app/src/main/java/com/github/catvod/spider/XPath.java index c260a9b9..de126c0d 100644 --- a/app/src/main/java/com/github/catvod/spider/XPath.java +++ b/app/src/main/java/com/github/catvod/spider/XPath.java @@ -324,7 +324,7 @@ public class XPath extends Spider { if (rule == null) { if (ext != null) { if (ext.startsWith("http")) { - String json = OkHttp.string(ext, null); + String json = OkHttp.string(proxy(), ext, null); rule = Rule.fromJson(json); loadRuleExt(json); } else { @@ -340,6 +340,6 @@ public class XPath extends Spider { protected String fetch(String webUrl) { SpiderDebug.log(webUrl); - return OkHttp.string(webUrl, getHeaders()); + return OkHttp.string(proxy(), webUrl, getHeaders()); } } \ No newline at end of file diff --git a/app/src/main/java/com/github/catvod/spider/XiaoZhiTiao.java b/app/src/main/java/com/github/catvod/spider/XiaoZhiTiao.java index fb8a9f43..38bbc9bf 100644 --- a/app/src/main/java/com/github/catvod/spider/XiaoZhiTiao.java +++ b/app/src/main/java/com/github/catvod/spider/XiaoZhiTiao.java @@ -12,7 +12,11 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * XiaoZhiTiao @@ -46,7 +50,7 @@ public class XiaoZhiTiao extends Ali { params.put("from", "web"); params.put("keyword", key); params.put("token", getToken()); - JSONObject object = new JSONObject(OkHttp.post(siteUrl, params, getHeaders())); + JSONObject object = new JSONObject(OkHttp.post(siteUrl, params, getHeaders()).getBody()); if (!object.optBoolean("success")) return ""; List list = new ArrayList<>(); JSONArray data = object.getJSONArray("data"); @@ -65,9 +69,9 @@ public class XiaoZhiTiao extends Ali { Map params = new HashMap<>(); params.put("action", "get_token"); params.put("from", "web"); - JSONObject jsonObject = new JSONObject(OkHttp.post(siteUrl, params, getHeaders())); - if (jsonObject.getBoolean("success")) { - token = jsonObject.getString("data"); + JSONObject object = new JSONObject(OkHttp.post(siteUrl, params, getHeaders()).getBody()); + if (object.getBoolean("success")) { + token = object.getString("data"); date = new Date(System.currentTimeMillis() + 24 * 60 * 60 * 1000); } } diff --git a/app/src/main/java/com/github/catvod/spider/Zhaozy.java b/app/src/main/java/com/github/catvod/spider/Zhaozy.java index 56195c09..88ea187a 100644 --- a/app/src/main/java/com/github/catvod/spider/Zhaozy.java +++ b/app/src/main/java/com/github/catvod/spider/Zhaozy.java @@ -44,9 +44,8 @@ public class Zhaozy extends Ali { headers.put("User-Agent", Utils.CHROME); headers.put("Referer", siteUrl + "stop.html"); headers.put("Origin", siteUrl); - Map> resp = new HashMap<>(); - OkHttp.post(siteUrl + "logiu.html", params, headers, resp); StringBuilder sb = new StringBuilder(); + Map> resp = OkHttp.post(siteUrl + "logiu.html", params, headers).getResp(); for (String item : resp.get("set-cookie")) sb.append(item.split(";")[0]).append(";"); return sb.toString(); }