dongge
/
AndroidCatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean code

This commit is contained in:
FongMi 2023-09-24 01:52:02 +08:00
parent 3487be4637
commit 2a6cbc0c7c
17 changed files with 186 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/src/main/java/com/github/catvod/api/AliYun.java
View File

@ -125,7 +125,7 @@ public class AliYun {
private boolean alist(String url, JsonObject param) {
String api = "https://aliapi.ewwe.gq/alist/ali_open/" + url;
OkResult result = OkHttp.postJson(api, param.toString(), getHeader());
OkResult result = OkHttp.post(api, param.toString(), getHeader());
SpiderDebug.log(result.getCode() + "," + api + "," + result.getBody());
if (isManyRequest(result.getBody())) return false;
oauth = OAuth.objectFrom(result.getBody()).save();
@ -134,14 +134,14 @@ public class AliYun {
private String post(String url, JsonObject param) {
url = url.startsWith("https") ? url : "https://api.aliyundrive.com/" + url;
OkResult result = OkHttp.postJson(url, param.toString(), getHeader());
OkResult result = OkHttp.post(url, param.toString(), getHeader());
SpiderDebug.log(result.getCode() + "," + url + "," + result.getBody());
return result.getBody();
}
private String auth(String url, String json, boolean retry) {
url = url.startsWith("https") ? url : "https://api.aliyundrive.com/" + url;
OkResult result = OkHttp.postJson(url, json, getHeaderAuth());
OkResult result = OkHttp.post(url, json, getHeaderAuth());
SpiderDebug.log(result.getCode() + "," + url + "," + result.getBody());
if (retry && result.getCode() == 401 && refreshAccessToken()) return auth(url, json, false);
if (retry && result.getCode() == 429) return auth(url, json, false);
@ -150,7 +150,7 @@ public class AliYun {
private String oauth(String url, String json, boolean retry) {
url = url.startsWith("https") ? url : "https://open.aliyundrive.com/adrive/v1.0/" + url;
OkResult result = OkHttp.postJson(url, json, getHeaderOpen());
OkResult result = OkHttp.post(url, json, getHeaderOpen());
SpiderDebug.log(result.getCode() + "," + url + "," + result.getBody());
if (retry && (result.getCode() == 400 || result.getCode() == 401) && refreshOpenToken()) return oauth(url, json, false);
return result.getBody();

22
app/src/main/java/com/github/catvod/crawler/Spider.java
View File

@ -4,9 +4,19 @@ import android.content.Context;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import okhttp3.Dns;
public abstract class Spider {
private boolean proxy;
public Spider proxy(boolean proxy) {
this.proxy = proxy;
return this;
}
public void init(Context context) throws Exception {
}
@ -50,6 +60,18 @@ public abstract class Spider {
return false;
}
public boolean proxy() {
return proxy;
}
public Object[] proxyLocal(Map<String, String> params) throws Exception {
return null;
}
public void destroy() {
}
public static Dns safeDns() {
return Dns.SYSTEM;
}
}

43
app/src/main/java/com/github/catvod/net/Cronet.java
View File

@ -5,10 +5,12 @@ import com.google.net.cronet.okhttptransport.CronetInterceptor;
import org.chromium.net.CronetEngine;
import java.util.List;
import java.io.IOException;
import java.util.Map;
import okhttp3.Headers;
import okhttp3.OkHttpClient;
import okhttp3.Request;
public class Cronet {
@ -53,36 +55,20 @@ public class Cronet {
return get().noRedirect;
}
public static void stringNoRedirect(String url, Map<String, String> header, Map<String, List<String>> respHeader) {
string(noRedirect(), GET, url, null, null, header, respHeader);
}
public static String string(OkHttpClient client, String method, String url, String tag, Map<String, String> params, Map<String, String> header, Map<String, List<String>> respHeader) {
return new OkRequest(method, url, params, header, respHeader).tag(tag).execute(client).getBody();
}
public static String string(String url) {
return string(url, null);
}
public static String string(String url, Map<String, String> header) {
return string(url, header, null);
return string(url, null, header);
}
public static String string(String url, Map<String, String> header, Map<String, List<String>> respHeader) {
return string(url, null, header, respHeader);
public static String string(String url, Map<String, String> params, Map<String, String> header) {
return string(client(), GET, url, params, header);
}
public static String string(String url, Map<String, String> params, Map<String, String> header, Map<String, List<String>> respHeader) {
return string(url, null, params, header, respHeader);
}
public static String string(String url, String tag, Map<String, String> header) {
return string(url, tag, null, header, null);
}
public static String string(String url, String tag, Map<String, String> params, Map<String, String> header, Map<String, List<String>> respHeader) {
return string(client(), GET, url, tag, params, header, respHeader);
public static String string(OkHttpClient client, String method, String url, Map<String, String> params, Map<String, String> header) {
return new OkRequest(method, url, params, header).execute(client).getBody();
}
public static String post(String url, Map<String, String> params) {
@ -90,11 +76,7 @@ public class Cronet {
}
public static String post(String url, Map<String, String> params, Map<String, String> header) {
return post(url, params, header, null);
}
public static String post(String url, Map<String, String> params, Map<String, String> header, Map<String, List<String>> respHeader) {
return string(client(), POST, url, null, params, header, respHeader);
return string(client(), POST, url, params, header);
}
public static OkResult postJson(String url, String json) {
@ -105,10 +87,7 @@ public class Cronet {
return new OkRequest(POST, url, json, header).execute(client());
}
public static String getRedirectLocation(Map<String, List<String>> headers) {
if (headers == null) return null;
if (headers.containsKey("location")) return headers.get("location").get(0);
if (headers.containsKey("Location")) return headers.get("Location").get(0);
return null;
public static String getLocation(String url, Map<String, String> header) throws IOException {
return OkHttp.getLocation(noRedirect().newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute().headers().toMultimap());
}
}

164
app/src/main/java/com/github/catvod/net/OkHttp.java
View File

@ -3,15 +3,17 @@ package com.github.catvod.net;
import android.net.Uri;
import com.github.catvod.crawler.Spider;
import com.github.catvod.utils.Prefers;
import java.io.IOException;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import okhttp3.Call;
import okhttp3.Credentials;
import okhttp3.Dns;
import okhttp3.Headers;
@ -24,8 +26,8 @@ public class OkHttp {
public static final String POST = "POST";
public static final String GET = "GET";
private final OkHttpClient noRedirect;
private final OkHttpClient client;
private OkHttpClient proxy;
private OkHttpClient client;
private static class Loader {
static volatile OkHttp INSTANCE = new OkHttp();
@ -35,139 +37,129 @@ public class OkHttp {
return Loader.INSTANCE;
}
public OkHttp() {
client = getBuilder().build();
noRedirect = client.newBuilder().followRedirects(false).followSslRedirects(false).build();
public static Dns dns() {
return Spider.safeDns();
}
public static OkHttpClient.Builder getBuilder() {
return getBuilder(proxy());
}
public static OkHttpClient.Builder getBuilder(Uri proxy) {
OkHttpClient.Builder builder = new OkHttpClient.Builder().dns(safeDns()).readTimeout(30, TimeUnit.SECONDS).writeTimeout(30, TimeUnit.SECONDS).connectTimeout(30, TimeUnit.SECONDS).hostnameVerifier(SSLCompat.VERIFIER).sslSocketFactory(new SSLCompat(), SSLCompat.TM);
if (proxy != null && proxy.getScheme() != null && proxy.getHost() != null && proxy.getPort() > 0) setProxy(builder, proxy);
return builder;
public void resetProxy() {
Authenticator.setDefault(null);
proxy = null;
}
public static OkHttpClient client() {
return get().client;
if (get().client != null) return get().client;
return get().client = getBuilder().build();
}
private static OkHttpClient noRedirect() {
return get().noRedirect;
public static OkHttpClient proxy() {
if (get().proxy != null) return get().proxy;
return get().proxy = getBuilder(Prefers.getString("proxy")).build();
}
public static Dns safeDns() {
try {
return (Dns) Spider.class.getMethod("safeDns").invoke(null);
} catch (Exception e) {
return Dns.SYSTEM;
}
public static OkHttpClient client(boolean proxy) {
return proxy ? proxy() : client();
}
public static Uri proxy() {
try {
return (Uri) Spider.class.getMethod("proxy").invoke(null);
} catch (Exception e) {
return null;
}
}
public static Response newCall(String url) throws IOException {
return client().newCall(new Request.Builder().url(url).build()).execute();
public static OkHttpClient noRedirect(boolean proxy) {
return client(proxy).newBuilder().followRedirects(false).followSslRedirects(false).build();
}
public static Response newCall(String url, Map<String, String> header) throws IOException {
return client().newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute();
}
public static void stringNoRedirect(String url, Map<String, String> header, Map<String, List<String>> respHeader) {
string(noRedirect(), GET, url, null, null, header, respHeader);
}
public static String string(OkHttpClient client, String method, String url, String tag, Map<String, String> params, Map<String, String> header, Map<String, List<String>> respHeader) {
return new OkRequest(method, url, params, header, respHeader).tag(tag).execute(client).getBody();
return client(false).newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute();
}
public static String string(String url) {
return string(url, null);
return string(false, url);
}
public static String string(boolean proxy, String url) {
return string(proxy, url, null);
}
public static String string(String url, Map<String, String> header) {
return string(url, header, null);
return string(false, url, header);
}
public static String string(String url, Map<String, String> header, Map<String, List<String>> respHeader) {
return string(url, null, header, respHeader);
public static String string(boolean proxy, String url, Map<String, String> header) {
return string(proxy, url, null, header);
}
public static String get(String url, Map<String, String> params, Map<String, String> header) {
return string(url, params, header, null);
public static String string(boolean proxy, String url, Map<String, String> params, Map<String, String> header) {
return string(client(proxy), url, params, header);
}
public static String string(String url, Map<String, String> params, Map<String, String> header, Map<String, List<String>> respHeader) {
return string(url, null, params, header, respHeader);
}
public static String string(String url, String tag, Map<String, String> header) {
return string(url, tag, null, header, null);
}
public static String string(String url, String tag, Map<String, String> params, Map<String, String> header, Map<String, List<String>> respHeader) {
return string(client(), GET, url, tag, params, header, respHeader);
public static String string(OkHttpClient client, String url, Map<String, String> params, Map<String, String> header) {
return new OkRequest(GET, url, params, header).execute(client).getBody();
}
public static String post(String url, Map<String, String> params) {
return post(url, params, null);
return post(false, url, params);
}
public static String post(String url, Map<String, String> params, Map<String, String> header) {
return post(url, params, header, null);
public static String post(boolean proxy, String url, Map<String, String> params) {
return post(proxy, url, params, null).getBody();
}
public static String post(String url, Map<String, String> params, Map<String, String> header, Map<String, List<String>> respHeader) {
return string(client(), POST, url, null, params, header, respHeader);
public static String post(String url, String json) {
return post(false, url, json);
}
public static OkResult postJson(String url, String json) {
return postJson(url, json, null);
public static String post(boolean proxy, String url, String json) {
return post(proxy, url, json, null).getBody();
}
public static OkResult postJson(String url, String json, Map<String, String> header) {
return new OkRequest(POST, url, json, header).execute(client());
public static OkResult post(String url, String json, Map<String, String> header) {
return post(false, url, json, header);
}
public static void cancel(Object tag) {
for (Call call : client().dispatcher().queuedCalls()) {
if (tag.equals(call.request().tag())) {
call.cancel();
}
}
for (Call call : client().dispatcher().runningCalls()) {
if (tag.equals(call.request().tag())) {
call.cancel();
}
}
public static OkResult post(boolean proxy, String url, String json, Map<String, String> header) {
return new OkRequest(POST, url, json, header).execute(client(proxy));
}
public static String getRedirectLocation(Map<String, List<String>> headers) {
public static OkResult post(String url, Map<String, String> params, Map<String, String> header) {
return post(false, url, params, header);
}
public static OkResult post(boolean proxy, String url, Map<String, String> params, Map<String, String> header) {
return new OkRequest(POST, url, params, header).execute(client(proxy));
}
public static String getLocation(String url, Map<String, String> header) throws IOException {
return getLocation(noRedirect(false).newCall(new Request.Builder().url(url).headers(Headers.of(header)).build()).execute().headers().toMultimap());
}
public static String getLocation(Map<String, List<String>> headers) {
if (headers == null) return null;
if (headers.containsKey("location")) return headers.get("location").get(0);
if (headers.containsKey("Location")) return headers.get("Location").get(0);
return null;
}
private static void setProxy(OkHttpClient.Builder builder, Uri proxy) {
String userInfo = proxy.getUserInfo();
if (proxy.getScheme() == null || proxy.getScheme().startsWith("socks")) {
builder.proxy(new Proxy(Proxy.Type.SOCKS, InetSocketAddress.createUnresolved(proxy.getHost(), proxy.getPort())));
} else if (proxy.getScheme().startsWith("http")) {
builder.proxy(new Proxy(Proxy.Type.HTTP, InetSocketAddress.createUnresolved(proxy.getHost(), proxy.getPort())));
if (userInfo != null && userInfo.contains(":")) builder.proxyAuthenticator((route, response) -> {
public static OkHttpClient.Builder getBuilder() {
return new OkHttpClient.Builder().dns(dns()).connectTimeout(30, TimeUnit.SECONDS).readTimeout(30, TimeUnit.SECONDS).writeTimeout(30, TimeUnit.SECONDS).hostnameVerifier(SSLCompat.VERIFIER).sslSocketFactory(new SSLCompat(), SSLCompat.TM);
}
private static OkHttpClient.Builder getBuilder(String proxy) {
Uri uri = Uri.parse(proxy);
String userInfo = uri.getUserInfo();
OkHttpClient.Builder builder = client().newBuilder();
if (userInfo != null && userInfo.contains(":")) setAuthenticator(builder, userInfo);
if (uri.getScheme() == null || uri.getHost() == null || uri.getPort() <= 0) return builder;
if (uri.getScheme().startsWith("http")) builder.proxy(new Proxy(Proxy.Type.HTTP, InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort())));
if (uri.getScheme().startsWith("socks")) builder.proxy(new Proxy(Proxy.Type.SOCKS, InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort())));
return builder;
}
private static void setAuthenticator(OkHttpClient.Builder builder, String userInfo) {
builder.proxyAuthenticator((route, response) -> {
String credential = Credentials.basic(userInfo.split(":")[0], userInfo.split(":")[1]);
return response.request().newBuilder().header("Proxy-Authorization", credential).build();
});
Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(userInfo.split(":")[0], userInfo.split(":")[1].toCharArray());
}
});
}
}

15
app/src/main/java/com/github/catvod/net/OkRequest.java
View File

@ -5,7 +5,6 @@ import android.text.TextUtils;
import com.github.catvod.utils.Utils;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import okhttp3.FormBody;
@ -17,7 +16,6 @@ import okhttp3.Response;
class OkRequest {
private final Map<String, List<String>> respHeader;
private final Map<String, String> header;
private final Map<String, String> params;
private final String method;
@ -26,21 +24,20 @@ class OkRequest {
private String url;
private Object tag;
OkRequest(String method, String url, Map<String, String> params, Map<String, String> header, Map<String, List<String>> respHeader) {
this(method, url, null, params, header, respHeader);
OkRequest(String method, String url, Map<String, String> params, Map<String, String> header) {
this(method, url, null, params, header);
}
OkRequest(String method, String url, String json, Map<String, String> header) {
this(method, url, json, null, header, null);
this(method, url, json, null, header);
}
private OkRequest(String method, String url, String json, Map<String, String> params, Map<String, String> header, Map<String, List<String>> respHeader) {
private OkRequest(String method, String url, String json, Map<String, String> params, Map<String, String> header) {
this.url = url;
this.json = json;
this.method = method;
this.params = params;
this.header = header;
this.respHeader = respHeader;
getInstance();
}
@ -74,9 +71,7 @@ class OkRequest {
public OkResult execute(OkHttpClient client) {
try {
Response response = client.newCall(request).execute();
if (respHeader != null) respHeader.clear();
if (respHeader != null) respHeader.putAll(response.headers().toMultimap());
return new OkResult(response.code(), response.body().string());
return new OkResult(response.code(), response.body().string(), response.headers().toMultimap());
} catch (IOException e) {
return new OkResult();
}

13
app/src/main/java/com/github/catvod/net/OkResult.java
View File

@ -2,19 +2,26 @@ package com.github.catvod.net;
import android.text.TextUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class OkResult {
private final int code;
private final String body;
private final Map<String, List<String>> resp;
public OkResult() {
this.code = 500;
this.body = "";
this.resp = new HashMap<>();
}
public OkResult(int code, String body) {
public OkResult(int code, String body, Map<String, List<String>> resp) {
this.code = code;
this.body = body;
this.resp = resp;
}
public int getCode() {
@ -24,4 +31,8 @@ public class OkResult {
public String getBody() {
return TextUtils.isEmpty(body) ? "" : body;
}
public Map<String, List<String>> getResp() {
return resp;
}
}

4
app/src/main/java/com/github/catvod/spider/AList.java
View File

@ -61,7 +61,7 @@ public class AList extends Spider {
}
private String post(Drive drive, String url, String param, boolean retry) {
String response = OkHttp.postJson(url, param, drive.getHeader()).getBody();
String response = OkHttp.post(url, param, drive.getHeader()).getBody();
SpiderDebug.log(response);
if (retry && response.contains("Guest user is disabled") && login(drive)) return post(drive, url, param, false);
return response;
@ -152,7 +152,7 @@ public class AList extends Spider {
JSONObject params = new JSONObject();
params.put("username", drive.getLogin().getUsername());
params.put("password", drive.getLogin().getPassword());
String response = OkHttp.postJson(drive.loginApi(), params.toString()).getBody();
String response = OkHttp.post(drive.loginApi(), params.toString());
drive.setToken(new JSONObject(response).getJSONObject("data").getString("token"));
return true;
} catch (Exception e) {

18
app/src/main/java/com/github/catvod/spider/AppYsV2.java
View File

@ -13,7 +13,11 @@ import org.json.JSONException;
import org.json.JSONObject;
import java.net.URLEncoder;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -38,7 +42,7 @@ public class AppYsV2 extends Spider {
JSONArray jsonArray = null;
if (!url.isEmpty()) {
SpiderDebug.log(url);
String json = OkHttp.string(url, getHeaders(url));
String json = OkHttp.string(proxy(), url, getHeaders(url));
JSONObject obj = new JSONObject(json);
if (obj.has("list") && obj.get("list") instanceof JSONArray) {
jsonArray = obj.getJSONArray("list");
@ -141,7 +145,7 @@ public class AppYsV2 extends Spider {
isTV = true;
}
SpiderDebug.log(url);
String json = OkHttp.string(url, getHeaders(url));
String json = OkHttp.string(proxy(), url, getHeaders(url));
JSONObject obj = new JSONObject(json);
JSONArray videos = new JSONArray();
if (isTV) {
@ -193,7 +197,7 @@ public class AppYsV2 extends Spider {
url = url.replace("筛选year", (extend != null && extend.containsKey("year")) ? extend.get("year") : "");
url = url.replace("排序", (extend != null && extend.containsKey("排序")) ? extend.get("排序") : "");
SpiderDebug.log(url);
String json = OkHttp.string(url, getHeaders(url));
String json = OkHttp.string(proxy(), url, getHeaders(url));
JSONObject obj = new JSONObject(json);
int totalPg = Integer.MAX_VALUE;
try {
@ -253,7 +257,7 @@ public class AppYsV2 extends Spider {
String apiUrl = getApiUrl();
String url = getPlayUrlPrefix(apiUrl) + ids.get(0);
SpiderDebug.log(url);
String json = OkHttp.string(url, getHeaders(url));
String json = OkHttp.string(proxy(), url, getHeaders(url));
JSONObject obj = new JSONObject(json);
JSONObject result = new JSONObject();
JSONObject vod = new JSONObject();
@ -268,7 +272,7 @@ public class AppYsV2 extends Spider {
public String searchContent(String key, boolean quick) throws Exception {
String apiUrl = getApiUrl();
String url = getSearchUrl(apiUrl, URLEncoder.encode(key));
String json = OkHttp.string(url, getHeaders(url));
String json = OkHttp.string(proxy(), url, getHeaders(url));
JSONObject obj = new JSONObject(json);
JSONArray jsonArray = null;
JSONArray videos = new JSONArray();
@ -690,7 +694,7 @@ public class AppYsV2 extends Spider {
for (String parseUrl : parseUrls) {
if (parseUrl.isEmpty() || parseUrl.equals("null")) continue;
String playUrl = parseUrl + url;
String content = OkHttp.string(playUrl);
String content = OkHttp.string(proxy(), playUrl);
JSONObject tryJson = null;
try {
tryJson = jsonParse(url, content);

8
app/src/main/java/com/github/catvod/spider/Eighteen.java
View File

@ -27,7 +27,7 @@ public class Eighteen extends Spider {
public String homeContent(boolean filter) throws Exception {
List<Class> classes = new ArrayList<>();
List<Vod> list = new ArrayList<>();
Document doc = Jsoup.parse(OkHttp.string(url));
Document doc = Jsoup.parse(OkHttp.string(proxy(), url));
for (Element a : doc.select("ul.animenu__nav > li > a")) {
String typeName = a.text();
String typeId = a.attr("href").replace(url, "");
@ -50,7 +50,7 @@ public class Eighteen extends Spider {
List<Vod> list = new ArrayList<>();
tid = tid.replace("random", "list");
tid = tid.replace("index", pg);
Document doc = Jsoup.parse(OkHttp.string(url + tid));
Document doc = Jsoup.parse(OkHttp.string(proxy(), url + tid));
for (Element div : doc.select("div.post")) {
String id = div.select("a").attr("href").replace(url, "");
String name = div.select("h3").text();
@ -63,7 +63,7 @@ public class Eighteen extends Spider {
@Override
public String detailContent(List<String> ids) throws Exception {
Document doc = Jsoup.parse(OkHttp.string(url + ids.get(0)));
Document doc = Jsoup.parse(OkHttp.string(proxy(), url + ids.get(0)));
Element wrap = doc.select("div.video-wrap").get(0);
String name = wrap.select("div.archive-title > h1").text();
String pic = wrap.select("div.player-wrap > img").attr("src");
@ -99,7 +99,7 @@ public class Eighteen extends Spider {
params.put("search_keyword", key);
params.put("search_type", "fc");
params.put("op", "search");
String res = OkHttp.post(url + "searchform_search/all/" + pg + ".html", params);
String res = OkHttp.post(proxy(), url + "searchform_search/all/" + pg + ".html", params);
List<Vod> list = new ArrayList<>();
for (Element div : Jsoup.parse(res).select("div.post")) {
String id = div.select("a").attr("href").replace(url, "");

10
app/src/main/java/com/github/catvod/spider/Hanime.java
View File

@ -43,10 +43,10 @@ public class Hanime extends Spider {
List<Vod> list = new ArrayList<>();
List<Class> classes = new ArrayList<>();
LinkedHashMap<String, List<Filter>> filters = new LinkedHashMap<>();
Document doc1 = Jsoup.parse(OkHttp.string(siteUrl.concat("/search?genre=裏番"), getHeaders()));
Document doc1 = Jsoup.parse(OkHttp.string(proxy(), siteUrl.concat("/search?genre=裏番"), getHeaders()));
List<String> sorts = doc1.select("div.hentai-sort-options-wrapper").eachText();
List<String> years = doc1.getElementById("year").select("option").eachAttr("value");
Document doc2 = Jsoup.parse(OkHttp.string(siteUrl, getHeaders()));
Document doc2 = Jsoup.parse(OkHttp.string(proxy(), siteUrl, getHeaders()));
for (Element element : doc2.select("a.nav-item")) {
String text = element.text();
if (text.equals("新番預告") || text.equals("H漫畫")) continue;
@ -75,7 +75,7 @@ public class Hanime extends Spider {
if (extend.get("by") == null) extend.put("by", "最新上市");
if (extend.get("year") == null) extend.put("year", "");
String target = siteUrl.concat("/search?genre=").concat(tid).concat("&page=").concat(pg).concat("&sort=").concat(extend.get("by")).concat("&year=").concat(extend.get("year"));
Document doc = Jsoup.parse(OkHttp.string(target, getHeaders()));
Document doc = Jsoup.parse(OkHttp.string(proxy(), target, getHeaders()));
for (Element element : doc.select("div.col-xs-6")) {
String pic = element.select("img").get(1).attr("src");
String url = element.select("a.overlay").attr("href");
@ -100,7 +100,7 @@ public class Hanime extends Spider {
@Override
public String detailContent(List<String> ids) throws Exception {
Document doc = Jsoup.parse(OkHttp.string(siteUrl.concat("/watch?v=").concat(ids.get(0)), getHeaders()));
Document doc = Jsoup.parse(OkHttp.string(proxy(), siteUrl.concat("/watch?v=").concat(ids.get(0)), getHeaders()));
String name = doc.getElementById("shareBtn-title").text();
JSONObject json = new JSONObject(doc.select("script[type=application/ld+json]").html().trim());
String content = json.optString("description");
@ -120,7 +120,7 @@ public class Hanime extends Spider {
public String searchContent(String key, boolean quick) throws Exception {
List<Vod> list = new ArrayList<>();
String target = siteUrl.concat("/search?query=").concat(key).concat("&genre=&sort=&year=&month=&duration=");
Document doc = Jsoup.parse(OkHttp.string(target, getHeaders()));
Document doc = Jsoup.parse(OkHttp.string(proxy(), target, getHeaders()));
for (Element element : doc.select("div.col-xs-6")) {
String pic = element.select("img").get(1).attr("src");
String url = element.select("a.overlay").attr("href");

10
app/src/main/java/com/github/catvod/spider/Jable.java
View File

@ -34,13 +34,13 @@ public class Jable extends Spider {
public String homeContent(boolean filter) throws Exception {
List<Vod> list = new ArrayList<>();
List<Class> classes = new ArrayList<>();
Document doc = Jsoup.parse(OkHttp.string(cateUrl, getHeaders()));
Document doc = Jsoup.parse(OkHttp.string(proxy(), cateUrl, getHeaders()));
for (Element element : doc.select("div.img-box > a")) {
String typeId = element.attr("href").split("/")[4];
String typeName = element.select("div.absolute-center > h4").text();
classes.add(new Class(typeId, typeName));
}
doc = Jsoup.parse(OkHttp.string(siteUrl, getHeaders()));
doc = Jsoup.parse(OkHttp.string(proxy(), siteUrl, getHeaders()));
for (Element element : doc.select("div.video-img-box")) {
String pic = element.select("img").attr("data-src");
String url = element.select("a").attr("href");
@ -56,7 +56,7 @@ public class Jable extends Spider {
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws Exception {
List<Vod> list = new ArrayList<>();
String target = cateUrl + tid + "/?mode=async&function=get_block&block_id=list_videos_common_videos_list&sort_by=post_date&from=" + String.format(Locale.getDefault(), "%02d", Integer.parseInt(pg)) + "&_=" + System.currentTimeMillis();
Document doc = Jsoup.parse(OkHttp.string(target, getHeaders()));
Document doc = Jsoup.parse(OkHttp.string(proxy(), target, getHeaders()));
for (Element element : doc.select("div.video-img-box")) {
String pic = element.select("img").attr("data-src");
String url = element.select("a").attr("href");
@ -69,7 +69,7 @@ public class Jable extends Spider {
@Override
public String detailContent(List<String> ids) throws Exception {
Document doc = Jsoup.parse(OkHttp.string(detailUrl.concat(ids.get(0)).concat("/"), getHeaders()));
Document doc = Jsoup.parse(OkHttp.string(proxy(), detailUrl.concat(ids.get(0)).concat("/"), getHeaders()));
String name = doc.select("meta[property=og:title]").attr("content");
String pic = doc.select("meta[property=og:image]").attr("content");
String year = doc.select("span.inactive-color").get(0).text();
@ -86,7 +86,7 @@ public class Jable extends Spider {
@Override
public String searchContent(String key, boolean quick) throws Exception {
List<Vod> list = new ArrayList<>();
Document doc = Jsoup.parse(OkHttp.string(searchUrl.concat(URLEncoder.encode(key)).concat("/"), getHeaders()));
Document doc = Jsoup.parse(OkHttp.string(proxy(), searchUrl.concat(URLEncoder.encode(key)).concat("/"), getHeaders()));
for (Element element : doc.select("div.video-img-box")) {
String pic = element.select("img").attr("data-src");
String url = element.select("a").attr("href");

8
app/src/main/java/com/github/catvod/spider/Miss.java
View File

@ -28,7 +28,7 @@ public class Miss extends Spider {
List<Vod> list = new ArrayList<>();
List<Class> classes = new ArrayList<>();
LinkedHashMap<String, List<Filter>> filters = new LinkedHashMap<>();
Document doc = Jsoup.parse(OkHttp.string(url));
Document doc = Jsoup.parse(OkHttp.string(proxy(), url));
List<String> typeIds = Arrays.asList("chinese-subtitle", "new", "release", "uncensored-leak", "genres/VR", "today-hot", "weekly-hot", "monthly-hot", "siro", "luxu", "gana", "maan", "scute", "ara", "uncensored-leak", "fc2", "heyzo", "tokyohot", "1pondo", "caribbeancom", "caribbeancompr", "10musume", "pacopacomama", "gachinco", "xxxav", "marriedslash", "naughty4610", "naughty0930", "madou", "twav", "furuke");
for (Element a : doc.select("nav a")) {
String typeId = a.attr("href").replace(url, "");
@ -55,7 +55,7 @@ public class Miss extends Spider {
String filters = extend.get("filters");
if (TextUtils.isEmpty(filters)) target += "?page=" + pg;
else target += "?filters=" + extend.get("filters") + "&page=" + pg;
Document doc = Jsoup.parse(OkHttp.string(target));
Document doc = Jsoup.parse(OkHttp.string(proxy(), target));
for (Element div : doc.select("div.thumbnail")) {
String id = div.select("a.text-secondary").attr("href").replace(url, "");
String name = div.select("a.text-secondary").text();
@ -70,7 +70,7 @@ public class Miss extends Spider {
@Override
public String detailContent(List<String> ids) throws Exception {
Document doc = Jsoup.parse(OkHttp.string(url + ids.get(0)));
Document doc = Jsoup.parse(OkHttp.string(proxy(), url + ids.get(0)));
String name = doc.select("meta[property=og:title]").attr("content");
String pic = doc.select("meta[property=og:image]").attr("content");
Vod vod = new Vod();
@ -99,7 +99,7 @@ public class Miss extends Spider {
private String searchContent(String key, String pg) {
List<Vod> list = new ArrayList<>();
Document doc = Jsoup.parse(OkHttp.string(url + "search/" + key + "?page=" + pg));
Document doc = Jsoup.parse(OkHttp.string(proxy(), url + "search/" + key + "?page=" + pg));
for (Element div : doc.select("div.thumbnail")) {
String id = div.select("a.text-secondary").attr("href").replace(url, "");
String name = div.select("a.text-secondary").text();

4
app/src/main/java/com/github/catvod/spider/PanSou.java
View File

@ -35,9 +35,7 @@ public class PanSou extends Ali {
public String detailContent(List<String> ids) throws Exception {
if (pattern.matcher(ids.get(0)).find()) return super.detailContent(ids);
String url = siteUrl + ids.get(0).replace("/s/", "/cv/");
Map<String, List<String>> respHeaders = new HashMap<>();
OkHttp.stringNoRedirect(url, getHeaders(ids.get(0)), respHeaders);
url = OkHttp.getRedirectLocation(respHeaders);
url = OkHttp.getLocation(url, getHeaders(ids.get(0)));
return super.detailContent(Arrays.asList(url));
}

12
app/src/main/java/com/github/catvod/spider/Star.java
View File

@ -44,7 +44,7 @@ public class Star extends Spider {
}
private String getVer() {
for (Element script : Jsoup.parse(OkHttp.string(siteUrl, getHeader())).select("script")) if (script.attr("src").contains("buildManifest.js")) return script.attr("src").split("/")[3];
for (Element script : Jsoup.parse(OkHttp.string(proxy(), siteUrl, getHeader())).select("script")) if (script.attr("src").contains("buildManifest.js")) return script.attr("src").split("/")[3];
return "";
}
@ -65,7 +65,7 @@ public class Star extends Spider {
LinkedHashMap<String, List<Filter>> filters = new LinkedHashMap<>();
for (Map.Entry<String, String> entry : map.entrySet()) classes.add(new Class(entry.getKey(), entry.getValue()));
for (Class type : classes) {
Element script = Jsoup.parse(OkHttp.string(siteUrl + type.getTypeId() + "/all/all/all", getHeader())).select("#__NEXT_DATA__").get(0);
Element script = Jsoup.parse(OkHttp.string(proxy(), siteUrl + type.getTypeId() + "/all/all/all", getHeader())).select("#__NEXT_DATA__").get(0);
JSONObject obj = new JSONObject(script.data()).getJSONObject("props").getJSONObject("pageProps").getJSONObject("filterCondition");
Condition item = Condition.objectFrom(obj.toString());
filters.put(type.getTypeId(), item.getFilter());
@ -76,7 +76,7 @@ public class Star extends Spider {
@Override
public String homeVideoContent() throws Exception {
List<Vod> list = new ArrayList<>();
Element script = Jsoup.parse(OkHttp.string(siteUrl)).select("#__NEXT_DATA__").get(0);
Element script = Jsoup.parse(OkHttp.string(proxy(), siteUrl)).select("#__NEXT_DATA__").get(0);
List<Card> cards = Card.arrayFrom(new JSONObject(script.data()).getJSONObject("props").getJSONObject("pageProps").getJSONArray("cards").toString());
for (Card card : cards) if (!card.getName().equals("电视直播")) for (Card item : card.getCards()) list.add(item.vod());
return Result.string(list);
@ -94,7 +94,7 @@ public class Star extends Spider {
if (year.length() > 0) query.setYear(year);
if (type.length() > 0) query.setLabel(type);
if (area.length() > 0) query.setCountry(area);
String body = OkHttp.postJson(apiUrl, query.toString()).getBody();
String body = OkHttp.post(proxy(), apiUrl, query.toString());
List<Card> cards = Card.arrayFrom(new JSONObject(body).getJSONObject("data").getJSONArray("list").toString());
List<Vod> list = new ArrayList<>();
for (Card card : cards) list.add(card.vod());
@ -103,7 +103,7 @@ public class Star extends Spider {
@Override
public String detailContent(List<String> ids) throws Exception {
Element script = Jsoup.parse(OkHttp.string(detail.concat(ids.get(0)), getHeader())).select("#__NEXT_DATA__").get(0);
Element script = Jsoup.parse(OkHttp.string(proxy(), detail.concat(ids.get(0)), getHeader())).select("#__NEXT_DATA__").get(0);
Detail detail = Detail.objectFrom(new JSONObject(script.data()).getJSONObject("props").getJSONObject("pageProps").getJSONObject("pageData").toString());
Vod vod = new Vod();
vod.setVodId(ids.get(0));
@ -126,7 +126,7 @@ public class Star extends Spider {
@Override
public String searchContent(String key, boolean quick) throws Exception {
List<Vod> list = new ArrayList<>();
String json = OkHttp.string(siteUrl + data + ver + "/search.json?word=" + URLEncoder.encode(key), getHeader());
String json = OkHttp.string(proxy(), siteUrl + data + ver + "/search.json?word=" + URLEncoder.encode(key), getHeader());
List<Card> items = Card.arrayFrom(new JSONObject(json).getJSONObject("pageProps").getJSONArray("initList").toString());
for (Card item : items) list.add(item.vod());
return Result.string(list);

4
app/src/main/java/com/github/catvod/spider/XPath.java
View File

@ -324,7 +324,7 @@ public class XPath extends Spider {
if (rule == null) {
if (ext != null) {
if (ext.startsWith("http")) {
String json = OkHttp.string(ext, null);
String json = OkHttp.string(proxy(), ext, null);
rule = Rule.fromJson(json);
loadRuleExt(json);
} else {
@ -340,6 +340,6 @@ public class XPath extends Spider {
protected String fetch(String webUrl) {
SpiderDebug.log(webUrl);
return OkHttp.string(webUrl, getHeaders());
return OkHttp.string(proxy(), webUrl, getHeaders());
}
}

14
app/src/main/java/com/github/catvod/spider/XiaoZhiTiao.java
View File

@ -12,7 +12,11 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* XiaoZhiTiao
@ -46,7 +50,7 @@ public class XiaoZhiTiao extends Ali {
params.put("from", "web");
params.put("keyword", key);
params.put("token", getToken());
JSONObject object = new JSONObject(OkHttp.post(siteUrl, params, getHeaders()));
JSONObject object = new JSONObject(OkHttp.post(siteUrl, params, getHeaders()).getBody());
if (!object.optBoolean("success")) return "";
List<Vod> list = new ArrayList<>();
JSONArray data = object.getJSONArray("data");
@ -65,9 +69,9 @@ public class XiaoZhiTiao extends Ali {
Map<String, String> params = new HashMap<>();
params.put("action", "get_token");
params.put("from", "web");
JSONObject jsonObject = new JSONObject(OkHttp.post(siteUrl, params, getHeaders()));
if (jsonObject.getBoolean("success")) {
token = jsonObject.getString("data");
JSONObject object = new JSONObject(OkHttp.post(siteUrl, params, getHeaders()).getBody());
if (object.getBoolean("success")) {
token = object.getString("data");
date = new Date(System.currentTimeMillis() + 24 * 60 * 60 * 1000);
}
}

3
app/src/main/java/com/github/catvod/spider/Zhaozy.java
View File

@ -44,9 +44,8 @@ public class Zhaozy extends Ali {
headers.put("User-Agent", Utils.CHROME);
headers.put("Referer", siteUrl + "stop.html");
headers.put("Origin", siteUrl);
Map<String, List<String>> resp = new HashMap<>();
OkHttp.post(siteUrl + "logiu.html", params, headers, resp);
StringBuilder sb = new StringBuilder();
Map<String, List<String>> resp = OkHttp.post(siteUrl + "logiu.html", params, headers).getResp();
for (String item : resp.get("set-cookie")) sb.append(item.split(";")[0]).append(";");
return sb.toString();
}