dongge
/
AndroidCatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean code

This commit is contained in:
FongMi 2022-08-30 23:03:08 +08:00
parent 3da80b13bd
commit feaecc88c1
5 changed files with 165 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/build.gradle
View File

@ -33,7 +33,7 @@ android {
dependencies {
implementation 'com.squareup.okhttp3:okhttp:4.10.0'
implementation 'com.google.code.gson:gson:2.9.0'
implementation 'cn.wanghaomiao:JsoupXpath:2.5.0'
implementation 'org.jsoup:jsoup:1.14.3'
implementation 'com.google.code.gson:gson:2.9.1'
implementation 'cn.wanghaomiao:JsoupXpath:2.5.1'
implementation 'org.jsoup:jsoup:1.15.3'
}

14
app/src/main/java/com/github/catvod/crawler/Spider.java
View File

@ -2,6 +2,8 @@ package com.github.catvod.crawler;
import android.content.Context;
import org.json.JSONException;
import java.util.HashMap;
import java.util.List;
@ -14,27 +16,27 @@ public abstract class Spider {
init(context);
}
public String homeContent(boolean filter) {
public String homeContent(boolean filter) throws JSONException {
return "";
}
public String homeVideoContent() {
public String homeVideoContent() throws JSONException {
return "";
}
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws JSONException {
return "";
}
public String detailContent(List<String> ids) {
public String detailContent(List<String> ids) throws JSONException {
return "";
}
public String searchContent(String key, boolean quick) {
public String searchContent(String key, boolean quick) throws JSONException {
return "";
}
public String playerContent(String flag, String id, List<String> vipFlags) {
public String playerContent(String flag, String id, List<String> vipFlags) throws JSONException {
return "";
}
}

38
app/src/main/java/com/github/catvod/spider/Biubiu.java
View File

@ -7,10 +7,12 @@ import com.github.catvod.bean.Class;
import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttpUtil;
import com.github.catvod.utils.Misc;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.util.ArrayList;
@ -21,7 +23,6 @@ import java.util.regex.Pattern;
public class Biubiu extends Spider {
private String ext = null;
private JSONObject rule = null;
private HashMap<String, String> getHeaders() {
@ -32,15 +33,23 @@ public class Biubiu extends Spider {
return headers;
}
private void fetchRule(String ext) {
try {
if (ext.startsWith("http")) rule = new JSONObject(OkHttpUtil.string(ext, null));
else rule = new JSONObject(ext);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
@Override
public void init(Context context, String extend) {
super.init(context, extend);
this.ext = extend;
fetchRule(extend);
}
@Override
public String homeContent(boolean filter) {
fetchRule();
List<Class> classes = new ArrayList<>();
String[] fenleis = getRuleVal("fenlei", "").split("#");
for (String fenlei : fenleis) {
@ -52,7 +61,6 @@ public class Biubiu extends Spider {
@Override
public String homeVideoContent() {
fetchRule();
if (getRuleVal("shouye").equals("1")) return "";
List<Vod> videos = new ArrayList<>();
String[] fenleis = getRuleVal("fenlei", "").split("#");
@ -68,7 +76,6 @@ public class Biubiu extends Spider {
}
private Result category(String tid, String pg) {
fetchRule();
String webUrl = getRuleVal("url") + tid + pg + getRuleVal("houzhui");
String html = fetch(webUrl);
String parseContent = html;
@ -108,7 +115,6 @@ public class Biubiu extends Spider {
@Override
public String detailContent(List<String> ids) {
fetchRule();
String[] idInfo = ids.get(0).split("\\$\\$\\$");
String webUrl = getRuleVal("url") + idInfo[2];
String html = fetch(webUrl);
@ -158,15 +164,12 @@ public class Biubiu extends Spider {
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = getRuleVal("url") + id;
return Result.get().parse().url(webUrl).string();
}
@Override
public String searchContent(String key, boolean quick) {
try {
fetchRule();
public String searchContent(String key, boolean quick) throws JSONException {
boolean ssmoshiJson = getRuleVal("ssmoshi").equals("0");
String webUrlTmp = getRuleVal("url") + getRuleVal("sousuoqian") + key + getRuleVal("sousuohou");
String webUrl = webUrlTmp.split(";")[0];
@ -217,21 +220,6 @@ public class Biubiu extends Spider {
}
}
return Result.string(videos);
} catch (Exception e) {
e.printStackTrace();
return "";
}
}
private void fetchRule() {
try {
if (rule != null) return;
if (ext == null) return;
if (ext.startsWith("http")) rule = new JSONObject(OkHttpUtil.string(ext, null));
else rule = new JSONObject(ext);
} catch (Exception e) {
e.printStackTrace();
}
}
private String fetch(String webUrl) {

62
app/src/main/java/com/github/catvod/spider/XPath.java
View File

@ -13,6 +13,7 @@ import com.github.catvod.utils.Misc;
import com.github.catvod.xpath.XPathRule;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;
@ -25,24 +26,33 @@ import java.util.Set;
public class XPath extends Spider {
protected String ext = null;
protected XPathRule rule = null;
protected HashMap<String, String> getHeaders() {
private HashMap<String, String> getHeaders() {
HashMap<String, String> headers = new HashMap<>();
headers.put("User-Agent", rule.getUa().isEmpty() ? Misc.CHROME : rule.getUa());
return headers;
}
@Override
public void init(Context context, String extend) {
super.init(context, extend);
this.ext = extend;
private void fetchRule(String ext) {
if (ext.startsWith("http")) {
String json = OkHttpUtil.string(ext);
rule = XPathRule.fromJson(json);
loadRuleExt(json);
} else {
rule = XPathRule.fromJson(ext);
loadRuleExt(ext);
}
}
@Override
public String homeContent(boolean filter) {
fetchRule();
public void init(Context context, String extend) {
super.init(context, extend);
fetchRule(extend);
}
@Override
public String homeContent(boolean filter) throws JSONException {
List<Vod> videos = new ArrayList<>();
List<Class> classes = new ArrayList<>();
if (rule.getCateManual().size() > 0) {
@ -86,19 +96,12 @@ public class XPath extends Spider {
return Result.string(classes, videos, rule.getFilter());
}
@Override
public String homeVideoContent() {
fetchRule();
return "";
}
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
return rule.getCateUrl().replace("{cateId}", tid).replace("{catePg}", pg);
}
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
fetchRule();
String webUrl = categoryUrl(tid, pg, filter, extend);
List<Vod> videos = new ArrayList<>();
JXDocument doc = JXDocument.create(fetch(webUrl));
@ -125,12 +128,8 @@ public class XPath extends Spider {
return Result.string(videos);
}
protected void detailContentExt(String content, Vod vod) {
}
@Override
public String detailContent(List<String> ids) {
fetchRule();
public String detailContent(List<String> ids) throws JSONException {
String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0));
String webContent = fetch(webUrl);
JXDocument doc = JXDocument.create(webContent);
@ -254,22 +253,18 @@ public class XPath extends Spider {
String vod_play_url = TextUtils.join("$$$", playList);
vod.setVodPlayFrom(vod_play_from);
vod.setVodPlayUrl(vod_play_url);
detailContentExt(webContent, vod);
return Result.string(vod);
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
SpiderDebug.log(webUrl);
return Result.get().parse().url(webUrl).toString();
}
@Override
public String searchContent(String key, boolean quick) {
try {
fetchRule();
public String searchContent(String key, boolean quick) throws JSONException {
if (rule.getSearchUrl().isEmpty()) return "";
String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key));
String webContent = fetch(webUrl);
@ -321,23 +316,6 @@ public class XPath extends Spider {
}
}
return Result.string(videos);
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
}
protected void fetchRule() {
if (rule == null && ext != null) {
if (ext.startsWith("http")) {
String json = OkHttpUtil.string(ext);
rule = XPathRule.fromJson(json);
loadRuleExt(json);
} else {
rule = XPathRule.fromJson(ext);
loadRuleExt(ext);
}
}
}
protected void loadRuleExt(String json) {

31
app/src/main/java/com/github/catvod/spider/XPathMac.java
View File

@ -48,19 +48,19 @@ public class XPathMac extends XPath {
}
playerConfigJs = jsonObj.optString("pCfgJs").trim();
playerConfigJsRegex = jsonObj.optString("pCfgJsR", playerConfigJsRegex).trim();
} catch (JSONException e) {
} catch (Exception e) {
SpiderDebug.log(e);
}
}
@Override
public String homeContent(boolean filter) {
public String homeContent(boolean filter) throws JSONException {
String result = super.homeContent(filter);
if (result.length() > 0 && playerConfigJs.length() > 0) { //嘗試通過playerConfigJs獲取展示和flag匹配關系
if (result.isEmpty() || playerConfigJs.isEmpty()) return result;
//嘗試通過playerConfigJs獲取展示和flag匹配關系
String webContent = fetch(playerConfigJs);
Matcher matcher = Pattern.compile(playerConfigJsRegex).matcher(webContent);
if (!matcher.find()) return result;
try {
JSONObject jsonObject = new JSONObject(matcher.group(1));
Iterator<String> keys = jsonObject.keys();
while (keys.hasNext()) {
@ -71,38 +71,23 @@ public class XPathMac extends XPath {
if (show.isEmpty()) continue;
show2VipFlag.put(show, key);
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
return result;
}
@Override
public String detailContent(List<String> ids) {
public String detailContent(List<String> ids) throws JSONException {
String result = super.detailContent(ids);
if (!decodeVipFlag || result.isEmpty()) return result;
try {
JSONObject jsonObject = new JSONObject(result);
String[] playFrom = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$");
if (playFrom.length > 0) {
for (int i = 0; i < playFrom.length; i++) {
if (show2VipFlag.containsKey(playFrom[i])) {
playFrom[i] = show2VipFlag.get(playFrom[i]);
}
}
if (playFrom.length == 0) return result;
for (int i = 0; i < playFrom.length; i++) if (show2VipFlag.containsKey(playFrom[i])) playFrom[i] = show2VipFlag.get(playFrom[i]);
jsonObject.optJSONArray("list").getJSONObject(0).put("vod_play_from", TextUtils.join("$$$", playFrom));
result = jsonObject.toString();
}
} catch (Exception e) {
SpiderDebug.log(e);
}
return result;
return jsonObject.toString();
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
String videoUrl = null;
// 嘗試分析直連