dongge
/
CatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean code

This commit is contained in:
FongMi 2022-08-30 23:03:08 +08:00
parent 3da80b13bd
commit feaecc88c1
5 changed files with 165 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/build.gradle
View File

@ -33,7 +33,7 @@ android {
dependencies {
implementation 'com.squareup.okhttp3:okhttp:4.10.0'
implementation 'com.google.code.gson:gson:2.9.0'
implementation 'cn.wanghaomiao:JsoupXpath:2.5.0'
implementation 'org.jsoup:jsoup:1.14.3'
implementation 'com.google.code.gson:gson:2.9.1'
implementation 'cn.wanghaomiao:JsoupXpath:2.5.1'
implementation 'org.jsoup:jsoup:1.15.3'
}

14
app/src/main/java/com/github/catvod/crawler/Spider.java
View File

@ -2,6 +2,8 @@ package com.github.catvod.crawler;
import android.content.Context;
import org.json.JSONException;
import java.util.HashMap;
import java.util.List;
@ -14,27 +16,27 @@ public abstract class Spider {
init(context);
}
public String homeContent(boolean filter) {
public String homeContent(boolean filter) throws JSONException {
return "";
}
public String homeVideoContent() {
public String homeVideoContent() throws JSONException {
return "";
}
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws JSONException {
return "";
}
public String detailContent(List<String> ids) {
public String detailContent(List<String> ids) throws JSONException {
return "";
}
public String searchContent(String key, boolean quick) {
public String searchContent(String key, boolean quick) throws JSONException {
return "";
}
public String playerContent(String flag, String id, List<String> vipFlags) {
public String playerContent(String flag, String id, List<String> vipFlags) throws JSONException {
return "";
}
}

124
app/src/main/java/com/github/catvod/spider/Biubiu.java
View File

@ -7,10 +7,12 @@ import com.github.catvod.bean.Class;
import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttpUtil;
import com.github.catvod.utils.Misc;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.util.ArrayList;
@ -21,7 +23,6 @@ import java.util.regex.Pattern;
public class Biubiu extends Spider {
private String ext = null;
private JSONObject rule = null;
private HashMap<String, String> getHeaders() {
@ -32,15 +33,23 @@ public class Biubiu extends Spider {
return headers;
}
private void fetchRule(String ext) {
try {
if (ext.startsWith("http")) rule = new JSONObject(OkHttpUtil.string(ext, null));
else rule = new JSONObject(ext);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
@Override
public void init(Context context, String extend) {
super.init(context, extend);
this.ext = extend;
fetchRule(extend);
}
@Override
public String homeContent(boolean filter) {
fetchRule();
List<Class> classes = new ArrayList<>();
String[] fenleis = getRuleVal("fenlei", "").split("#");
for (String fenlei : fenleis) {
@ -52,7 +61,6 @@ public class Biubiu extends Spider {
@Override
public String homeVideoContent() {
fetchRule();
if (getRuleVal("shouye").equals("1")) return "";
List<Vod> videos = new ArrayList<>();
String[] fenleis = getRuleVal("fenlei", "").split("#");
@ -68,7 +76,6 @@ public class Biubiu extends Spider {
}
private Result category(String tid, String pg) {
fetchRule();
String webUrl = getRuleVal("url") + tid + pg + getRuleVal("houzhui");
String html = fetch(webUrl);
String parseContent = html;
@ -108,7 +115,6 @@ public class Biubiu extends Spider {
@Override
public String detailContent(List<String> ids) {
fetchRule();
String[] idInfo = ids.get(0).split("\\$\\$\\$");
String webUrl = getRuleVal("url") + idInfo[2];
String html = fetch(webUrl);
@ -158,80 +164,62 @@ public class Biubiu extends Spider {
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = getRuleVal("url") + id;
return Result.get().parse().url(webUrl).string();
}
@Override
public String searchContent(String key, boolean quick) {
try {
fetchRule();
boolean ssmoshiJson = getRuleVal("ssmoshi").equals("0");
String webUrlTmp = getRuleVal("url") + getRuleVal("sousuoqian") + key + getRuleVal("sousuohou");
String webUrl = webUrlTmp.split(";")[0];
String webContent = webUrlTmp.contains(";post") ? fetchPost(webUrl) : fetch(webUrl);
List<Vod> videos = new ArrayList<>();
if (ssmoshiJson) {
JSONObject data = new JSONObject(webContent);
JSONArray vodArray = data.getJSONArray("list");
for (int j = 0; j < vodArray.length(); j++) {
JSONObject vod = vodArray.getJSONObject(j);
String name = vod.optString(getRuleVal("jsname")).trim();
String id = vod.optString(getRuleVal("jsid")).trim();
String pic = vod.optString(getRuleVal("jspic")).trim();
public String searchContent(String key, boolean quick) throws JSONException {
boolean ssmoshiJson = getRuleVal("ssmoshi").equals("0");
String webUrlTmp = getRuleVal("url") + getRuleVal("sousuoqian") + key + getRuleVal("sousuohou");
String webUrl = webUrlTmp.split(";")[0];
String webContent = webUrlTmp.contains(";post") ? fetchPost(webUrl) : fetch(webUrl);
List<Vod> videos = new ArrayList<>();
if (ssmoshiJson) {
JSONObject data = new JSONObject(webContent);
JSONArray vodArray = data.getJSONArray("list");
for (int j = 0; j < vodArray.length(); j++) {
JSONObject vod = vodArray.getJSONObject(j);
String name = vod.optString(getRuleVal("jsname")).trim();
String id = vod.optString(getRuleVal("jsid")).trim();
String pic = vod.optString(getRuleVal("jspic")).trim();
pic = Misc.fixUrl(webUrl, pic);
Vod video = new Vod();
video.setVodId(name + "$$$" + pic + "$$$" + getRuleVal("sousuohouzhui") + id);
video.setVodName(name);
video.setVodPic(pic);
videos.add(video);
}
} else {
String parseContent = webContent;
boolean shifouercijiequ = getRuleVal("sousuoshifouercijiequ").equals("1");
if (shifouercijiequ) {
String jiequqian = getRuleVal("ssjiequqian");
String jiequhou = getRuleVal("ssjiequhou");
parseContent = subContent(webContent, jiequqian, jiequhou).get(0);
}
String jiequshuzuqian = getRuleVal("ssjiequshuzuqian");
String jiequshuzuhou = getRuleVal("ssjiequshuzuhou");
ArrayList<String> jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou);
for (int i = 0; i < jiequContents.size(); i++) {
try {
String jiequContent = jiequContents.get(i);
String title = subContent(jiequContent, getRuleVal("ssbiaotiqian"), getRuleVal("ssbiaotihou")).get(0);
String pic = subContent(jiequContent, getRuleVal("sstupianqian"), getRuleVal("sstupianhou")).get(0);
pic = Misc.fixUrl(webUrl, pic);
String link = subContent(jiequContent, getRuleVal("sslianjieqian"), getRuleVal("sslianjiehou")).get(0);
Vod video = new Vod();
video.setVodId(name + "$$$" + pic + "$$$" + getRuleVal("sousuohouzhui") + id);
video.setVodName(name);
video.setVodId(title + "$$$" + pic + "$$$" + link);
video.setVodName(title);
video.setVodPic(pic);
videos.add(video);
}
} else {
String parseContent = webContent;
boolean shifouercijiequ = getRuleVal("sousuoshifouercijiequ").equals("1");
if (shifouercijiequ) {
String jiequqian = getRuleVal("ssjiequqian");
String jiequhou = getRuleVal("ssjiequhou");
parseContent = subContent(webContent, jiequqian, jiequhou).get(0);
}
String jiequshuzuqian = getRuleVal("ssjiequshuzuqian");
String jiequshuzuhou = getRuleVal("ssjiequshuzuhou");
ArrayList<String> jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou);
for (int i = 0; i < jiequContents.size(); i++) {
try {
String jiequContent = jiequContents.get(i);
String title = subContent(jiequContent, getRuleVal("ssbiaotiqian"), getRuleVal("ssbiaotihou")).get(0);
String pic = subContent(jiequContent, getRuleVal("sstupianqian"), getRuleVal("sstupianhou")).get(0);
pic = Misc.fixUrl(webUrl, pic);
String link = subContent(jiequContent, getRuleVal("sslianjieqian"), getRuleVal("sslianjiehou")).get(0);
Vod video = new Vod();
video.setVodId(title + "$$$" + pic + "$$$" + link);
video.setVodName(title);
video.setVodPic(pic);
videos.add(video);
} catch (Exception e) {
e.printStackTrace();
break;
}
} catch (Exception e) {
e.printStackTrace();
break;
}
}
return Result.string(videos);
} catch (Exception e) {
e.printStackTrace();
return "";
}
}
private void fetchRule() {
try {
if (rule != null) return;
if (ext == null) return;
if (ext.startsWith("http")) rule = new JSONObject(OkHttpUtil.string(ext, null));
else rule = new JSONObject(ext);
} catch (Exception e) {
e.printStackTrace();
}
return Result.string(videos);
}
private String fetch(String webUrl) {

154
app/src/main/java/com/github/catvod/spider/XPath.java
View File

@ -13,6 +13,7 @@ import com.github.catvod.utils.Misc;
import com.github.catvod.xpath.XPathRule;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;
@ -25,24 +26,33 @@ import java.util.Set;
public class XPath extends Spider {
protected String ext = null;
protected XPathRule rule = null;
protected HashMap<String, String> getHeaders() {
private HashMap<String, String> getHeaders() {
HashMap<String, String> headers = new HashMap<>();
headers.put("User-Agent", rule.getUa().isEmpty() ? Misc.CHROME : rule.getUa());
return headers;
}
@Override
public void init(Context context, String extend) {
super.init(context, extend);
this.ext = extend;
private void fetchRule(String ext) {
if (ext.startsWith("http")) {
String json = OkHttpUtil.string(ext);
rule = XPathRule.fromJson(json);
loadRuleExt(json);
} else {
rule = XPathRule.fromJson(ext);
loadRuleExt(ext);
}
}
@Override
public String homeContent(boolean filter) {
fetchRule();
public void init(Context context, String extend) {
super.init(context, extend);
fetchRule(extend);
}
@Override
public String homeContent(boolean filter) throws JSONException {
List<Vod> videos = new ArrayList<>();
List<Class> classes = new ArrayList<>();
if (rule.getCateManual().size() > 0) {
@ -86,19 +96,12 @@ public class XPath extends Spider {
return Result.string(classes, videos, rule.getFilter());
}
@Override
public String homeVideoContent() {
fetchRule();
return "";
}
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
return rule.getCateUrl().replace("{cateId}", tid).replace("{catePg}", pg);
}
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
fetchRule();
String webUrl = categoryUrl(tid, pg, filter, extend);
List<Vod> videos = new ArrayList<>();
JXDocument doc = JXDocument.create(fetch(webUrl));
@ -125,12 +128,8 @@ public class XPath extends Spider {
return Result.string(videos);
}
protected void detailContentExt(String content, Vod vod) {
}
@Override
public String detailContent(List<String> ids) {
fetchRule();
public String detailContent(List<String> ids) throws JSONException {
String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0));
String webContent = fetch(webUrl);
JXDocument doc = JXDocument.create(webContent);
@ -254,90 +253,69 @@ public class XPath extends Spider {
String vod_play_url = TextUtils.join("$$$", playList);
vod.setVodPlayFrom(vod_play_from);
vod.setVodPlayUrl(vod_play_url);
detailContentExt(webContent, vod);
return Result.string(vod);
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
SpiderDebug.log(webUrl);
return Result.get().parse().url(webUrl).toString();
}
@Override
public String searchContent(String key, boolean quick) {
try {
fetchRule();
if (rule.getSearchUrl().isEmpty()) return "";
String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key));
String webContent = fetch(webUrl);
List<Vod> videos = new ArrayList<>();
if (rule.getSearchVodNode().startsWith("json:")) {
String[] node = rule.getSearchVodNode().substring(5).split(">");
JSONObject data = new JSONObject(webContent);
for (int i = 0; i < node.length; i++) {
if (i == node.length - 1) {
JSONArray vodArray = data.getJSONArray(node[i]);
for (int j = 0; j < vodArray.length(); j++) {
JSONObject vod = vodArray.getJSONObject(j);
String name = vod.optString(rule.getSearchVodName()).trim();
name = rule.getSearchVodNameR(name);
String id = vod.optString(rule.getSearchVodId()).trim();
id = rule.getSearchVodIdR(id);
String pic = vod.optString(rule.getSearchVodImg()).trim();
pic = rule.getSearchVodImgR(pic);
pic = Misc.fixUrl(webUrl, pic);
String mark = vod.optString(rule.getSearchVodMark()).trim();
mark = rule.getSearchVodMarkR(mark);
videos.add(new Vod(id, name, pic, mark));
}
} else {
data = data.getJSONObject(node[i]);
public String searchContent(String key, boolean quick) throws JSONException {
if (rule.getSearchUrl().isEmpty()) return "";
String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key));
String webContent = fetch(webUrl);
List<Vod> videos = new ArrayList<>();
if (rule.getSearchVodNode().startsWith("json:")) {
String[] node = rule.getSearchVodNode().substring(5).split(">");
JSONObject data = new JSONObject(webContent);
for (int i = 0; i < node.length; i++) {
if (i == node.length - 1) {
JSONArray vodArray = data.getJSONArray(node[i]);
for (int j = 0; j < vodArray.length(); j++) {
JSONObject vod = vodArray.getJSONObject(j);
String name = vod.optString(rule.getSearchVodName()).trim();
name = rule.getSearchVodNameR(name);
String id = vod.optString(rule.getSearchVodId()).trim();
id = rule.getSearchVodIdR(id);
String pic = vod.optString(rule.getSearchVodImg()).trim();
pic = rule.getSearchVodImgR(pic);
pic = Misc.fixUrl(webUrl, pic);
String mark = vod.optString(rule.getSearchVodMark()).trim();
mark = rule.getSearchVodMarkR(mark);
videos.add(new Vod(id, name, pic, mark));
}
}
} else {
JXDocument doc = JXDocument.create(webContent);
List<JXNode> vodNodes = doc.selN(rule.getSearchVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
String name = vodNodes.get(i).selOne(rule.getSearchVodName()).asString().trim();
name = rule.getSearchVodNameR(name);
String id = vodNodes.get(i).selOne(rule.getSearchVodId()).asString().trim();
id = rule.getSearchVodIdR(id);
String pic = vodNodes.get(i).selOne(rule.getSearchVodImg()).asString().trim();
pic = rule.getSearchVodImgR(pic);
pic = Misc.fixUrl(webUrl, pic);
String mark = "";
if (!rule.getCateVodMark().isEmpty()) {
try {
mark = vodNodes.get(i).selOne(rule.getSearchVodMark()).asString().trim();
mark = rule.getSearchVodMarkR(mark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
videos.add(new Vod(id, name, pic, mark));
} else {
data = data.getJSONObject(node[i]);
}
}
return Result.string(videos);
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
}
protected void fetchRule() {
if (rule == null && ext != null) {
if (ext.startsWith("http")) {
String json = OkHttpUtil.string(ext);
rule = XPathRule.fromJson(json);
loadRuleExt(json);
} else {
rule = XPathRule.fromJson(ext);
loadRuleExt(ext);
} else {
JXDocument doc = JXDocument.create(webContent);
List<JXNode> vodNodes = doc.selN(rule.getSearchVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
String name = vodNodes.get(i).selOne(rule.getSearchVodName()).asString().trim();
name = rule.getSearchVodNameR(name);
String id = vodNodes.get(i).selOne(rule.getSearchVodId()).asString().trim();
id = rule.getSearchVodIdR(id);
String pic = vodNodes.get(i).selOne(rule.getSearchVodImg()).asString().trim();
pic = rule.getSearchVodImgR(pic);
pic = Misc.fixUrl(webUrl, pic);
String mark = "";
if (!rule.getCateVodMark().isEmpty()) {
try {
mark = vodNodes.get(i).selOne(rule.getSearchVodMark()).asString().trim();
mark = rule.getSearchVodMarkR(mark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
videos.add(new Vod(id, name, pic, mark));
}
}
return Result.string(videos);
}
protected void loadRuleExt(String json) {

79
app/src/main/java/com/github/catvod/spider/XPathMac.java
View File

@ -48,61 +48,46 @@ public class XPathMac extends XPath {
}
playerConfigJs = jsonObj.optString("pCfgJs").trim();
playerConfigJsRegex = jsonObj.optString("pCfgJsR", playerConfigJsRegex).trim();
} catch (JSONException e) {
SpiderDebug.log(e);
}
}
@Override
public String homeContent(boolean filter) {
String result = super.homeContent(filter);
if (result.length() > 0 && playerConfigJs.length() > 0) { //嘗試通過playerConfigJs獲取展示和flag匹配關系
String webContent = fetch(playerConfigJs);
Matcher matcher = Pattern.compile(playerConfigJsRegex).matcher(webContent);
if (!matcher.find()) return result;
try {
JSONObject jsonObject = new JSONObject(matcher.group(1));
Iterator<String> keys = jsonObject.keys();
while (keys.hasNext()) {
String key = keys.next();
JSONObject keyObj = jsonObject.optJSONObject(key);
if (keyObj == null) continue;
String show = keyObj.optString("show").trim();
if (show.isEmpty()) continue;
show2VipFlag.put(show, key);
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
return result;
}
@Override
public String detailContent(List<String> ids) {
String result = super.detailContent(ids);
if (!decodeVipFlag || result.isEmpty()) return result;
try {
JSONObject jsonObject = new JSONObject(result);
String[] playFrom = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$");
if (playFrom.length > 0) {
for (int i = 0; i < playFrom.length; i++) {
if (show2VipFlag.containsKey(playFrom[i])) {
playFrom[i] = show2VipFlag.get(playFrom[i]);
}
}
jsonObject.optJSONArray("list").getJSONObject(0).put("vod_play_from", TextUtils.join("$$$", playFrom));
result = jsonObject.toString();
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
@Override
public String homeContent(boolean filter) throws JSONException {
String result = super.homeContent(filter);
if (result.isEmpty() || playerConfigJs.isEmpty()) return result;
//嘗試通過playerConfigJs獲取展示和flag匹配關系
String webContent = fetch(playerConfigJs);
Matcher matcher = Pattern.compile(playerConfigJsRegex).matcher(webContent);
if (!matcher.find()) return result;
JSONObject jsonObject = new JSONObject(matcher.group(1));
Iterator<String> keys = jsonObject.keys();
while (keys.hasNext()) {
String key = keys.next();
JSONObject keyObj = jsonObject.optJSONObject(key);
if (keyObj == null) continue;
String show = keyObj.optString("show").trim();
if (show.isEmpty()) continue;
show2VipFlag.put(show, key);
}
return result;
}
@Override
public String detailContent(List<String> ids) throws JSONException {
String result = super.detailContent(ids);
if (!decodeVipFlag || result.isEmpty()) return result;
JSONObject jsonObject = new JSONObject(result);
String[] playFrom = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$");
if (playFrom.length == 0) return result;
for (int i = 0; i < playFrom.length; i++) if (show2VipFlag.containsKey(playFrom[i])) playFrom[i] = show2VipFlag.get(playFrom[i]);
jsonObject.optJSONArray("list").getJSONObject(0).put("vod_play_from", TextUtils.join("$$$", playFrom));
return jsonObject.toString();
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
String videoUrl = null;
// 嘗試分析直連