dongge
/
AndroidCatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add BiuBiu and XPath

This commit is contained in:
FongMi 2022-08-22 20:56:33 +08:00
parent ce3aa70ea3
commit b6f335259e
9 changed files with 732 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/src/main/java/com/github/catvod/bean/Result.java
View File

@ -6,6 +6,7 @@ import com.google.gson.annotations.SerializedName;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.json.JSONObject; import org.json.JSONObject;
import java.util.Collections;
import java.util.List; import java.util.List;
public class Result { public class Result {
@ -31,6 +32,10 @@ public class Result {
this.list = list; this.list = list;
} }
public List<Vod> getList() {
return list == null ? Collections.emptyList() : list;
}
public void setFilters(JSONObject filters) { public void setFilters(JSONObject filters) {
this.filters = filters; this.filters = filters;
} }

4
app/src/main/java/com/github/catvod/crawler/SpiderDebug.java
View File

@ -6,8 +6,8 @@ public class SpiderDebug {
private static final String TAG = SpiderDebug.class.getSimpleName(); private static final String TAG = SpiderDebug.class.getSimpleName();
public static void log(Throwable th) { public static void log(Exception e) {
Log.d(TAG, th.getMessage()); Log.d(TAG, e.getMessage());
} }
public static void log(String msg) { public static void log(String msg) {

6
app/src/main/java/com/github/catvod/live/TxtSubscribe.java
View File

@ -70,7 +70,7 @@ public class TxtSubscribe {
if (!noGroup.isEmpty()) { if (!noGroup.isEmpty()) {
allLives.put("未分類", noGroup); allLives.put("未分類", noGroup);
} }
} catch (Throwable ignored) { } catch (Exception ignored) {
} }
} }
@ -118,8 +118,8 @@ public class TxtSubscribe {
ByteArrayInputStream baos = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)); ByteArrayInputStream baos = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
result[2] = baos; result[2] = baos;
return result; return result;
} catch (Throwable th) { } catch (Exception e) {
th.printStackTrace(); e.printStackTrace();
} }
return null; return null;
} }

4
app/src/main/java/com/github/catvod/parser/JsonParallel.java
View File

@ -55,8 +55,8 @@ public class JsonParallel {
for (int j = 0; j < futures.size(); j++) { for (int j = 0; j < futures.size(); j++) {
try { try {
futures.get(j).cancel(true); futures.get(j).cancel(true);
} catch (Throwable th) { } catch (Exception e) {
SpiderDebug.log(th); SpiderDebug.log(e);
} }
} }
futures.clear(); futures.clear();

276
app/src/main/java/com/github/catvod/spider/Biubiu.java Normal file
View File

@ -0,0 +1,276 @@
package com.github.catvod.spider;
import android.content.Context;
import android.text.TextUtils;
import com.github.catvod.bean.Class;
import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider;
import com.github.catvod.net.OkHttpUtil;
import com.github.catvod.utils.Misc;
import org.json.JSONArray;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Biubiu extends Spider {
private String ext = null;
private JSONObject rule = null;
private HashMap<String, String> getHeaders() {
HashMap<String, String> headers = new HashMap<>();
String ua = getRuleVal("UserW", Misc.CHROME).trim();
if (ua.isEmpty()) ua = Misc.CHROME;
headers.put("User-Agent", ua);
return headers;
}
@Override
public void init(Context context, String extend) {
super.init(context, extend);
this.ext = extend;
}
@Override
public String homeContent(boolean filter) {
fetchRule();
Result result = new Result();
List<Class> classes = new ArrayList<>();
String[] fenleis = getRuleVal("fenlei", "").split("#");
for (String fenlei : fenleis) {
String[] info = fenlei.split("\\$");
classes.add(new Class(info[0], info[1]));
}
result.setClasses(classes);
return result.toString();
}
@Override
public String homeVideoContent() {
fetchRule();
if (getRuleVal("shouye").equals("1")) return "";
List<Vod> videos = new ArrayList<>();
String[] fenleis = getRuleVal("fenlei", "").split("#");
for (String fenlei : fenleis) {
String[] info = fenlei.split("\\$");
Result result = category(info[1], "1");
for (int i = 0; i < result.getList().size(); i++) {
videos.add(result.getList().get(i));
if (videos.size() >= 30) break;
}
}
Result result = new Result();
result.setList(videos);
return result.toString();
}
private Result category(String tid, String pg) {
fetchRule();
String webUrl = getRuleVal("url") + tid + pg + getRuleVal("houzhui");
String html = fetch(webUrl);
String parseContent = html;
boolean shifouercijiequ = getRuleVal("shifouercijiequ").equals("1");
if (shifouercijiequ) {
String jiequqian = getRuleVal("jiequqian");
String jiequhou = getRuleVal("jiequhou");
parseContent = subContent(html, jiequqian, jiequhou).get(0);
}
String jiequshuzuqian = getRuleVal("jiequshuzuqian");
String jiequshuzuhou = getRuleVal("jiequshuzuhou");
List<Vod> videos = new ArrayList<>();
ArrayList<String> jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou);
for (int i = 0; i < jiequContents.size(); i++) {
try {
String jiequContent = jiequContents.get(i);
String title = subContent(jiequContent, getRuleVal("biaotiqian"), getRuleVal("biaotihou")).get(0);
String pic = subContent(jiequContent, getRuleVal("tupianqian"), getRuleVal("tupianhou")).get(0);
pic = Misc.fixUrl(webUrl, pic);
String link = subContent(jiequContent, getRuleVal("lianjieqian"), getRuleVal("lianjiehou")).get(0);
Vod vod = new Vod();
vod.setVodId(title + "$$$" + pic + "$$$" + link);
vod.setVodName(title);
vod.setVodPic(pic);
videos.add(vod);
} catch (Exception e) {
break;
}
}
Result result = new Result();
result.setList(videos);
return result;
}
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
return category(tid, pg).toString();
}
@Override
public String detailContent(List<String> ids) {
fetchRule();
String[] idInfo = ids.get(0).split("\\$\\$\\$");
String webUrl = getRuleVal("url") + idInfo[2];
String html = fetch(webUrl);
String parseContent = html;
boolean bfshifouercijiequ = getRuleVal("bfshifouercijiequ").equals("1");
if (bfshifouercijiequ) {
String jiequqian = getRuleVal("bfjiequqian");
String jiequhou = getRuleVal("bfjiequhou");
parseContent = subContent(html, jiequqian, jiequhou).get(0);
}
List<String> playList = new ArrayList<>();
List<String> playFrom = new ArrayList<>();
String jiequshuzuqian = getRuleVal("bfjiequshuzuqian");
String jiequshuzuhou = getRuleVal("bfjiequshuzuhou");
boolean bfyshifouercijiequ = getRuleVal("bfyshifouercijiequ").equals("1");
ArrayList<String> jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou);
for (int i = 0; i < jiequContents.size(); i++) {
try {
String jiequContent = jiequContents.get(i);
String parseJqContent = bfyshifouercijiequ ? subContent(jiequContent, getRuleVal("bfyjiequqian"), getRuleVal("bfyjiequhou")).get(0) : jiequContent;
ArrayList<String> lastParseContents = subContent(parseJqContent, getRuleVal("bfyjiequshuzuqian"), getRuleVal("bfyjiequshuzuhou"));
List<String> vodItems = new ArrayList<>();
for (int j = 0; j < lastParseContents.size(); j++) {
String title = subContent(lastParseContents.get(j), getRuleVal("bfbiaotiqian"), getRuleVal("bfbiaotihou")).get(0);
String link = subContent(lastParseContents.get(j), getRuleVal("bflianjieqian"), getRuleVal("bflianjiehou")).get(0);
vodItems.add(title + "$" + link);
}
playList.add(TextUtils.join("#", vodItems));
} catch (Exception e) {
break;
}
}
Vod video = new Vod();
video.setVodId(ids.get(0));
video.setVodName(idInfo[0]);
video.setVodPic(idInfo[1]);
for (int i = 0; i < playList.size(); i++) playFrom.add("播放列表" + (i + 1));
video.setVodPlayFrom(TextUtils.join("$$$", playFrom));
video.setVodPlayUrl(TextUtils.join("$$$", playList));
Result result = new Result();
result.setList(Arrays.asList(video));
return result.toString();
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = getRuleVal("url") + id;
Result result = new Result();
result.setParse("1");
result.setUrl(webUrl);
return result.toString();
}
@Override
public String searchContent(String key, boolean quick) {
try {
fetchRule();
boolean ssmoshiJson = getRuleVal("ssmoshi").equals("0");
String webUrlTmp = getRuleVal("url") + getRuleVal("sousuoqian") + key + getRuleVal("sousuohou");
String webUrl = webUrlTmp.split(";")[0];
String webContent = webUrlTmp.contains(";post") ? fetchPost(webUrl) : fetch(webUrl);
List<Vod> videos = new ArrayList<>();
if (ssmoshiJson) {
JSONObject data = new JSONObject(webContent);
JSONArray vodArray = data.getJSONArray("list");
for (int j = 0; j < vodArray.length(); j++) {
JSONObject vod = vodArray.getJSONObject(j);
String name = vod.optString(getRuleVal("jsname")).trim();
String id = vod.optString(getRuleVal("jsid")).trim();
String pic = vod.optString(getRuleVal("jspic")).trim();
pic = Misc.fixUrl(webUrl, pic);
Vod video = new Vod();
video.setVodId(name + "$$$" + pic + "$$$" + getRuleVal("sousuohouzhui") + id);
video.setVodName(name);
video.setVodPic(pic);
videos.add(video);
}
} else {
String parseContent = webContent;
boolean shifouercijiequ = getRuleVal("sousuoshifouercijiequ").equals("1");
if (shifouercijiequ) {
String jiequqian = getRuleVal("ssjiequqian");
String jiequhou = getRuleVal("ssjiequhou");
parseContent = subContent(webContent, jiequqian, jiequhou).get(0);
}
String jiequshuzuqian = getRuleVal("ssjiequshuzuqian");
String jiequshuzuhou = getRuleVal("ssjiequshuzuhou");
ArrayList<String> jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou);
for (int i = 0; i < jiequContents.size(); i++) {
try {
String jiequContent = jiequContents.get(i);
String title = subContent(jiequContent, getRuleVal("ssbiaotiqian"), getRuleVal("ssbiaotihou")).get(0);
String pic = subContent(jiequContent, getRuleVal("sstupianqian"), getRuleVal("sstupianhou")).get(0);
pic = Misc.fixUrl(webUrl, pic);
String link = subContent(jiequContent, getRuleVal("sslianjieqian"), getRuleVal("sslianjiehou")).get(0);
Vod video = new Vod();
video.setVodId(title + "$$$" + pic + "$$$" + link);
video.setVodName(title);
video.setVodPic(pic);
videos.add(video);
} catch (Exception e) {
e.printStackTrace();
break;
}
}
}
Result result = new Result();
result.setList(videos);
return result.toString();
} catch (Exception e) {
e.printStackTrace();
return "";
}
}
private void fetchRule() {
try {
if (rule != null) return;
if (ext == null) return;
if (ext.startsWith("http")) rule = new JSONObject(OkHttpUtil.string(ext, null));
else rule = new JSONObject(ext);
} catch (Exception e) {
e.printStackTrace();
}
}
private String fetch(String webUrl) {
return OkHttpUtil.string(webUrl, getHeaders()).replaceAll("\r|\n", "");
}
private String fetchPost(String webUrl) {
return OkHttpUtil.post(webUrl).replaceAll("\r|\n", "");
}
private String getRuleVal(String key, String defaultVal) {
String v = rule.optString(key);
if (v.isEmpty() || v.equals("空")) return defaultVal;
return v;
}
private String getRuleVal(String key) {
return getRuleVal(key, "");
}
private ArrayList<String> subContent(String content, String startFlag, String endFlag) {
ArrayList<String> result = new ArrayList<>();
Pattern pattern = Pattern.compile(startFlag + "(.*?)" + endFlag);
Matcher matcher = pattern.matcher(content);
while (matcher.find()) result.add(matcher.group(1));
return result;
}
}

449
app/src/main/java/com/github/catvod/spider/XPath.java
View File

@ -43,74 +43,57 @@ public class XPath extends Spider {
@Override @Override
public String homeContent(boolean filter) { public String homeContent(boolean filter) {
try { fetchRule();
fetchRule(); Result result = new Result();
Result result = new Result(); List<Vod> videos = new ArrayList<>();
List<Vod> videos = new ArrayList<>(); List<Class> classes = new ArrayList<>();
List<Class> classes = new ArrayList<>(); if (rule.getCateManual().size() > 0) {
if (rule.getCateManual().size() > 0) { Set<String> keys = rule.getCateManual().keySet();
Set<String> keys = rule.getCateManual().keySet(); for (String k : keys) classes.add(new Class(k, rule.getCateManual().get(k)));
for (String k : keys) classes.add(new Class(k, rule.getCateManual().get(k))); }
String webUrl = rule.getHomeUrl();
JXDocument doc = JXDocument.create(fetch(webUrl));
if (rule.getCateManual().size() == 0) {
List<JXNode> navNodes = doc.selN(rule.getCateNode());
for (int i = 0; i < navNodes.size(); i++) {
String name = navNodes.get(i).selOne(rule.getCateName()).asString().trim();
name = rule.getCateNameR(name);
String id = navNodes.get(i).selOne(rule.getCateId()).asString().trim();
id = rule.getCateIdR(id);
classes.add(new Class(id, name));
} }
try { }
String webUrl = rule.getHomeUrl(); if (!rule.getHomeVodNode().isEmpty()) {
JXDocument doc = JXDocument.create(fetch(webUrl)); List<JXNode> vodNodes = doc.selN(rule.getHomeVodNode());
if (rule.getCateManual().size() == 0) { for (int i = 0; i < vodNodes.size(); i++) {
List<JXNode> navNodes = doc.selN(rule.getCateNode()); String name = vodNodes.get(i).selOne(rule.getHomeVodName()).asString().trim();
for (int i = 0; i < navNodes.size(); i++) { name = rule.getHomeVodNameR(name);
String name = navNodes.get(i).selOne(rule.getCateName()).asString().trim(); String id = vodNodes.get(i).selOne(rule.getHomeVodId()).asString().trim();
name = rule.getCateNameR(name); id = rule.getHomeVodIdR(id);
String id = navNodes.get(i).selOne(rule.getCateId()).asString().trim(); String pic = vodNodes.get(i).selOne(rule.getHomeVodImg()).asString().trim();
id = rule.getCateIdR(id); pic = rule.getHomeVodImgR(pic);
classes.add(new Class(id, name)); pic = Misc.fixUrl(webUrl, pic);
} String mark = "";
} if (!rule.getHomeVodMark().isEmpty()) {
if (!rule.getHomeVodNode().isEmpty()) {
try { try {
List<JXNode> vodNodes = doc.selN(rule.getHomeVodNode()); mark = vodNodes.get(i).selOne(rule.getHomeVodMark()).asString().trim();
for (int i = 0; i < vodNodes.size(); i++) { mark = rule.getHomeVodMarkR(mark);
String name = vodNodes.get(i).selOne(rule.getHomeVodName()).asString().trim();
name = rule.getHomeVodNameR(name);
String id = vodNodes.get(i).selOne(rule.getHomeVodId()).asString().trim();
id = rule.getHomeVodIdR(id);
String pic = vodNodes.get(i).selOne(rule.getHomeVodImg()).asString().trim();
pic = rule.getHomeVodImgR(pic);
pic = Misc.fixUrl(webUrl, pic);
String mark = "";
if (!rule.getHomeVodMark().isEmpty()) {
try {
mark = vodNodes.get(i).selOne(rule.getHomeVodMark()).asString().trim();
mark = rule.getHomeVodMarkR(mark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
videos.add(new Vod(id, name, pic, mark));
}
} catch (Exception e) { } catch (Exception e) {
SpiderDebug.log(e); SpiderDebug.log(e);
} }
} }
} catch (Exception e) { videos.add(new Vod(id, name, pic, mark));
SpiderDebug.log(e);
} }
result.setList(videos);
result.setClasses(classes);
result.setFilters(filter && rule.getFilter() != null ? rule.getFilter() : null);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
} }
result.setList(videos);
result.setClasses(classes);
result.setFilters(filter && rule.getFilter() != null ? rule.getFilter() : null);
return result.toString();
} }
@Override @Override
public String homeVideoContent() { public String homeVideoContent() {
try { fetchRule();
fetchRule();
} catch (Exception e) {
SpiderDebug.log(e);
}
return ""; return "";
} }
@ -120,199 +103,180 @@ public class XPath extends Spider {
@Override @Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) { public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
try { fetchRule();
fetchRule(); String webUrl = categoryUrl(tid, pg, filter, extend);
String webUrl = categoryUrl(tid, pg, filter, extend); List<Vod> videos = new ArrayList<>();
List<Vod> videos = new ArrayList<>(); JXDocument doc = JXDocument.create(fetch(webUrl));
JXDocument doc = JXDocument.create(fetch(webUrl)); List<JXNode> vodNodes = doc.selN(rule.getCateVodNode());
List<JXNode> vodNodes = doc.selN(rule.getCateVodNode()); for (int i = 0; i < vodNodes.size(); i++) {
for (int i = 0; i < vodNodes.size(); i++) { String name = vodNodes.get(i).selOne(rule.getCateVodName()).asString().trim();
String name = vodNodes.get(i).selOne(rule.getCateVodName()).asString().trim(); name = rule.getCateVodNameR(name);
name = rule.getCateVodNameR(name); String id = vodNodes.get(i).selOne(rule.getCateVodId()).asString().trim();
String id = vodNodes.get(i).selOne(rule.getCateVodId()).asString().trim(); id = rule.getCateVodIdR(id);
id = rule.getCateVodIdR(id); String pic = vodNodes.get(i).selOne(rule.getCateVodImg()).asString().trim();
String pic = vodNodes.get(i).selOne(rule.getCateVodImg()).asString().trim(); pic = rule.getCateVodImgR(pic);
pic = rule.getCateVodImgR(pic); pic = Misc.fixUrl(webUrl, pic);
pic = Misc.fixUrl(webUrl, pic); String mark = "";
String mark = ""; if (!rule.getCateVodMark().isEmpty()) {
if (!rule.getCateVodMark().isEmpty()) { try {
try { mark = vodNodes.get(i).selOne(rule.getCateVodMark()).asString().trim();
mark = vodNodes.get(i).selOne(rule.getCateVodMark()).asString().trim(); mark = rule.getCateVodMarkR(mark);
mark = rule.getCateVodMarkR(mark); } catch (Exception e) {
} catch (Exception e) { SpiderDebug.log(e);
SpiderDebug.log(e);
}
} }
videos.add(new Vod(id, name, pic, mark));
} }
Result result = new Result(); videos.add(new Vod(id, name, pic, mark));
result.setList(videos);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
} }
Result result = new Result();
result.setList(videos);
return result.toString();
} }
protected void detailContentExt(String content, Vod vod) { protected void detailContentExt(String content, Vod vod) {
} }
@Override @Override
public String detailContent(List<String> ids) { public String detailContent(List<String> ids) {
try { fetchRule();
fetchRule(); String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0));
String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0)); String webContent = fetch(webUrl);
String webContent = fetch(webUrl); JXDocument doc = JXDocument.create(webContent);
JXDocument doc = JXDocument.create(webContent); JXNode vodNode = doc.selNOne(rule.getDetailNode());
JXNode vodNode = doc.selNOne(rule.getDetailNode()); String cover = "", title = "", desc = "", category = "", area = "", year = "", remark = "", director = "", actor = "";
String cover = "", title = "", desc = "", category = "", area = "", year = "", remark = "", director = "", actor = ""; title = vodNode.selOne(rule.getDetailName()).asString().trim();
title = vodNode.selOne(rule.getDetailName()).asString().trim(); title = rule.getDetailNameR(title);
title = rule.getDetailNameR(title); if (!rule.getDetailImg().isEmpty()) {
if (!rule.getDetailImg().isEmpty()) { try {
try { cover = vodNode.selOne(rule.getDetailImg()).asString().trim();
cover = vodNode.selOne(rule.getDetailImg()).asString().trim(); cover = rule.getDetailImgR(cover);
cover = rule.getDetailImgR(cover); cover = Misc.fixUrl(webUrl, cover);
cover = Misc.fixUrl(webUrl, cover); } catch (Exception e) {
} catch (Exception e) { SpiderDebug.log(e);
SpiderDebug.log(e);
}
} }
if (!rule.getDetailCate().isEmpty()) {
try {
category = vodNode.selOne(rule.getDetailCate()).asString().trim();
category = rule.getDetailCateR(category);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailYear().isEmpty()) {
try {
year = vodNode.selOne(rule.getDetailYear()).asString().trim();
year = rule.getDetailYearR(year);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailArea().isEmpty()) {
try {
area = vodNode.selOne(rule.getDetailArea()).asString().trim();
area = rule.getDetailAreaR(area);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailMark().isEmpty()) {
try {
remark = vodNode.selOne(rule.getDetailMark()).asString().trim();
remark = rule.getDetailMarkR(remark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailActor().isEmpty()) {
try {
actor = vodNode.selOne(rule.getDetailActor()).asString().trim();
actor = rule.getDetailActorR(actor);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailDirector().isEmpty()) {
try {
director = vodNode.selOne(rule.getDetailDirector()).asString().trim();
director = rule.getDetailDirectorR(director);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailDesc().isEmpty()) {
try {
desc = vodNode.selOne(rule.getDetailDesc()).asString().trim();
desc = rule.getDetailDescR(desc);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
Vod vod = new Vod();
vod.setVodId(ids.get(0));
vod.setVodName(title);
vod.setVodPic(cover);
vod.setTypeName(category);
vod.setVodYear(year);
vod.setVodArea(area);
vod.setVodRemarks(remark);
vod.setVodActor(actor);
vod.setVodDirector(director);
vod.setVodContent(desc);
ArrayList<String> playFrom = new ArrayList<>();
List<JXNode> fromNodes = doc.selN(rule.getDetailFromNode());
for (int i = 0; i < fromNodes.size(); i++) {
String name = fromNodes.get(i).selOne(rule.getDetailFromName()).asString().trim();
name = rule.getDetailFromNameR(name);
playFrom.add(name);
}
ArrayList<String> playList = new ArrayList<>();
List<JXNode> urlListNodes = doc.selN(rule.getDetailUrlNode());
for (int i = 0; i < urlListNodes.size(); i++) {
List<JXNode> urlNodes = urlListNodes.get(i).sel(rule.getDetailUrlSubNode());
List<String> vodItems = new ArrayList<>();
for (int j = 0; j < urlNodes.size(); j++) {
String name = urlNodes.get(j).selOne(rule.getDetailUrlName()).asString().trim();
name = rule.getDetailUrlNameR(name);
String id = urlNodes.get(j).selOne(rule.getDetailUrlId()).asString().trim();
id = rule.getDetailUrlIdR(id);
vodItems.add(name + "$" + id);
}
if (vodItems.size() == 0 && playFrom.size() > i) {
playFrom.set(i, "");
}
playList.add(TextUtils.join("#", vodItems));
}
for (int i = playFrom.size() - 1; i >= 0; i--) {
if (playFrom.get(i).isEmpty())
playFrom.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (playList.get(i).isEmpty())
playList.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (i >= playFrom.size())
playList.remove(i);
}
String vod_play_from = TextUtils.join("$$$", playFrom);
String vod_play_url = TextUtils.join("$$$", playList);
vod.setVodPlayFrom(vod_play_from);
vod.setVodPlayUrl(vod_play_url);
detailContentExt(webContent, vod);
Result result = new Result();
result.setList(Arrays.asList(vod));
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
} }
if (!rule.getDetailCate().isEmpty()) {
try {
category = vodNode.selOne(rule.getDetailCate()).asString().trim();
category = rule.getDetailCateR(category);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailYear().isEmpty()) {
try {
year = vodNode.selOne(rule.getDetailYear()).asString().trim();
year = rule.getDetailYearR(year);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailArea().isEmpty()) {
try {
area = vodNode.selOne(rule.getDetailArea()).asString().trim();
area = rule.getDetailAreaR(area);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailMark().isEmpty()) {
try {
remark = vodNode.selOne(rule.getDetailMark()).asString().trim();
remark = rule.getDetailMarkR(remark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailActor().isEmpty()) {
try {
actor = vodNode.selOne(rule.getDetailActor()).asString().trim();
actor = rule.getDetailActorR(actor);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailDirector().isEmpty()) {
try {
director = vodNode.selOne(rule.getDetailDirector()).asString().trim();
director = rule.getDetailDirectorR(director);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailDesc().isEmpty()) {
try {
desc = vodNode.selOne(rule.getDetailDesc()).asString().trim();
desc = rule.getDetailDescR(desc);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
Vod vod = new Vod();
vod.setVodId(ids.get(0));
vod.setVodName(title);
vod.setVodPic(cover);
vod.setTypeName(category);
vod.setVodYear(year);
vod.setVodArea(area);
vod.setVodRemarks(remark);
vod.setVodActor(actor);
vod.setVodDirector(director);
vod.setVodContent(desc);
ArrayList<String> playFrom = new ArrayList<>();
List<JXNode> fromNodes = doc.selN(rule.getDetailFromNode());
for (int i = 0; i < fromNodes.size(); i++) {
String name = fromNodes.get(i).selOne(rule.getDetailFromName()).asString().trim();
name = rule.getDetailFromNameR(name);
playFrom.add(name);
}
ArrayList<String> playList = new ArrayList<>();
List<JXNode> urlListNodes = doc.selN(rule.getDetailUrlNode());
for (int i = 0; i < urlListNodes.size(); i++) {
List<JXNode> urlNodes = urlListNodes.get(i).sel(rule.getDetailUrlSubNode());
List<String> vodItems = new ArrayList<>();
for (int j = 0; j < urlNodes.size(); j++) {
String name = urlNodes.get(j).selOne(rule.getDetailUrlName()).asString().trim();
name = rule.getDetailUrlNameR(name);
String id = urlNodes.get(j).selOne(rule.getDetailUrlId()).asString().trim();
id = rule.getDetailUrlIdR(id);
vodItems.add(name + "$" + id);
}
if (vodItems.size() == 0 && playFrom.size() > i) {
playFrom.set(i, "");
}
playList.add(TextUtils.join("#", vodItems));
}
for (int i = playFrom.size() - 1; i >= 0; i--) {
if (playFrom.get(i).isEmpty()) playFrom.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (playList.get(i).isEmpty()) playList.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (i >= playFrom.size()) playList.remove(i);
}
String vod_play_from = TextUtils.join("$$$", playFrom);
String vod_play_url = TextUtils.join("$$$", playList);
vod.setVodPlayFrom(vod_play_from);
vod.setVodPlayUrl(vod_play_url);
detailContentExt(webContent, vod);
Result result = new Result();
result.setList(Arrays.asList(vod));
return result.toString();
} }
@Override @Override
public String playerContent(String flag, String id, List<String> vipFlags) { public String playerContent(String flag, String id, List<String> vipFlags) {
try { fetchRule();
fetchRule(); String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id); SpiderDebug.log(webUrl);
SpiderDebug.log(webUrl); Result result = new Result();
Result result = new Result(); result.setParse("1");
result.setParse("1"); if (!rule.getPlayUa().isEmpty()) result.setUa(rule.getPlayUa());
if (!rule.getPlayUa().isEmpty()) result.setUa(rule.getPlayUa()); result.setUrl(webUrl);
result.setUrl(webUrl); return result.toString();
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
} }
@Override @Override
@ -379,22 +343,19 @@ public class XPath extends Spider {
} }
protected void fetchRule() { protected void fetchRule() {
if (rule == null) { if (rule == null && ext != null) {
if (ext != null) { if (ext.startsWith("http")) {
if (ext.startsWith("http")) { String json = OkHttpUtil.string(ext);
String json = OkHttpUtil.string(ext); rule = XPathRule.fromJson(json);
rule = XPathRule.fromJson(json); loadRuleExt(json);
loadRuleExt(json); } else {
} else { rule = XPathRule.fromJson(ext);
rule = XPathRule.fromJson(ext); loadRuleExt(ext);
loadRuleExt(ext);
}
} }
} }
} }
protected void loadRuleExt(String json) { protected void loadRuleExt(String json) {
} }
protected String fetch(String webUrl) { protected String fetch(String webUrl) {

30
app/src/main/java/com/github/catvod/spider/XPathFilter.java Normal file
View File

@ -0,0 +1,30 @@
package com.github.catvod.spider;
import android.text.TextUtils;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class XPathFilter extends XPath {
@Override
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
String cateUrl = rule.getCateUrl();
if (filter && extend != null && extend.size() > 0) {
for (String key : extend.keySet()) {
String value = extend.get(key);
if (TextUtils.isEmpty(value)) continue;
cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value));
}
}
cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg);
Matcher m = Pattern.compile("\\{(.*?)\\}").matcher(cateUrl);
while (m.find()) {
String n = m.group(0).replace("{", "").replace("}", "");
cateUrl = cateUrl.replace(m.group(0), "").replace("/" + n + "/", "");
}
return cateUrl;
}
}

179
app/src/main/java/com/github/catvod/spider/XPathMac.java Normal file
View File

@ -0,0 +1,179 @@
package com.github.catvod.spider;
import android.text.TextUtils;
import android.util.Base64;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.utils.Misc;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class XPathMac extends XPath {
// 嘗試分析直連
private boolean decodePlayUrl;
// 嘗試匹配官源標識以調用應用配置中的解析列表
private boolean decodeVipFlag;
// 播放器配置js
private String playerConfigJs = "";
// 播放器配置js取值正則
private String playerConfigJsRegex = "[\\W|\\S|.]*?MacPlayerConfig.player_list[\\W|\\S|.]*?=([\\W|\\S|.]*?),MacPlayerConfig.downer_list";
// 站點里播放源對應的真實官源
private HashMap<String, String> show2VipFlag = new HashMap<>();
@Override
protected void loadRuleExt(String json) {
try {
JSONObject jsonObj = new JSONObject(json);
decodePlayUrl = jsonObj.optBoolean("dcPlayUrl", false);
decodeVipFlag = jsonObj.optBoolean("dcVipFlag", false);
JSONObject dcShow2Vip = jsonObj.optJSONObject("dcShow2Vip");
if (dcShow2Vip != null) {
Iterator<String> keys = dcShow2Vip.keys();
while (keys.hasNext()) {
String name = keys.next();
show2VipFlag.put(name.trim(), dcShow2Vip.getString(name).trim());
}
}
playerConfigJs = jsonObj.optString("pCfgJs").trim();
playerConfigJsRegex = jsonObj.optString("pCfgJsR", playerConfigJsRegex).trim();
} catch (JSONException e) {
SpiderDebug.log(e);
}
}
@Override
public String homeContent(boolean filter) {
String result = super.homeContent(filter);
if (result.length() > 0 && playerConfigJs.length() > 0) { //嘗試通過playerConfigJs獲取展示和flag匹配關系
String webContent = fetch(playerConfigJs);
Matcher matcher = Pattern.compile(playerConfigJsRegex).matcher(webContent);
if (!matcher.find()) return result;
try {
JSONObject jsonObject = new JSONObject(matcher.group(1));
Iterator<String> keys = jsonObject.keys();
while (keys.hasNext()) {
String key = keys.next();
JSONObject keyObj = jsonObject.optJSONObject(key);
if (keyObj == null) continue;
String show = keyObj.optString("show").trim();
if (show.isEmpty()) continue;
show2VipFlag.put(show, key);
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
return result;
}
@Override
public String detailContent(List<String> ids) {
String result = super.detailContent(ids);
if (!decodeVipFlag || result.isEmpty()) return result;
try {
JSONObject jsonObject = new JSONObject(result);
String[] playFrom = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$");
if (playFrom.length > 0) {
for (int i = 0; i < playFrom.length; i++) {
if (show2VipFlag.containsKey(playFrom[i])) {
playFrom[i] = show2VipFlag.get(playFrom[i]);
}
}
jsonObject.optJSONArray("list").getJSONObject(0).put("vod_play_from", TextUtils.join("$$$", playFrom));
result = jsonObject.toString();
}
} catch (Exception e) {
SpiderDebug.log(e);
}
return result;
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
String videoUrl = null;
// 嘗試分析直連
if (decodePlayUrl) {
try {
Document doc = Jsoup.parse(fetch(webUrl));
Elements allScript = doc.select("script");
for (int i = 0; i < allScript.size(); i++) {
String scContent = allScript.get(i).html().trim();
if (scContent.startsWith("var player_")) {
int start = scContent.indexOf('{');
int end = scContent.lastIndexOf('}') + 1;
String json = scContent.substring(start, end);
JSONObject player = new JSONObject(json);
String videoUrlTmp = player.getString("url");
if (player.has("encrypt")) {
int encrypt = player.getInt("encrypt");
if (encrypt == 1) {
videoUrlTmp = URLDecoder.decode(videoUrlTmp);
} else if (encrypt == 2) {
videoUrlTmp = new String(Base64.decode(videoUrlTmp, Base64.DEFAULT));
videoUrlTmp = URLDecoder.decode(videoUrlTmp);
}
}
videoUrl = videoUrlTmp;
break;
}
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (videoUrl != null) {
// 適配2.0.6的調用應用內解析列表的支持, 需要配合直連分析和匹配官源解析一起使用，參考cjt影視和極品直連
if (decodeVipFlag && Misc.isVip(videoUrl)) { // 使用jx:1
try {
JSONObject result = new JSONObject();
result.put("parse", 1);
result.put("jx", "1");
result.put("url", videoUrl);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
} else if (decodeVipFlag && vipFlags.contains(flag)) { // 是否使用應用內解析列表解析官源
try {
JSONObject result = new JSONObject();
result.put("parse", 1);
result.put("playUrl", "");
result.put("url", videoUrl);
result.put("header", "");
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
}
// 如果是視頻直連 直接返回免解
else if (Misc.isVideoFormat(videoUrl)) {
try {
JSONObject result = new JSONObject();
result.put("parse", 0);
result.put("playUrl", "");
result.put("url", videoUrl);
result.put("header", "");
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
}
}
// 上述都失敗了就按默認模式走
return super.playerContent(flag, id, vipFlags);
}
}

30
app/src/main/java/com/github/catvod/spider/XPathMacFilter.java Normal file
View File

@ -0,0 +1,30 @@
package com.github.catvod.spider;
import android.text.TextUtils;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class XPathMacFilter extends XPathMac {
@Override
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
String cateUrl = rule.getCateUrl();
if (filter && extend != null && extend.size() > 0) {
for (String key : extend.keySet()) {
String value = extend.get(key);
if (TextUtils.isEmpty(value)) continue;
cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value));
}
}
cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg);
Matcher m = Pattern.compile("\\{(.*?)\\}").matcher(cateUrl);
while (m.find()) {
String n = m.group(0).replace("{", "").replace("}", "");
cateUrl = cateUrl.replace(m.group(0), "").replace("/" + n + "/", "");
}
return cateUrl;
}
}