dongge
/
AndroidCatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix XPathMac UA

This commit is contained in:
FongMi 2023-02-15 18:16:22 +08:00
parent 58026bd091
commit d2aa52b588
6 changed files with 445 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

615
app/src/main/java/com/github/catvod/spider/XPath.java
View File

@ -3,18 +3,14 @@ package com.github.catvod.spider;
import android.content.Context; import android.content.Context;
import android.text.TextUtils; import android.text.TextUtils;
import com.github.catvod.bean.Class;
import com.github.catvod.bean.Result; import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod; import com.github.catvod.bean.xpath.Rule;
import com.github.catvod.crawler.Spider; import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug; import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp; import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils; import com.github.catvod.utils.Utils;
import com.github.catvod.utils.Trans;
import com.github.catvod.bean.xpath.Rule;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.seimicrawler.xpath.JXDocument; import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode; import org.seimicrawler.xpath.JXNode;
@ -27,74 +23,109 @@ import java.util.Set;
public class XPath extends Spider { public class XPath extends Spider {
protected Rule rule = null;
private HashMap<String, String> getHeaders() {
HashMap<String, String> headers = new HashMap<>();
headers.put("User-Agent", rule.getUa().isEmpty() ? Utils.CHROME : rule.getUa());
return headers;
}
private void fetchRule(String ext) {
if (ext.startsWith("http")) {
String json = OkHttp.string(ext);
rule = Rule.fromJson(json);
loadRuleExt(json);
} else {
rule = Rule.fromJson(ext);
loadRuleExt(ext);
}
}
@Override @Override
public void init(Context context) {
super.init(context);
}
public void init(Context context, String extend) { public void init(Context context, String extend) {
super.init(context, extend); super.init(context, extend);
fetchRule(extend); this.ext = extend;
} }
@Override @Override
public String homeContent(boolean filter) throws JSONException { public String homeContent(boolean filter) {
List<Vod> list = new ArrayList<>(); try {
List<Class> classes = new ArrayList<>(); fetchRule();
if (rule.getCateManual().size() > 0) { JSONObject result = new JSONObject();
Set<String> keys = rule.getCateManual().keySet(); JSONArray classes = new JSONArray();
for (String k : keys) classes.add(new Class(rule.getCateManual().get(k), k)); if (rule.getCateManual().size() > 0) {
} Set<String> keys = rule.getCateManual().keySet();
String webUrl = rule.getHomeUrl(); for (String k : keys) {
JXDocument doc = JXDocument.create(fetch(webUrl)); JSONObject jsonObject = new JSONObject();
if (rule.getCateManual().size() == 0) { jsonObject.put("type_name", k);
List<JXNode> navNodes = doc.selN(rule.getCateNode()); jsonObject.put("type_id", rule.getCateManual().get(k));
for (int i = 0; i < navNodes.size(); i++) { classes.put(jsonObject);
String name = navNodes.get(i).selOne(rule.getCateName()).asString().trim(); }
name = rule.getCateNameR(name);
String id = navNodes.get(i).selOne(rule.getCateId()).asString().trim();
id = rule.getCateIdR(id);
classes.add(new Class(id, name));
} }
} try {
if (!rule.getHomeVodNode().isEmpty()) { String webUrl = rule.getHomeUrl();
List<JXNode> vodNodes = doc.selN(rule.getHomeVodNode()); JXDocument doc = JXDocument.create(fetch(webUrl));
for (int i = 0; i < vodNodes.size(); i++) { if (rule.getCateManual().size() == 0) {
String name = vodNodes.get(i).selOne(rule.getHomeVodName()).asString().trim(); List<JXNode> navNodes = doc.selN(rule.getCateNode());
name = rule.getHomeVodNameR(name); for (int i = 0; i < navNodes.size(); i++) {
String id = vodNodes.get(i).selOne(rule.getHomeVodId()).asString().trim(); String name = navNodes.get(i).selOne(rule.getCateName()).asString().trim();
id = rule.getHomeVodIdR(id); name = rule.getCateNameR(name);
String pic = vodNodes.get(i).selOne(rule.getHomeVodImg()).asString().trim(); String id = navNodes.get(i).selOne(rule.getCateId()).asString().trim();
pic = rule.getHomeVodImgR(pic); id = rule.getCateIdR(id);
pic = Utils.fixUrl(webUrl, pic); JSONObject jsonObject = new JSONObject();
String mark = ""; jsonObject.put("type_id", id);
if (!rule.getHomeVodMark().isEmpty()) { jsonObject.put("type_name", name);
classes.put(jsonObject);
}
}
if (!rule.getHomeVodNode().isEmpty()) {
try { try {
mark = vodNodes.get(i).selOne(rule.getHomeVodMark()).asString().trim(); JSONArray videos = new JSONArray();
mark = rule.getHomeVodMarkR(mark); List<JXNode> vodNodes = doc.selN(rule.getHomeVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
String name = vodNodes.get(i).selOne(rule.getHomeVodName()).asString().trim();
name = rule.getHomeVodNameR(name);
String id = vodNodes.get(i).selOne(rule.getHomeVodId()).asString().trim();
id = rule.getHomeVodIdR(id);
String pic = vodNodes.get(i).selOne(rule.getHomeVodImg()).asString().trim();
pic = rule.getHomeVodImgR(pic);
pic = Utils.fixUrl(webUrl, pic);
String mark = "";
if (!rule.getHomeVodMark().isEmpty()) {
try {
mark = vodNodes.get(i).selOne(rule.getHomeVodMark()).asString().trim();
mark = rule.getHomeVodMarkR(mark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", name);
v.put("vod_pic", pic);
v.put("vod_remarks", mark);
videos.put(v);
}
result.put("list", videos);
} catch (Exception e) { } catch (Exception e) {
SpiderDebug.log(e); SpiderDebug.log(e);
} }
} }
list.add(new Vod(id, name, pic, mark)); } catch (Exception e) {
SpiderDebug.log(e);
} }
result.put("class", classes);
if (filter && rule.getFilter() != null) {
result.put("filters", rule.getFilter());
}
return result.toString();
} catch (
Exception e) {
SpiderDebug.log(e);
} }
return Result.string(classes, list, rule.getFilter()); return "";
}
protected HashMap<String, String> getHeaders() {
HashMap<String, String> headers = new HashMap<>();
headers.put("User-Agent", rule.getUa().isEmpty() ? "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36" : rule.getUa());
return headers;
}
@Override
public String homeVideoContent() {
try {
fetchRule();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
} }
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) { protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
@ -103,160 +134,195 @@ public class XPath extends Spider {
@Override @Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) { public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
String webUrl = categoryUrl(tid, pg, filter, extend); try {
List<Vod> list = new ArrayList<>(); fetchRule();
JXDocument doc = JXDocument.create(fetch(webUrl)); String webUrl = categoryUrl(tid, pg, filter, extend);
List<JXNode> vodNodes = doc.selN(rule.getCateVodNode()); JSONArray videos = new JSONArray();
for (int i = 0; i < vodNodes.size(); i++) { JXDocument doc = JXDocument.create(fetch(webUrl));
String name = vodNodes.get(i).selOne(rule.getCateVodName()).asString().trim(); List<JXNode> vodNodes = doc.selN(rule.getCateVodNode());
name = rule.getCateVodNameR(name); for (int i = 0; i < vodNodes.size(); i++) {
String id = vodNodes.get(i).selOne(rule.getCateVodId()).asString().trim(); String name = vodNodes.get(i).selOne(rule.getCateVodName()).asString().trim();
id = rule.getCateVodIdR(id); name = rule.getCateVodNameR(name);
String pic = vodNodes.get(i).selOne(rule.getCateVodImg()).asString().trim(); String id = vodNodes.get(i).selOne(rule.getCateVodId()).asString().trim();
pic = rule.getCateVodImgR(pic); id = rule.getCateVodIdR(id);
pic = Utils.fixUrl(webUrl, pic); String pic = vodNodes.get(i).selOne(rule.getCateVodImg()).asString().trim();
String mark = ""; pic = rule.getCateVodImgR(pic);
if (!rule.getCateVodMark().isEmpty()) { pic = Utils.fixUrl(webUrl, pic);
String mark = "";
if (!rule.getCateVodMark().isEmpty()) {
try {
mark = vodNodes.get(i).selOne(rule.getCateVodMark()).asString().trim();
mark = rule.getCateVodMarkR(mark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", name);
v.put("vod_pic", pic);
v.put("vod_remarks", mark);
videos.put(v);
}
JSONObject result = new JSONObject();
result.put("page", pg);
result.put("pagecount", Integer.MAX_VALUE);
result.put("limit", 90);
result.put("total", Integer.MAX_VALUE);
result.put("list", videos);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
@Override
public String detailContent(List<String> ids) {
try {
fetchRule();
String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0));
String webContent = fetch(webUrl);
JXDocument doc = JXDocument.create(webContent);
JXNode vodNode = doc.selNOne(rule.getDetailNode());
String cover = "", title = "", desc = "", category = "", area = "", year = "", remark = "", director = "", actor = "";
title = vodNode.selOne(rule.getDetailName()).asString().trim();
title = rule.getDetailNameR(title);
cover = vodNode.selOne(rule.getDetailImg()).asString().trim();
cover = rule.getDetailImgR(cover);
cover = Utils.fixUrl(webUrl, cover);
if (!rule.getDetailCate().isEmpty()) {
try { try {
mark = vodNodes.get(i).selOne(rule.getCateVodMark()).asString().trim(); category = vodNode.selOne(rule.getDetailCate()).asString().trim();
mark = rule.getCateVodMarkR(mark); category = rule.getDetailCateR(category);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailYear().isEmpty()) {
try {
year = vodNode.selOne(rule.getDetailYear()).asString().trim();
year = rule.getDetailYearR(year);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailArea().isEmpty()) {
try {
area = vodNode.selOne(rule.getDetailArea()).asString().trim();
area = rule.getDetailAreaR(area);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailMark().isEmpty()) {
try {
remark = vodNode.selOne(rule.getDetailMark()).asString().trim();
remark = rule.getDetailMarkR(remark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailActor().isEmpty()) {
try {
actor = vodNode.selOne(rule.getDetailActor()).asString().trim();
actor = rule.getDetailActorR(actor);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailDirector().isEmpty()) {
try {
director = vodNode.selOne(rule.getDetailDirector()).asString().trim();
director = rule.getDetailDirectorR(director);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailDesc().isEmpty()) {
try {
desc = vodNode.selOne(rule.getDetailDesc()).asString().trim();
desc = rule.getDetailDescR(desc);
} catch (Exception e) { } catch (Exception e) {
SpiderDebug.log(e); SpiderDebug.log(e);
} }
} }
list.add(new Vod(id, name, pic, mark));
}
return Result.string(list);
}
@Override JSONObject vod = new JSONObject();
public String detailContent(List<String> ids) throws JSONException { vod.put("vod_id", ids.get(0));
String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0)); vod.put("vod_name", title);
String webContent = fetch(webUrl); vod.put("vod_pic", cover);
JXDocument doc = JXDocument.create(webContent); vod.put("type_name", category);
JXNode vodNode = doc.selNOne(rule.getDetailNode()); vod.put("vod_year", year);
String cover = "", title = "", desc = "", category = "", area = "", year = "", remark = "", director = "", actor = ""; vod.put("vod_area", area);
title = vodNode.selOne(rule.getDetailName()).asString().trim(); vod.put("vod_remarks", remark);
title = rule.getDetailNameR(title); vod.put("vod_actor", actor);
if (!rule.getDetailImg().isEmpty()) { vod.put("vod_director", director);
try { vod.put("vod_content", desc);
cover = vodNode.selOne(rule.getDetailImg()).asString().trim();
cover = rule.getDetailImgR(cover);
cover = Utils.fixUrl(webUrl, cover);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailCate().isEmpty()) {
try {
category = vodNode.selOne(rule.getDetailCate()).asString().trim();
category = rule.getDetailCateR(category);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailYear().isEmpty()) {
try {
year = vodNode.selOne(rule.getDetailYear()).asString().trim();
year = rule.getDetailYearR(year);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailArea().isEmpty()) {
try {
area = vodNode.selOne(rule.getDetailArea()).asString().trim();
area = rule.getDetailAreaR(area);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailMark().isEmpty()) {
try {
remark = vodNode.selOne(rule.getDetailMark()).asString().trim();
remark = rule.getDetailMarkR(remark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailActor().isEmpty()) {
try {
actor = vodNode.selOne(rule.getDetailActor()).asString().trim();
actor = rule.getDetailActorR(actor);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailDirector().isEmpty()) {
try {
director = vodNode.selOne(rule.getDetailDirector()).asString().trim();
director = rule.getDetailDirectorR(director);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailDesc().isEmpty()) {
try {
desc = vodNode.selOne(rule.getDetailDesc()).asString().trim();
desc = rule.getDetailDescR(desc);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
Vod vod = new Vod(); ArrayList<String> playFrom = new ArrayList<>();
vod.setVodId(ids.get(0));
vod.setVodName(title);
vod.setVodPic(cover);
vod.setTypeName(category);
vod.setVodYear(year);
vod.setVodArea(area);
vod.setVodRemarks(remark);
vod.setVodActor(actor);
vod.setVodDirector(director);
vod.setVodContent(desc);
ArrayList<String> playFrom = new ArrayList<>(); List<JXNode> fromNodes = doc.selN(rule.getDetailFromNode());
List<JXNode> fromNodes = doc.selN(rule.getDetailFromNode()); for (int i = 0; i < fromNodes.size(); i++) {
for (int i = 0; i < fromNodes.size(); i++) { String name = fromNodes.get(i).selOne(rule.getDetailFromName()).asString().trim();
String name = fromNodes.get(i).selOne(rule.getDetailFromName()).asString().trim(); name = rule.getDetailFromNameR(name);
name = rule.getDetailFromNameR(name); playFrom.add(name);
playFrom.add(name); }
}
ArrayList<String> playList = new ArrayList<>(); ArrayList<String> playList = new ArrayList<>();
List<JXNode> urlListNodes = doc.selN(rule.getDetailUrlNode()); List<JXNode> urlListNodes = doc.selN(rule.getDetailUrlNode());
for (int i = 0; i < urlListNodes.size(); i++) { for (int i = 0; i < urlListNodes.size(); i++) {
List<JXNode> urlNodes = urlListNodes.get(i).sel(rule.getDetailUrlSubNode()); List<JXNode> urlNodes = urlListNodes.get(i).sel(rule.getDetailUrlSubNode());
List<String> vodItems = new ArrayList<>(); List<String> vodItems = new ArrayList<>();
for (int j = 0; j < urlNodes.size(); j++) { for (int j = 0; j < urlNodes.size(); j++) {
String name = urlNodes.get(j).selOne(rule.getDetailUrlName()).asString().trim(); String name = urlNodes.get(j).selOne(rule.getDetailUrlName()).asString().trim();
name = rule.getDetailUrlNameR(name); name = rule.getDetailUrlNameR(name);
String id = urlNodes.get(j).selOne(rule.getDetailUrlId()).asString().trim(); String id = urlNodes.get(j).selOne(rule.getDetailUrlId()).asString().trim();
id = rule.getDetailUrlIdR(id); id = rule.getDetailUrlIdR(id);
vodItems.add(Trans.get(name) + "$" + id); vodItems.add(name + "$" + id);
}
// 排除播放列表為空的播放源
if (vodItems.size() == 0 && playFrom.size() > i) {
playFrom.set(i, "");
}
playList.add(TextUtils.join("#", vodItems));
} }
if (vodItems.size() == 0 && playFrom.size() > i) { // 排除播放列表為空的播放源
playFrom.set(i, ""); for (int i = playFrom.size() - 1; i >= 0; i--) {
if (playFrom.get(i).isEmpty())
playFrom.remove(i);
} }
playList.add(TextUtils.join("#", vodItems)); for (int i = playList.size() - 1; i >= 0; i--) {
if (playList.get(i).isEmpty())
playList.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (i >= playFrom.size())
playList.remove(i);
}
String vod_play_from = TextUtils.join("$$$", playFrom);
String vod_play_url = TextUtils.join("$$$", playList);
vod.put("vod_play_from", vod_play_from);
vod.put("vod_play_url", vod_play_url);
JSONObject result = new JSONObject();
JSONArray list = new JSONArray();
list.put(vod);
result.put("list", list);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
} }
for (int i = playFrom.size() - 1; i >= 0; i--) { return "";
if (playFrom.get(i).isEmpty()) playFrom.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (playList.get(i).isEmpty()) playList.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (i >= playFrom.size()) playList.remove(i);
}
vod.setVodPlayFrom(TextUtils.join("$$$", playFrom));
vod.setVodPlayUrl(TextUtils.join("$$$", playList));
return Result.string(vod);
} }
@Override @Override
public String playerContent(String flag, String id, List<String> vipFlags) { public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id); String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
SpiderDebug.log(webUrl); SpiderDebug.log(webUrl);
HashMap<String, String> headers = new HashMap<>(); HashMap<String, String> headers = new HashMap<>();
@ -266,61 +332,122 @@ public class XPath extends Spider {
} }
@Override @Override
public String searchContent(String key, boolean quick) throws JSONException { public String searchContent(String key, boolean quick) {
if (rule.getSearchUrl().isEmpty()) return ""; try {
String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key)); fetchRule();
String webContent = fetch(webUrl); if (rule.getSearchUrl().isEmpty()) {
List<Vod> list = new ArrayList<>(); return "";
if (rule.getSearchVodNode().startsWith("json:")) { }
String[] node = rule.getSearchVodNode().substring(5).split(">"); String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key));
JSONObject data = new JSONObject(webContent); String webContent = fetch(webUrl);
for (int i = 0; i < node.length; i++) { JSONObject result = new JSONObject();
if (i == node.length - 1) { JSONArray videos = new JSONArray();
JSONArray vodArray = data.getJSONArray(node[i]); if (rule.getSearchVodNode().startsWith("json:")) {
for (int j = 0; j < vodArray.length(); j++) { String[] node = rule.getSearchVodNode().substring(5).split(">");
JSONObject vod = vodArray.getJSONObject(j); JSONObject data = new JSONObject(webContent);
String name = vod.optString(rule.getSearchVodName()).trim(); for (int i = 0; i < node.length; i++) {
name = rule.getSearchVodNameR(name); if (i == node.length - 1) {
String id = vod.optString(rule.getSearchVodId()).trim(); JSONArray vodArray = data.getJSONArray(node[i]);
id = rule.getSearchVodIdR(id); for (int j = 0; j < vodArray.length(); j++) {
String pic = vod.optString(rule.getSearchVodImg()).trim(); JSONObject vod = vodArray.getJSONObject(j);
pic = rule.getSearchVodImgR(pic); String name = vod.optString(rule.getSearchVodName()).trim();
pic = Utils.fixUrl(webUrl, pic); name = rule.getSearchVodNameR(name);
String mark = vod.optString(rule.getSearchVodMark()).trim(); String id = vod.optString(rule.getSearchVodId()).trim();
mark = rule.getSearchVodMarkR(mark); id = rule.getSearchVodIdR(id);
list.add(new Vod(id, name, pic, mark)); String pic = vod.optString(rule.getSearchVodImg()).trim();
pic = rule.getSearchVodImgR(pic);
pic = Utils.fixUrl(webUrl, pic);
String mark = vod.optString(rule.getSearchVodMark()).trim();
mark = rule.getSearchVodMarkR(mark);
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", name);
v.put("vod_pic", pic);
v.put("vod_remarks", mark);
videos.put(v);
}
} else {
data = data.getJSONObject(node[i]);
} }
} else { }
data = data.getJSONObject(node[i]); } else {
JXDocument doc = JXDocument.create(webContent);
List<JXNode> vodNodes = doc.selN(rule.getSearchVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
String name = vodNodes.get(i).selOne(rule.getSearchVodName()).asString().trim();
name = rule.getSearchVodNameR(name);
String id = vodNodes.get(i).selOne(rule.getSearchVodId()).asString().trim();
id = rule.getSearchVodIdR(id);
String pic = vodNodes.get(i).selOne(rule.getSearchVodImg()).asString().trim();
pic = rule.getSearchVodImgR(pic);
pic = Utils.fixUrl(webUrl, pic);
String mark = "";
if (!rule.getCateVodMark().isEmpty()) {
try {
mark = vodNodes.get(i).selOne(rule.getSearchVodMark()).asString().trim();
mark = rule.getSearchVodMarkR(mark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", name);
v.put("vod_pic", pic);
v.put("vod_remarks", mark);
videos.put(v);
} }
} }
} else { result.put("list", videos);
JXDocument doc = JXDocument.create(webContent); return result.toString();
List<JXNode> vodNodes = doc.selN(rule.getSearchVodNode()); } catch (
for (int i = 0; i < vodNodes.size(); i++) { Exception e) {
String name = vodNodes.get(i).selOne(rule.getSearchVodName()).asString().trim(); SpiderDebug.log(e);
name = rule.getSearchVodNameR(name); }
String id = vodNodes.get(i).selOne(rule.getSearchVodId()).asString().trim(); return "";
id = rule.getSearchVodIdR(id); }
String pic = vodNodes.get(i).selOne(rule.getSearchVodImg()).asString().trim();
pic = rule.getSearchVodImgR(pic); @Override
pic = Utils.fixUrl(webUrl, pic); public boolean manualVideoCheck() {
String mark = ""; return false;
if (!rule.getCateVodMark().isEmpty()) { }
try {
mark = vodNodes.get(i).selOne(rule.getSearchVodMark()).asString().trim(); private final String[] videoFormatList = new String[]{".m3u8", ".mp4", ".mpeg", ".flv"};
mark = rule.getSearchVodMarkR(mark);
} catch (Exception e) { @Override
SpiderDebug.log(e); public boolean isVideoFormat(String url) {
} url = url.toLowerCase();
} if (url.contains("=http") || url.contains("=https") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) {
list.add(new Vod(id, name, pic, mark)); return false;
}
for (String format : videoFormatList) {
if (url.contains(format)) {
return true;
}
}
return false;
}
protected String ext = null;
protected Rule rule = null;
protected void fetchRule() {
if (rule == null) {
if (ext != null) {
if (ext.startsWith("http")) {
String json = OkHttp.string(ext, null);
rule = Rule.fromJson(json);
loadRuleExt(json);
} else {
rule = Rule.fromJson(ext);
loadRuleExt(ext);
}
} }
} }
return Result.string(list);
} }
protected void loadRuleExt(String json) { protected void loadRuleExt(String json) {
} }
protected String fetch(String webUrl) { protected String fetch(String webUrl) {

12
app/src/main/java/com/github/catvod/spider/XPathFilter.java
View File

@ -1,7 +1,5 @@
package com.github.catvod.spider; package com.github.catvod.spider;
import android.text.TextUtils;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.HashMap; import java.util.HashMap;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@ -9,14 +7,20 @@ import java.util.regex.Pattern;
public class XPathFilter extends XPath { public class XPathFilter extends XPath {
@Override
protected void loadRuleExt(String json) {
super.loadRuleExt(json);
}
@Override @Override
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) { protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
String cateUrl = rule.getCateUrl(); String cateUrl = rule.getCateUrl();
if (filter && extend != null && extend.size() > 0) { if (filter && extend != null && extend.size() > 0) {
for (String key : extend.keySet()) { for (String key : extend.keySet()) {
String value = extend.get(key); String value = extend.get(key);
if (TextUtils.isEmpty(value)) continue; if (value.length() > 0) {
cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value)); cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value));
}
} }
} }
cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg); cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg);

87
app/src/main/java/com/github/catvod/spider/XPathMac.java
View File

@ -1,10 +1,12 @@
package com.github.catvod.spider; package com.github.catvod.spider;
import android.content.Context;
import android.text.TextUtils; import android.text.TextUtils;
import android.util.Base64; import android.util.Base64;
import com.github.catvod.crawler.SpiderDebug; import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.utils.Utils; import com.github.catvod.utils.Utils;
import com.google.gson.Gson;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
@ -32,6 +34,16 @@ public class XPathMac extends XPath {
// 站點里播放源對應的真實官源 // 站點里播放源對應的真實官源
private HashMap<String, String> show2VipFlag = new HashMap<>(); private HashMap<String, String> show2VipFlag = new HashMap<>();
/**
* mac cms 直連和官源調用應用內播放列表支持
*
* @param context
* @param extend
*/
public void init(Context context, String extend) {
super.init(context, extend);
}
@Override @Override
protected void loadRuleExt(String json) { protected void loadRuleExt(String json) {
try { try {
@ -48,46 +60,66 @@ public class XPathMac extends XPath {
} }
playerConfigJs = jsonObj.optString("pCfgJs").trim(); playerConfigJs = jsonObj.optString("pCfgJs").trim();
playerConfigJsRegex = jsonObj.optString("pCfgJsR", playerConfigJsRegex).trim(); playerConfigJsRegex = jsonObj.optString("pCfgJsR", playerConfigJsRegex).trim();
} catch (Exception e) { } catch (JSONException e) {
SpiderDebug.log(e); SpiderDebug.log(e);
} }
} }
@Override @Override
public String homeContent(boolean filter) throws JSONException { public String homeContent(boolean filter) {
String result = super.homeContent(filter); String result = super.homeContent(filter);
if (result.isEmpty() || playerConfigJs.isEmpty()) return result; if (result.length() > 0 && playerConfigJs.length() > 0) { // 嘗試通過playerConfigJs獲取展示和flag匹配關系
//嘗試通過playerConfigJs獲取展示和flag匹配關系 String webContent = fetch(playerConfigJs);
String webContent = fetch(playerConfigJs); Matcher matcher = Pattern.compile(playerConfigJsRegex).matcher(webContent);
Matcher matcher = Pattern.compile(playerConfigJsRegex).matcher(webContent); if (matcher.find()) {
if (!matcher.find()) return result; try {
JSONObject jsonObject = new JSONObject(matcher.group(1)); JSONObject jsonObject = new JSONObject(matcher.group(1));
Iterator<String> keys = jsonObject.keys(); Iterator<String> keys = jsonObject.keys();
while (keys.hasNext()) { while (keys.hasNext()) {
String key = keys.next(); String key = keys.next();
JSONObject keyObj = jsonObject.optJSONObject(key); JSONObject keyObj = jsonObject.optJSONObject(key);
if (keyObj == null) continue; if (keyObj == null)
String show = keyObj.optString("show").trim(); continue;
if (show.isEmpty()) continue; String show = keyObj.optString("show").trim();
show2VipFlag.put(show, key); if (show.isEmpty())
continue;
show2VipFlag.put(show, key);
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
// SpiderDebug.log(webContent);
} }
return result; return result;
} }
@Override @Override
public String detailContent(List<String> ids) throws JSONException { public String detailContent(List<String> ids) {
String result = super.detailContent(ids); String result = super.detailContent(ids);
if (!decodeVipFlag || result.isEmpty()) return result; if (decodeVipFlag && result.length() > 0) {
JSONObject jsonObject = new JSONObject(result); try {
String[] playFrom = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$"); JSONObject jsonObject = new JSONObject(result);
if (playFrom.length == 0) return result; String playFrom[] = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$");
for (int i = 0; i < playFrom.length; i++) if (show2VipFlag.containsKey(playFrom[i])) playFrom[i] = show2VipFlag.get(playFrom[i]); if (playFrom.length > 0) {
jsonObject.optJSONArray("list").getJSONObject(0).put("vod_play_from", TextUtils.join("$$$", playFrom)); for (int i = 0; i < playFrom.length; i++) {
return jsonObject.toString(); if (show2VipFlag.containsKey(playFrom[i])) {
playFrom[i] = show2VipFlag.get(playFrom[i]);
}
}
jsonObject.optJSONArray("list").getJSONObject(0).put("vod_play_from", TextUtils.join("$$$", playFrom));
result = jsonObject.toString();
}
} catch (Throwable th) {
SpiderDebug.log(th);
}
}
return result;
} }
@Override @Override
public String playerContent(String flag, String id, List<String> vipFlags) { public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id); String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
String videoUrl = null; String videoUrl = null;
// 嘗試分析直連 // 嘗試分析直連
@ -145,13 +177,16 @@ public class XPathMac extends XPath {
} }
} }
// 如果是視頻直連 直接返回免解 // 如果是視頻直連 直接返回免解
else if (Utils.isVideoFormat(videoUrl)) { else if (isVideoFormat(videoUrl)) {
try { try {
JSONObject result = new JSONObject(); JSONObject result = new JSONObject();
result.put("parse", 0); result.put("parse", 0);
result.put("playUrl", ""); result.put("playUrl", "");
result.put("url", videoUrl); result.put("url", videoUrl);
result.put("header", ""); HashMap<String, String> headers = new HashMap<>();
if (rule.getPlayUa().length() > 0) headers.put("User-Agent", rule.getPlayUa());
if (rule.getPlayReferer().length() > 0) headers.put("Referer", rule.getPlayReferer());
result.put("header", new Gson().toJson(headers));
return result.toString(); return result.toString();
} catch (Exception e) { } catch (Exception e) {
SpiderDebug.log(e); SpiderDebug.log(e);

7
app/src/main/java/com/github/catvod/spider/XPathMacFilter.java
View File

@ -1,7 +1,5 @@
package com.github.catvod.spider; package com.github.catvod.spider;
import android.text.TextUtils;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.HashMap; import java.util.HashMap;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@ -15,8 +13,9 @@ public class XPathMacFilter extends XPathMac {
if (filter && extend != null && extend.size() > 0) { if (filter && extend != null && extend.size() > 0) {
for (String key : extend.keySet()) { for (String key : extend.keySet()) {
String value = extend.get(key); String value = extend.get(key);
if (TextUtils.isEmpty(value)) continue; if (value.length() > 0) {
cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value)); cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value));
}
} }
} }
cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg); cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg);

BIN
jar/custom_spider.jar
View File

Binary file not shown.

2
jar/custom_spider.jar.md5
View File

@ -1 +1 @@
2060883055c8e0ca5e7bb45d824eafa4 5115b919853874c483c641dbfdf2b667