dongge
/
AndroidCatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix XPathMac UA

This commit is contained in:
FongMi 2023-02-15 18:16:22 +08:00
parent 58026bd091
commit d2aa52b588
6 changed files with 445 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

259
app/src/main/java/com/github/catvod/spider/XPath.java
View File

@ -3,18 +3,14 @@ package com.github.catvod.spider;
import android.content.Context;
import android.text.TextUtils;
import com.github.catvod.bean.Class;
import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod;
import com.github.catvod.bean.xpath.Rule;
import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils;
import com.github.catvod.utils.Trans;
import com.github.catvod.bean.xpath.Rule;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;
@ -27,39 +23,32 @@ import java.util.Set;
public class XPath extends Spider {
protected Rule rule = null;
private HashMap<String, String> getHeaders() {
HashMap<String, String> headers = new HashMap<>();
headers.put("User-Agent", rule.getUa().isEmpty() ? Utils.CHROME : rule.getUa());
return headers;
}
private void fetchRule(String ext) {
if (ext.startsWith("http")) {
String json = OkHttp.string(ext);
rule = Rule.fromJson(json);
loadRuleExt(json);
} else {
rule = Rule.fromJson(ext);
loadRuleExt(ext);
}
}
@Override
public void init(Context context) {
super.init(context);
}
public void init(Context context, String extend) {
super.init(context, extend);
fetchRule(extend);
this.ext = extend;
}
@Override
public String homeContent(boolean filter) throws JSONException {
List<Vod> list = new ArrayList<>();
List<Class> classes = new ArrayList<>();
public String homeContent(boolean filter) {
try {
fetchRule();
JSONObject result = new JSONObject();
JSONArray classes = new JSONArray();
if (rule.getCateManual().size() > 0) {
Set<String> keys = rule.getCateManual().keySet();
for (String k : keys) classes.add(new Class(rule.getCateManual().get(k), k));
for (String k : keys) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("type_name", k);
jsonObject.put("type_id", rule.getCateManual().get(k));
classes.put(jsonObject);
}
}
try {
String webUrl = rule.getHomeUrl();
JXDocument doc = JXDocument.create(fetch(webUrl));
if (rule.getCateManual().size() == 0) {
@ -69,10 +58,15 @@ public class XPath extends Spider {
name = rule.getCateNameR(name);
String id = navNodes.get(i).selOne(rule.getCateId()).asString().trim();
id = rule.getCateIdR(id);
classes.add(new Class(id, name));
JSONObject jsonObject = new JSONObject();
jsonObject.put("type_id", id);
jsonObject.put("type_name", name);
classes.put(jsonObject);
}
}
if (!rule.getHomeVodNode().isEmpty()) {
try {
JSONArray videos = new JSONArray();
List<JXNode> vodNodes = doc.selN(rule.getHomeVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
String name = vodNodes.get(i).selOne(rule.getHomeVodName()).asString().trim();
@ -91,10 +85,47 @@ public class XPath extends Spider {
SpiderDebug.log(e);
}
}
list.add(new Vod(id, name, pic, mark));
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", name);
v.put("vod_pic", pic);
v.put("vod_remarks", mark);
videos.put(v);
}
result.put("list", videos);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
return Result.string(classes, list, rule.getFilter());
} catch (Exception e) {
SpiderDebug.log(e);
}
result.put("class", classes);
if (filter && rule.getFilter() != null) {
result.put("filters", rule.getFilter());
}
return result.toString();
} catch (
Exception e) {
SpiderDebug.log(e);
}
return "";
}
protected HashMap<String, String> getHeaders() {
HashMap<String, String> headers = new HashMap<>();
headers.put("User-Agent", rule.getUa().isEmpty() ? "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36" : rule.getUa());
return headers;
}
@Override
public String homeVideoContent() {
try {
fetchRule();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
@ -103,8 +134,10 @@ public class XPath extends Spider {
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
try {
fetchRule();
String webUrl = categoryUrl(tid, pg, filter, extend);
List<Vod> list = new ArrayList<>();
JSONArray videos = new JSONArray();
JXDocument doc = JXDocument.create(fetch(webUrl));
List<JXNode> vodNodes = doc.selN(rule.getCateVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
@ -124,29 +157,44 @@ public class XPath extends Spider {
SpiderDebug.log(e);
}
}
list.add(new Vod(id, name, pic, mark));
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", name);
v.put("vod_pic", pic);
v.put("vod_remarks", mark);
videos.put(v);
}
return Result.string(list);
JSONObject result = new JSONObject();
result.put("page", pg);
result.put("pagecount", Integer.MAX_VALUE);
result.put("limit", 90);
result.put("total", Integer.MAX_VALUE);
result.put("list", videos);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
@Override
public String detailContent(List<String> ids) throws JSONException {
public String detailContent(List<String> ids) {
try {
fetchRule();
String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0));
String webContent = fetch(webUrl);
JXDocument doc = JXDocument.create(webContent);
JXNode vodNode = doc.selNOne(rule.getDetailNode());
String cover = "", title = "", desc = "", category = "", area = "", year = "", remark = "", director = "", actor = "";
title = vodNode.selOne(rule.getDetailName()).asString().trim();
title = rule.getDetailNameR(title);
if (!rule.getDetailImg().isEmpty()) {
try {
cover = vodNode.selOne(rule.getDetailImg()).asString().trim();
cover = rule.getDetailImgR(cover);
cover = Utils.fixUrl(webUrl, cover);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailCate().isEmpty()) {
try {
category = vodNode.selOne(rule.getDetailCate()).asString().trim();
@ -204,19 +252,20 @@ public class XPath extends Spider {
}
}
Vod vod = new Vod();
vod.setVodId(ids.get(0));
vod.setVodName(title);
vod.setVodPic(cover);
vod.setTypeName(category);
vod.setVodYear(year);
vod.setVodArea(area);
vod.setVodRemarks(remark);
vod.setVodActor(actor);
vod.setVodDirector(director);
vod.setVodContent(desc);
JSONObject vod = new JSONObject();
vod.put("vod_id", ids.get(0));
vod.put("vod_name", title);
vod.put("vod_pic", cover);
vod.put("type_name", category);
vod.put("vod_year", year);
vod.put("vod_area", area);
vod.put("vod_remarks", remark);
vod.put("vod_actor", actor);
vod.put("vod_director", director);
vod.put("vod_content", desc);
ArrayList<String> playFrom = new ArrayList<>();
List<JXNode> fromNodes = doc.selN(rule.getDetailFromNode());
for (int i = 0; i < fromNodes.size(); i++) {
String name = fromNodes.get(i).selOne(rule.getDetailFromName()).asString().trim();
@ -234,29 +283,46 @@ public class XPath extends Spider {
name = rule.getDetailUrlNameR(name);
String id = urlNodes.get(j).selOne(rule.getDetailUrlId()).asString().trim();
id = rule.getDetailUrlIdR(id);
vodItems.add(Trans.get(name) + "$" + id);
vodItems.add(name + "$" + id);
}
// 排除播放列表為空的播放源
if (vodItems.size() == 0 && playFrom.size() > i) {
playFrom.set(i, "");
}
playList.add(TextUtils.join("#", vodItems));
}
// 排除播放列表為空的播放源
for (int i = playFrom.size() - 1; i >= 0; i--) {
if (playFrom.get(i).isEmpty()) playFrom.remove(i);
if (playFrom.get(i).isEmpty())
playFrom.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (playList.get(i).isEmpty()) playList.remove(i);
if (playList.get(i).isEmpty())
playList.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (i >= playFrom.size()) playList.remove(i);
if (i >= playFrom.size())
playList.remove(i);
}
vod.setVodPlayFrom(TextUtils.join("$$$", playFrom));
vod.setVodPlayUrl(TextUtils.join("$$$", playList));
return Result.string(vod);
String vod_play_from = TextUtils.join("$$$", playFrom);
String vod_play_url = TextUtils.join("$$$", playList);
vod.put("vod_play_from", vod_play_from);
vod.put("vod_play_url", vod_play_url);
JSONObject result = new JSONObject();
JSONArray list = new JSONArray();
list.put(vod);
result.put("list", list);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
SpiderDebug.log(webUrl);
HashMap<String, String> headers = new HashMap<>();
@ -266,11 +332,16 @@ public class XPath extends Spider {
}
@Override
public String searchContent(String key, boolean quick) throws JSONException {
if (rule.getSearchUrl().isEmpty()) return "";
public String searchContent(String key, boolean quick) {
try {
fetchRule();
if (rule.getSearchUrl().isEmpty()) {
return "";
}
String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key));
String webContent = fetch(webUrl);
List<Vod> list = new ArrayList<>();
JSONObject result = new JSONObject();
JSONArray videos = new JSONArray();
if (rule.getSearchVodNode().startsWith("json:")) {
String[] node = rule.getSearchVodNode().substring(5).split(">");
JSONObject data = new JSONObject(webContent);
@ -288,7 +359,12 @@ public class XPath extends Spider {
pic = Utils.fixUrl(webUrl, pic);
String mark = vod.optString(rule.getSearchVodMark()).trim();
mark = rule.getSearchVodMarkR(mark);
list.add(new Vod(id, name, pic, mark));
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", name);
v.put("vod_pic", pic);
v.put("vod_remarks", mark);
videos.put(v);
}
} else {
data = data.getJSONObject(node[i]);
@ -314,13 +390,64 @@ public class XPath extends Spider {
SpiderDebug.log(e);
}
}
list.add(new Vod(id, name, pic, mark));
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", name);
v.put("vod_pic", pic);
v.put("vod_remarks", mark);
videos.put(v);
}
}
result.put("list", videos);
return result.toString();
} catch (
Exception e) {
SpiderDebug.log(e);
}
return "";
}
@Override
public boolean manualVideoCheck() {
return false;
}
private final String[] videoFormatList = new String[]{".m3u8", ".mp4", ".mpeg", ".flv"};
@Override
public boolean isVideoFormat(String url) {
url = url.toLowerCase();
if (url.contains("=http") || url.contains("=https") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) {
return false;
}
for (String format : videoFormatList) {
if (url.contains(format)) {
return true;
}
}
return false;
}
protected String ext = null;
protected Rule rule = null;
protected void fetchRule() {
if (rule == null) {
if (ext != null) {
if (ext.startsWith("http")) {
String json = OkHttp.string(ext, null);
rule = Rule.fromJson(json);
loadRuleExt(json);
} else {
rule = Rule.fromJson(ext);
loadRuleExt(ext);
}
}
}
return Result.string(list);
}
protected void loadRuleExt(String json) {
}
protected String fetch(String webUrl) {

10
app/src/main/java/com/github/catvod/spider/XPathFilter.java
View File

@ -1,7 +1,5 @@
package com.github.catvod.spider;
import android.text.TextUtils;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.regex.Matcher;
@ -9,16 +7,22 @@ import java.util.regex.Pattern;
public class XPathFilter extends XPath {
@Override
protected void loadRuleExt(String json) {
super.loadRuleExt(json);
}
@Override
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
String cateUrl = rule.getCateUrl();
if (filter && extend != null && extend.size() > 0) {
for (String key : extend.keySet()) {
String value = extend.get(key);
if (TextUtils.isEmpty(value)) continue;
if (value.length() > 0) {
cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value));
}
}
}
cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg);
Matcher m = Pattern.compile("\\{(.*?)\\}").matcher(cateUrl);
while (m.find()) {

65
app/src/main/java/com/github/catvod/spider/XPathMac.java
View File

@ -1,10 +1,12 @@
package com.github.catvod.spider;
import android.content.Context;
import android.text.TextUtils;
import android.util.Base64;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.utils.Utils;
import com.google.gson.Gson;
import org.json.JSONException;
import org.json.JSONObject;
@ -32,6 +34,16 @@ public class XPathMac extends XPath {
// 站點里播放源對應的真實官源
private HashMap<String, String> show2VipFlag = new HashMap<>();
/**
* mac cms 直連和官源調用應用內播放列表支持
*
* @param context
* @param extend
*/
public void init(Context context, String extend) {
super.init(context, extend);
}
@Override
protected void loadRuleExt(String json) {
try {
@ -48,46 +60,66 @@ public class XPathMac extends XPath {
}
playerConfigJs = jsonObj.optString("pCfgJs").trim();
playerConfigJsRegex = jsonObj.optString("pCfgJsR", playerConfigJsRegex).trim();
} catch (Exception e) {
} catch (JSONException e) {
SpiderDebug.log(e);
}
}
@Override
public String homeContent(boolean filter) throws JSONException {
public String homeContent(boolean filter) {
String result = super.homeContent(filter);
if (result.isEmpty() || playerConfigJs.isEmpty()) return result;
//嘗試通過playerConfigJs獲取展示和flag匹配關系
if (result.length() > 0 && playerConfigJs.length() > 0) { // 嘗試通過playerConfigJs獲取展示和flag匹配關系
String webContent = fetch(playerConfigJs);
Matcher matcher = Pattern.compile(playerConfigJsRegex).matcher(webContent);
if (!matcher.find()) return result;
if (matcher.find()) {
try {
JSONObject jsonObject = new JSONObject(matcher.group(1));
Iterator<String> keys = jsonObject.keys();
while (keys.hasNext()) {
String key = keys.next();
JSONObject keyObj = jsonObject.optJSONObject(key);
if (keyObj == null) continue;
if (keyObj == null)
continue;
String show = keyObj.optString("show").trim();
if (show.isEmpty()) continue;
if (show.isEmpty())
continue;
show2VipFlag.put(show, key);
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
// SpiderDebug.log(webContent);
}
return result;
}
@Override
public String detailContent(List<String> ids) throws JSONException {
public String detailContent(List<String> ids) {
String result = super.detailContent(ids);
if (!decodeVipFlag || result.isEmpty()) return result;
if (decodeVipFlag && result.length() > 0) {
try {
JSONObject jsonObject = new JSONObject(result);
String[] playFrom = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$");
if (playFrom.length == 0) return result;
for (int i = 0; i < playFrom.length; i++) if (show2VipFlag.containsKey(playFrom[i])) playFrom[i] = show2VipFlag.get(playFrom[i]);
String playFrom[] = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$");
if (playFrom.length > 0) {
for (int i = 0; i < playFrom.length; i++) {
if (show2VipFlag.containsKey(playFrom[i])) {
playFrom[i] = show2VipFlag.get(playFrom[i]);
}
}
jsonObject.optJSONArray("list").getJSONObject(0).put("vod_play_from", TextUtils.join("$$$", playFrom));
return jsonObject.toString();
result = jsonObject.toString();
}
} catch (Throwable th) {
SpiderDebug.log(th);
}
}
return result;
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
String videoUrl = null;
// 嘗試分析直連
@ -145,13 +177,16 @@ public class XPathMac extends XPath {
}
}
// 如果是視頻直連 直接返回免解
else if (Utils.isVideoFormat(videoUrl)) {
else if (isVideoFormat(videoUrl)) {
try {
JSONObject result = new JSONObject();
result.put("parse", 0);
result.put("playUrl", "");
result.put("url", videoUrl);
result.put("header", "");
HashMap<String, String> headers = new HashMap<>();
if (rule.getPlayUa().length() > 0) headers.put("User-Agent", rule.getPlayUa());
if (rule.getPlayReferer().length() > 0) headers.put("Referer", rule.getPlayReferer());
result.put("header", new Gson().toJson(headers));
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);

5
app/src/main/java/com/github/catvod/spider/XPathMacFilter.java
View File

@ -1,7 +1,5 @@
package com.github.catvod.spider;
import android.text.TextUtils;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.regex.Matcher;
@ -15,10 +13,11 @@ public class XPathMacFilter extends XPathMac {
if (filter && extend != null && extend.size() > 0) {
for (String key : extend.keySet()) {
String value = extend.get(key);
if (TextUtils.isEmpty(value)) continue;
if (value.length() > 0) {
cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value));
}
}
}
cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg);
Matcher m = Pattern.compile("\\{(.*?)\\}").matcher(cateUrl);
while (m.find()) {

BIN
jar/custom_spider.jar
View File

Binary file not shown.

2
jar/custom_spider.jar.md5
View File

@ -1 +1 @@
2060883055c8e0ca5e7bb45d824eafa4
5115b919853874c483c641dbfdf2b667