Add BiuBiu and XPath

This commit is contained in:
FongMi 2022-08-22 20:56:33 +08:00
parent ce3aa70ea3
commit b6f335259e
9 changed files with 732 additions and 251 deletions

View File

@ -6,6 +6,7 @@ import com.google.gson.annotations.SerializedName;
import org.jetbrains.annotations.NotNull;
import org.json.JSONObject;
import java.util.Collections;
import java.util.List;
public class Result {
@ -31,6 +32,10 @@ public class Result {
this.list = list;
}
public List<Vod> getList() {
return list == null ? Collections.emptyList() : list;
}
public void setFilters(JSONObject filters) {
this.filters = filters;
}

View File

@ -6,8 +6,8 @@ public class SpiderDebug {
private static final String TAG = SpiderDebug.class.getSimpleName();
public static void log(Throwable th) {
Log.d(TAG, th.getMessage());
public static void log(Exception e) {
Log.d(TAG, e.getMessage());
}
public static void log(String msg) {

View File

@ -70,7 +70,7 @@ public class TxtSubscribe {
if (!noGroup.isEmpty()) {
allLives.put("未分類", noGroup);
}
} catch (Throwable ignored) {
} catch (Exception ignored) {
}
}
@ -118,8 +118,8 @@ public class TxtSubscribe {
ByteArrayInputStream baos = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
result[2] = baos;
return result;
} catch (Throwable th) {
th.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
return null;
}

View File

@ -55,8 +55,8 @@ public class JsonParallel {
for (int j = 0; j < futures.size(); j++) {
try {
futures.get(j).cancel(true);
} catch (Throwable th) {
SpiderDebug.log(th);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
futures.clear();

View File

@ -0,0 +1,276 @@
package com.github.catvod.spider;
import android.content.Context;
import android.text.TextUtils;
import com.github.catvod.bean.Class;
import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider;
import com.github.catvod.net.OkHttpUtil;
import com.github.catvod.utils.Misc;
import org.json.JSONArray;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Biubiu extends Spider {
private String ext = null;
private JSONObject rule = null;
private HashMap<String, String> getHeaders() {
HashMap<String, String> headers = new HashMap<>();
String ua = getRuleVal("UserW", Misc.CHROME).trim();
if (ua.isEmpty()) ua = Misc.CHROME;
headers.put("User-Agent", ua);
return headers;
}
@Override
public void init(Context context, String extend) {
super.init(context, extend);
this.ext = extend;
}
@Override
public String homeContent(boolean filter) {
fetchRule();
Result result = new Result();
List<Class> classes = new ArrayList<>();
String[] fenleis = getRuleVal("fenlei", "").split("#");
for (String fenlei : fenleis) {
String[] info = fenlei.split("\\$");
classes.add(new Class(info[0], info[1]));
}
result.setClasses(classes);
return result.toString();
}
@Override
public String homeVideoContent() {
fetchRule();
if (getRuleVal("shouye").equals("1")) return "";
List<Vod> videos = new ArrayList<>();
String[] fenleis = getRuleVal("fenlei", "").split("#");
for (String fenlei : fenleis) {
String[] info = fenlei.split("\\$");
Result result = category(info[1], "1");
for (int i = 0; i < result.getList().size(); i++) {
videos.add(result.getList().get(i));
if (videos.size() >= 30) break;
}
}
Result result = new Result();
result.setList(videos);
return result.toString();
}
private Result category(String tid, String pg) {
fetchRule();
String webUrl = getRuleVal("url") + tid + pg + getRuleVal("houzhui");
String html = fetch(webUrl);
String parseContent = html;
boolean shifouercijiequ = getRuleVal("shifouercijiequ").equals("1");
if (shifouercijiequ) {
String jiequqian = getRuleVal("jiequqian");
String jiequhou = getRuleVal("jiequhou");
parseContent = subContent(html, jiequqian, jiequhou).get(0);
}
String jiequshuzuqian = getRuleVal("jiequshuzuqian");
String jiequshuzuhou = getRuleVal("jiequshuzuhou");
List<Vod> videos = new ArrayList<>();
ArrayList<String> jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou);
for (int i = 0; i < jiequContents.size(); i++) {
try {
String jiequContent = jiequContents.get(i);
String title = subContent(jiequContent, getRuleVal("biaotiqian"), getRuleVal("biaotihou")).get(0);
String pic = subContent(jiequContent, getRuleVal("tupianqian"), getRuleVal("tupianhou")).get(0);
pic = Misc.fixUrl(webUrl, pic);
String link = subContent(jiequContent, getRuleVal("lianjieqian"), getRuleVal("lianjiehou")).get(0);
Vod vod = new Vod();
vod.setVodId(title + "$$$" + pic + "$$$" + link);
vod.setVodName(title);
vod.setVodPic(pic);
videos.add(vod);
} catch (Exception e) {
break;
}
}
Result result = new Result();
result.setList(videos);
return result;
}
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
return category(tid, pg).toString();
}
@Override
public String detailContent(List<String> ids) {
fetchRule();
String[] idInfo = ids.get(0).split("\\$\\$\\$");
String webUrl = getRuleVal("url") + idInfo[2];
String html = fetch(webUrl);
String parseContent = html;
boolean bfshifouercijiequ = getRuleVal("bfshifouercijiequ").equals("1");
if (bfshifouercijiequ) {
String jiequqian = getRuleVal("bfjiequqian");
String jiequhou = getRuleVal("bfjiequhou");
parseContent = subContent(html, jiequqian, jiequhou).get(0);
}
List<String> playList = new ArrayList<>();
List<String> playFrom = new ArrayList<>();
String jiequshuzuqian = getRuleVal("bfjiequshuzuqian");
String jiequshuzuhou = getRuleVal("bfjiequshuzuhou");
boolean bfyshifouercijiequ = getRuleVal("bfyshifouercijiequ").equals("1");
ArrayList<String> jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou);
for (int i = 0; i < jiequContents.size(); i++) {
try {
String jiequContent = jiequContents.get(i);
String parseJqContent = bfyshifouercijiequ ? subContent(jiequContent, getRuleVal("bfyjiequqian"), getRuleVal("bfyjiequhou")).get(0) : jiequContent;
ArrayList<String> lastParseContents = subContent(parseJqContent, getRuleVal("bfyjiequshuzuqian"), getRuleVal("bfyjiequshuzuhou"));
List<String> vodItems = new ArrayList<>();
for (int j = 0; j < lastParseContents.size(); j++) {
String title = subContent(lastParseContents.get(j), getRuleVal("bfbiaotiqian"), getRuleVal("bfbiaotihou")).get(0);
String link = subContent(lastParseContents.get(j), getRuleVal("bflianjieqian"), getRuleVal("bflianjiehou")).get(0);
vodItems.add(title + "$" + link);
}
playList.add(TextUtils.join("#", vodItems));
} catch (Exception e) {
break;
}
}
Vod video = new Vod();
video.setVodId(ids.get(0));
video.setVodName(idInfo[0]);
video.setVodPic(idInfo[1]);
for (int i = 0; i < playList.size(); i++) playFrom.add("播放列表" + (i + 1));
video.setVodPlayFrom(TextUtils.join("$$$", playFrom));
video.setVodPlayUrl(TextUtils.join("$$$", playList));
Result result = new Result();
result.setList(Arrays.asList(video));
return result.toString();
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = getRuleVal("url") + id;
Result result = new Result();
result.setParse("1");
result.setUrl(webUrl);
return result.toString();
}
@Override
public String searchContent(String key, boolean quick) {
try {
fetchRule();
boolean ssmoshiJson = getRuleVal("ssmoshi").equals("0");
String webUrlTmp = getRuleVal("url") + getRuleVal("sousuoqian") + key + getRuleVal("sousuohou");
String webUrl = webUrlTmp.split(";")[0];
String webContent = webUrlTmp.contains(";post") ? fetchPost(webUrl) : fetch(webUrl);
List<Vod> videos = new ArrayList<>();
if (ssmoshiJson) {
JSONObject data = new JSONObject(webContent);
JSONArray vodArray = data.getJSONArray("list");
for (int j = 0; j < vodArray.length(); j++) {
JSONObject vod = vodArray.getJSONObject(j);
String name = vod.optString(getRuleVal("jsname")).trim();
String id = vod.optString(getRuleVal("jsid")).trim();
String pic = vod.optString(getRuleVal("jspic")).trim();
pic = Misc.fixUrl(webUrl, pic);
Vod video = new Vod();
video.setVodId(name + "$$$" + pic + "$$$" + getRuleVal("sousuohouzhui") + id);
video.setVodName(name);
video.setVodPic(pic);
videos.add(video);
}
} else {
String parseContent = webContent;
boolean shifouercijiequ = getRuleVal("sousuoshifouercijiequ").equals("1");
if (shifouercijiequ) {
String jiequqian = getRuleVal("ssjiequqian");
String jiequhou = getRuleVal("ssjiequhou");
parseContent = subContent(webContent, jiequqian, jiequhou).get(0);
}
String jiequshuzuqian = getRuleVal("ssjiequshuzuqian");
String jiequshuzuhou = getRuleVal("ssjiequshuzuhou");
ArrayList<String> jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou);
for (int i = 0; i < jiequContents.size(); i++) {
try {
String jiequContent = jiequContents.get(i);
String title = subContent(jiequContent, getRuleVal("ssbiaotiqian"), getRuleVal("ssbiaotihou")).get(0);
String pic = subContent(jiequContent, getRuleVal("sstupianqian"), getRuleVal("sstupianhou")).get(0);
pic = Misc.fixUrl(webUrl, pic);
String link = subContent(jiequContent, getRuleVal("sslianjieqian"), getRuleVal("sslianjiehou")).get(0);
Vod video = new Vod();
video.setVodId(title + "$$$" + pic + "$$$" + link);
video.setVodName(title);
video.setVodPic(pic);
videos.add(video);
} catch (Exception e) {
e.printStackTrace();
break;
}
}
}
Result result = new Result();
result.setList(videos);
return result.toString();
} catch (Exception e) {
e.printStackTrace();
return "";
}
}
private void fetchRule() {
try {
if (rule != null) return;
if (ext == null) return;
if (ext.startsWith("http")) rule = new JSONObject(OkHttpUtil.string(ext, null));
else rule = new JSONObject(ext);
} catch (Exception e) {
e.printStackTrace();
}
}
private String fetch(String webUrl) {
return OkHttpUtil.string(webUrl, getHeaders()).replaceAll("\r|\n", "");
}
private String fetchPost(String webUrl) {
return OkHttpUtil.post(webUrl).replaceAll("\r|\n", "");
}
private String getRuleVal(String key, String defaultVal) {
String v = rule.optString(key);
if (v.isEmpty() || v.equals("")) return defaultVal;
return v;
}
private String getRuleVal(String key) {
return getRuleVal(key, "");
}
private ArrayList<String> subContent(String content, String startFlag, String endFlag) {
ArrayList<String> result = new ArrayList<>();
Pattern pattern = Pattern.compile(startFlag + "(.*?)" + endFlag);
Matcher matcher = pattern.matcher(content);
while (matcher.find()) result.add(matcher.group(1));
return result;
}
}

View File

@ -43,7 +43,6 @@ public class XPath extends Spider {
@Override
public String homeContent(boolean filter) {
try {
fetchRule();
Result result = new Result();
List<Vod> videos = new ArrayList<>();
@ -52,7 +51,6 @@ public class XPath extends Spider {
Set<String> keys = rule.getCateManual().keySet();
for (String k : keys) classes.add(new Class(k, rule.getCateManual().get(k)));
}
try {
String webUrl = rule.getHomeUrl();
JXDocument doc = JXDocument.create(fetch(webUrl));
if (rule.getCateManual().size() == 0) {
@ -66,7 +64,6 @@ public class XPath extends Spider {
}
}
if (!rule.getHomeVodNode().isEmpty()) {
try {
List<JXNode> vodNodes = doc.selN(rule.getHomeVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
String name = vodNodes.get(i).selOne(rule.getHomeVodName()).asString().trim();
@ -87,30 +84,16 @@ public class XPath extends Spider {
}
videos.add(new Vod(id, name, pic, mark));
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
} catch (Exception e) {
SpiderDebug.log(e);
}
result.setList(videos);
result.setClasses(classes);
result.setFilters(filter && rule.getFilter() != null ? rule.getFilter() : null);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
}
@Override
public String homeVideoContent() {
try {
fetchRule();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
@ -120,7 +103,6 @@ public class XPath extends Spider {
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
try {
fetchRule();
String webUrl = categoryUrl(tid, pg, filter, extend);
List<Vod> videos = new ArrayList<>();
@ -148,19 +130,13 @@ public class XPath extends Spider {
Result result = new Result();
result.setList(videos);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
}
protected void detailContentExt(String content, Vod vod) {
}
@Override
public String detailContent(List<String> ids) {
try {
fetchRule();
String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0));
String webContent = fetch(webUrl);
@ -273,16 +249,13 @@ public class XPath extends Spider {
playList.add(TextUtils.join("#", vodItems));
}
for (int i = playFrom.size() - 1; i >= 0; i--) {
if (playFrom.get(i).isEmpty())
playFrom.remove(i);
if (playFrom.get(i).isEmpty()) playFrom.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (playList.get(i).isEmpty())
playList.remove(i);
if (playList.get(i).isEmpty()) playList.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (i >= playFrom.size())
playList.remove(i);
if (i >= playFrom.size()) playList.remove(i);
}
String vod_play_from = TextUtils.join("$$$", playFrom);
String vod_play_url = TextUtils.join("$$$", playList);
@ -292,15 +265,10 @@ public class XPath extends Spider {
Result result = new Result();
result.setList(Arrays.asList(vod));
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
try {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
SpiderDebug.log(webUrl);
@ -309,10 +277,6 @@ public class XPath extends Spider {
if (!rule.getPlayUa().isEmpty()) result.setUa(rule.getPlayUa());
result.setUrl(webUrl);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
}
@Override
@ -379,8 +343,7 @@ public class XPath extends Spider {
}
protected void fetchRule() {
if (rule == null) {
if (ext != null) {
if (rule == null && ext != null) {
if (ext.startsWith("http")) {
String json = OkHttpUtil.string(ext);
rule = XPathRule.fromJson(json);
@ -391,10 +354,8 @@ public class XPath extends Spider {
}
}
}
}
protected void loadRuleExt(String json) {
}
protected String fetch(String webUrl) {

View File

@ -0,0 +1,30 @@
package com.github.catvod.spider;
import android.text.TextUtils;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class XPathFilter extends XPath {
@Override
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
String cateUrl = rule.getCateUrl();
if (filter && extend != null && extend.size() > 0) {
for (String key : extend.keySet()) {
String value = extend.get(key);
if (TextUtils.isEmpty(value)) continue;
cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value));
}
}
cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg);
Matcher m = Pattern.compile("\\{(.*?)\\}").matcher(cateUrl);
while (m.find()) {
String n = m.group(0).replace("{", "").replace("}", "");
cateUrl = cateUrl.replace(m.group(0), "").replace("/" + n + "/", "");
}
return cateUrl;
}
}

View File

@ -0,0 +1,179 @@
package com.github.catvod.spider;
import android.text.TextUtils;
import android.util.Base64;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.utils.Misc;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class XPathMac extends XPath {
// 嘗試分析直連
private boolean decodePlayUrl;
// 嘗試匹配官源標識以調用應用配置中的解析列表
private boolean decodeVipFlag;
// 播放器配置js
private String playerConfigJs = "";
// 播放器配置js取值正則
private String playerConfigJsRegex = "[\\W|\\S|.]*?MacPlayerConfig.player_list[\\W|\\S|.]*?=([\\W|\\S|.]*?),MacPlayerConfig.downer_list";
// 站點里播放源對應的真實官源
private HashMap<String, String> show2VipFlag = new HashMap<>();
@Override
protected void loadRuleExt(String json) {
try {
JSONObject jsonObj = new JSONObject(json);
decodePlayUrl = jsonObj.optBoolean("dcPlayUrl", false);
decodeVipFlag = jsonObj.optBoolean("dcVipFlag", false);
JSONObject dcShow2Vip = jsonObj.optJSONObject("dcShow2Vip");
if (dcShow2Vip != null) {
Iterator<String> keys = dcShow2Vip.keys();
while (keys.hasNext()) {
String name = keys.next();
show2VipFlag.put(name.trim(), dcShow2Vip.getString(name).trim());
}
}
playerConfigJs = jsonObj.optString("pCfgJs").trim();
playerConfigJsRegex = jsonObj.optString("pCfgJsR", playerConfigJsRegex).trim();
} catch (JSONException e) {
SpiderDebug.log(e);
}
}
@Override
public String homeContent(boolean filter) {
String result = super.homeContent(filter);
if (result.length() > 0 && playerConfigJs.length() > 0) { //嘗試通過playerConfigJs獲取展示和flag匹配關系
String webContent = fetch(playerConfigJs);
Matcher matcher = Pattern.compile(playerConfigJsRegex).matcher(webContent);
if (!matcher.find()) return result;
try {
JSONObject jsonObject = new JSONObject(matcher.group(1));
Iterator<String> keys = jsonObject.keys();
while (keys.hasNext()) {
String key = keys.next();
JSONObject keyObj = jsonObject.optJSONObject(key);
if (keyObj == null) continue;
String show = keyObj.optString("show").trim();
if (show.isEmpty()) continue;
show2VipFlag.put(show, key);
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
return result;
}
@Override
public String detailContent(List<String> ids) {
String result = super.detailContent(ids);
if (!decodeVipFlag || result.isEmpty()) return result;
try {
JSONObject jsonObject = new JSONObject(result);
String[] playFrom = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$");
if (playFrom.length > 0) {
for (int i = 0; i < playFrom.length; i++) {
if (show2VipFlag.containsKey(playFrom[i])) {
playFrom[i] = show2VipFlag.get(playFrom[i]);
}
}
jsonObject.optJSONArray("list").getJSONObject(0).put("vod_play_from", TextUtils.join("$$$", playFrom));
result = jsonObject.toString();
}
} catch (Exception e) {
SpiderDebug.log(e);
}
return result;
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
String videoUrl = null;
// 嘗試分析直連
if (decodePlayUrl) {
try {
Document doc = Jsoup.parse(fetch(webUrl));
Elements allScript = doc.select("script");
for (int i = 0; i < allScript.size(); i++) {
String scContent = allScript.get(i).html().trim();
if (scContent.startsWith("var player_")) {
int start = scContent.indexOf('{');
int end = scContent.lastIndexOf('}') + 1;
String json = scContent.substring(start, end);
JSONObject player = new JSONObject(json);
String videoUrlTmp = player.getString("url");
if (player.has("encrypt")) {
int encrypt = player.getInt("encrypt");
if (encrypt == 1) {
videoUrlTmp = URLDecoder.decode(videoUrlTmp);
} else if (encrypt == 2) {
videoUrlTmp = new String(Base64.decode(videoUrlTmp, Base64.DEFAULT));
videoUrlTmp = URLDecoder.decode(videoUrlTmp);
}
}
videoUrl = videoUrlTmp;
break;
}
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (videoUrl != null) {
// 適配2.0.6的調用應用內解析列表的支持, 需要配合直連分析和匹配官源解析一起使用參考cjt影視和極品直連
if (decodeVipFlag && Misc.isVip(videoUrl)) { // 使用jx:1
try {
JSONObject result = new JSONObject();
result.put("parse", 1);
result.put("jx", "1");
result.put("url", videoUrl);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
} else if (decodeVipFlag && vipFlags.contains(flag)) { // 是否使用應用內解析列表解析官源
try {
JSONObject result = new JSONObject();
result.put("parse", 1);
result.put("playUrl", "");
result.put("url", videoUrl);
result.put("header", "");
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
}
// 如果是視頻直連 直接返回免解
else if (Misc.isVideoFormat(videoUrl)) {
try {
JSONObject result = new JSONObject();
result.put("parse", 0);
result.put("playUrl", "");
result.put("url", videoUrl);
result.put("header", "");
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
}
}
// 上述都失敗了就按默認模式走
return super.playerContent(flag, id, vipFlags);
}
}

View File

@ -0,0 +1,30 @@
package com.github.catvod.spider;
import android.text.TextUtils;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class XPathMacFilter extends XPathMac {
@Override
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
String cateUrl = rule.getCateUrl();
if (filter && extend != null && extend.size() > 0) {
for (String key : extend.keySet()) {
String value = extend.get(key);
if (TextUtils.isEmpty(value)) continue;
cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value));
}
}
cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg);
Matcher m = Pattern.compile("\\{(.*?)\\}").matcher(cateUrl);
while (m.find()) {
String n = m.group(0).replace("{", "").replace("}", "");
cateUrl = cateUrl.replace(m.group(0), "").replace("/" + n + "/", "");
}
return cateUrl;
}
}