diff --git a/app/build.gradle b/app/build.gradle index 644ab022..d0161432 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -33,7 +33,7 @@ android { dependencies { implementation 'com.squareup.okhttp3:okhttp:4.10.0' - implementation 'com.google.code.gson:gson:2.9.0' - implementation 'cn.wanghaomiao:JsoupXpath:2.5.0' - implementation 'org.jsoup:jsoup:1.14.3' + implementation 'com.google.code.gson:gson:2.9.1' + implementation 'cn.wanghaomiao:JsoupXpath:2.5.1' + implementation 'org.jsoup:jsoup:1.15.3' } \ No newline at end of file diff --git a/app/src/main/java/com/github/catvod/crawler/Spider.java b/app/src/main/java/com/github/catvod/crawler/Spider.java index 386ffd18..d727eb0d 100644 --- a/app/src/main/java/com/github/catvod/crawler/Spider.java +++ b/app/src/main/java/com/github/catvod/crawler/Spider.java @@ -2,6 +2,8 @@ package com.github.catvod.crawler; import android.content.Context; +import org.json.JSONException; + import java.util.HashMap; import java.util.List; @@ -14,27 +16,27 @@ public abstract class Spider { init(context); } - public String homeContent(boolean filter) { + public String homeContent(boolean filter) throws JSONException { return ""; } - public String homeVideoContent() { + public String homeVideoContent() throws JSONException { return ""; } - public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) throws JSONException { return ""; } - public String detailContent(List ids) { + public String detailContent(List ids) throws JSONException { return ""; } - public String searchContent(String key, boolean quick) { + public String searchContent(String key, boolean quick) throws JSONException { return ""; } - public String playerContent(String flag, String id, List vipFlags) { + public String playerContent(String flag, String id, List vipFlags) throws JSONException { return ""; } } diff --git a/app/src/main/java/com/github/catvod/spider/Biubiu.java b/app/src/main/java/com/github/catvod/spider/Biubiu.java index 9fe41acd..de61547c 100644 --- a/app/src/main/java/com/github/catvod/spider/Biubiu.java +++ b/app/src/main/java/com/github/catvod/spider/Biubiu.java @@ -7,10 +7,12 @@ import com.github.catvod.bean.Class; import com.github.catvod.bean.Result; import com.github.catvod.bean.Vod; import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; import com.github.catvod.net.OkHttpUtil; import com.github.catvod.utils.Misc; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import java.util.ArrayList; @@ -21,7 +23,6 @@ import java.util.regex.Pattern; public class Biubiu extends Spider { - private String ext = null; private JSONObject rule = null; private HashMap getHeaders() { @@ -32,15 +33,23 @@ public class Biubiu extends Spider { return headers; } + private void fetchRule(String ext) { + try { + if (ext.startsWith("http")) rule = new JSONObject(OkHttpUtil.string(ext, null)); + else rule = new JSONObject(ext); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + @Override public void init(Context context, String extend) { super.init(context, extend); - this.ext = extend; + fetchRule(extend); } @Override public String homeContent(boolean filter) { - fetchRule(); List classes = new ArrayList<>(); String[] fenleis = getRuleVal("fenlei", "").split("#"); for (String fenlei : fenleis) { @@ -52,7 +61,6 @@ public class Biubiu extends Spider { @Override public String homeVideoContent() { - fetchRule(); if (getRuleVal("shouye").equals("1")) return ""; List videos = new ArrayList<>(); String[] fenleis = getRuleVal("fenlei", "").split("#"); @@ -68,7 +76,6 @@ public class Biubiu extends Spider { } private Result category(String tid, String pg) { - fetchRule(); String webUrl = getRuleVal("url") + tid + pg + getRuleVal("houzhui"); String html = fetch(webUrl); String parseContent = html; @@ -108,7 +115,6 @@ public class Biubiu extends Spider { @Override public String detailContent(List ids) { - fetchRule(); String[] idInfo = ids.get(0).split("\\$\\$\\$"); String webUrl = getRuleVal("url") + idInfo[2]; String html = fetch(webUrl); @@ -158,80 +164,62 @@ public class Biubiu extends Spider { @Override public String playerContent(String flag, String id, List vipFlags) { - fetchRule(); String webUrl = getRuleVal("url") + id; return Result.get().parse().url(webUrl).string(); } @Override - public String searchContent(String key, boolean quick) { - try { - fetchRule(); - boolean ssmoshiJson = getRuleVal("ssmoshi").equals("0"); - String webUrlTmp = getRuleVal("url") + getRuleVal("sousuoqian") + key + getRuleVal("sousuohou"); - String webUrl = webUrlTmp.split(";")[0]; - String webContent = webUrlTmp.contains(";post") ? fetchPost(webUrl) : fetch(webUrl); - List videos = new ArrayList<>(); - if (ssmoshiJson) { - JSONObject data = new JSONObject(webContent); - JSONArray vodArray = data.getJSONArray("list"); - for (int j = 0; j < vodArray.length(); j++) { - JSONObject vod = vodArray.getJSONObject(j); - String name = vod.optString(getRuleVal("jsname")).trim(); - String id = vod.optString(getRuleVal("jsid")).trim(); - String pic = vod.optString(getRuleVal("jspic")).trim(); + public String searchContent(String key, boolean quick) throws JSONException { + boolean ssmoshiJson = getRuleVal("ssmoshi").equals("0"); + String webUrlTmp = getRuleVal("url") + getRuleVal("sousuoqian") + key + getRuleVal("sousuohou"); + String webUrl = webUrlTmp.split(";")[0]; + String webContent = webUrlTmp.contains(";post") ? fetchPost(webUrl) : fetch(webUrl); + List videos = new ArrayList<>(); + if (ssmoshiJson) { + JSONObject data = new JSONObject(webContent); + JSONArray vodArray = data.getJSONArray("list"); + for (int j = 0; j < vodArray.length(); j++) { + JSONObject vod = vodArray.getJSONObject(j); + String name = vod.optString(getRuleVal("jsname")).trim(); + String id = vod.optString(getRuleVal("jsid")).trim(); + String pic = vod.optString(getRuleVal("jspic")).trim(); + pic = Misc.fixUrl(webUrl, pic); + Vod video = new Vod(); + video.setVodId(name + "$$$" + pic + "$$$" + getRuleVal("sousuohouzhui") + id); + video.setVodName(name); + video.setVodPic(pic); + videos.add(video); + } + } else { + String parseContent = webContent; + boolean shifouercijiequ = getRuleVal("sousuoshifouercijiequ").equals("1"); + if (shifouercijiequ) { + String jiequqian = getRuleVal("ssjiequqian"); + String jiequhou = getRuleVal("ssjiequhou"); + parseContent = subContent(webContent, jiequqian, jiequhou).get(0); + } + String jiequshuzuqian = getRuleVal("ssjiequshuzuqian"); + String jiequshuzuhou = getRuleVal("ssjiequshuzuhou"); + ArrayList jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou); + for (int i = 0; i < jiequContents.size(); i++) { + try { + String jiequContent = jiequContents.get(i); + String title = subContent(jiequContent, getRuleVal("ssbiaotiqian"), getRuleVal("ssbiaotihou")).get(0); + String pic = subContent(jiequContent, getRuleVal("sstupianqian"), getRuleVal("sstupianhou")).get(0); pic = Misc.fixUrl(webUrl, pic); + String link = subContent(jiequContent, getRuleVal("sslianjieqian"), getRuleVal("sslianjiehou")).get(0); Vod video = new Vod(); - video.setVodId(name + "$$$" + pic + "$$$" + getRuleVal("sousuohouzhui") + id); - video.setVodName(name); + video.setVodId(title + "$$$" + pic + "$$$" + link); + video.setVodName(title); video.setVodPic(pic); videos.add(video); - } - } else { - String parseContent = webContent; - boolean shifouercijiequ = getRuleVal("sousuoshifouercijiequ").equals("1"); - if (shifouercijiequ) { - String jiequqian = getRuleVal("ssjiequqian"); - String jiequhou = getRuleVal("ssjiequhou"); - parseContent = subContent(webContent, jiequqian, jiequhou).get(0); - } - String jiequshuzuqian = getRuleVal("ssjiequshuzuqian"); - String jiequshuzuhou = getRuleVal("ssjiequshuzuhou"); - ArrayList jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou); - for (int i = 0; i < jiequContents.size(); i++) { - try { - String jiequContent = jiequContents.get(i); - String title = subContent(jiequContent, getRuleVal("ssbiaotiqian"), getRuleVal("ssbiaotihou")).get(0); - String pic = subContent(jiequContent, getRuleVal("sstupianqian"), getRuleVal("sstupianhou")).get(0); - pic = Misc.fixUrl(webUrl, pic); - String link = subContent(jiequContent, getRuleVal("sslianjieqian"), getRuleVal("sslianjiehou")).get(0); - Vod video = new Vod(); - video.setVodId(title + "$$$" + pic + "$$$" + link); - video.setVodName(title); - video.setVodPic(pic); - videos.add(video); - } catch (Exception e) { - e.printStackTrace(); - break; - } + } catch (Exception e) { + e.printStackTrace(); + break; } } - return Result.string(videos); - } catch (Exception e) { - e.printStackTrace(); - return ""; - } - } - - private void fetchRule() { - try { - if (rule != null) return; - if (ext == null) return; - if (ext.startsWith("http")) rule = new JSONObject(OkHttpUtil.string(ext, null)); - else rule = new JSONObject(ext); - } catch (Exception e) { - e.printStackTrace(); } + return Result.string(videos); } private String fetch(String webUrl) { diff --git a/app/src/main/java/com/github/catvod/spider/XPath.java b/app/src/main/java/com/github/catvod/spider/XPath.java index ae3726ce..21978a33 100644 --- a/app/src/main/java/com/github/catvod/spider/XPath.java +++ b/app/src/main/java/com/github/catvod/spider/XPath.java @@ -13,6 +13,7 @@ import com.github.catvod.utils.Misc; import com.github.catvod.xpath.XPathRule; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.seimicrawler.xpath.JXDocument; import org.seimicrawler.xpath.JXNode; @@ -25,24 +26,33 @@ import java.util.Set; public class XPath extends Spider { - protected String ext = null; protected XPathRule rule = null; - protected HashMap getHeaders() { + private HashMap getHeaders() { HashMap headers = new HashMap<>(); headers.put("User-Agent", rule.getUa().isEmpty() ? Misc.CHROME : rule.getUa()); return headers; } - @Override - public void init(Context context, String extend) { - super.init(context, extend); - this.ext = extend; + private void fetchRule(String ext) { + if (ext.startsWith("http")) { + String json = OkHttpUtil.string(ext); + rule = XPathRule.fromJson(json); + loadRuleExt(json); + } else { + rule = XPathRule.fromJson(ext); + loadRuleExt(ext); + } } @Override - public String homeContent(boolean filter) { - fetchRule(); + public void init(Context context, String extend) { + super.init(context, extend); + fetchRule(extend); + } + + @Override + public String homeContent(boolean filter) throws JSONException { List videos = new ArrayList<>(); List classes = new ArrayList<>(); if (rule.getCateManual().size() > 0) { @@ -86,19 +96,12 @@ public class XPath extends Spider { return Result.string(classes, videos, rule.getFilter()); } - @Override - public String homeVideoContent() { - fetchRule(); - return ""; - } - protected String categoryUrl(String tid, String pg, boolean filter, HashMap extend) { return rule.getCateUrl().replace("{cateId}", tid).replace("{catePg}", pg); } @Override public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { - fetchRule(); String webUrl = categoryUrl(tid, pg, filter, extend); List videos = new ArrayList<>(); JXDocument doc = JXDocument.create(fetch(webUrl)); @@ -125,12 +128,8 @@ public class XPath extends Spider { return Result.string(videos); } - protected void detailContentExt(String content, Vod vod) { - } - @Override - public String detailContent(List ids) { - fetchRule(); + public String detailContent(List ids) throws JSONException { String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0)); String webContent = fetch(webUrl); JXDocument doc = JXDocument.create(webContent); @@ -254,90 +253,69 @@ public class XPath extends Spider { String vod_play_url = TextUtils.join("$$$", playList); vod.setVodPlayFrom(vod_play_from); vod.setVodPlayUrl(vod_play_url); - detailContentExt(webContent, vod); return Result.string(vod); } @Override public String playerContent(String flag, String id, List vipFlags) { - fetchRule(); String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id); SpiderDebug.log(webUrl); return Result.get().parse().url(webUrl).toString(); } @Override - public String searchContent(String key, boolean quick) { - try { - fetchRule(); - if (rule.getSearchUrl().isEmpty()) return ""; - String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key)); - String webContent = fetch(webUrl); - List videos = new ArrayList<>(); - if (rule.getSearchVodNode().startsWith("json:")) { - String[] node = rule.getSearchVodNode().substring(5).split(">"); - JSONObject data = new JSONObject(webContent); - for (int i = 0; i < node.length; i++) { - if (i == node.length - 1) { - JSONArray vodArray = data.getJSONArray(node[i]); - for (int j = 0; j < vodArray.length(); j++) { - JSONObject vod = vodArray.getJSONObject(j); - String name = vod.optString(rule.getSearchVodName()).trim(); - name = rule.getSearchVodNameR(name); - String id = vod.optString(rule.getSearchVodId()).trim(); - id = rule.getSearchVodIdR(id); - String pic = vod.optString(rule.getSearchVodImg()).trim(); - pic = rule.getSearchVodImgR(pic); - pic = Misc.fixUrl(webUrl, pic); - String mark = vod.optString(rule.getSearchVodMark()).trim(); - mark = rule.getSearchVodMarkR(mark); - videos.add(new Vod(id, name, pic, mark)); - } - } else { - data = data.getJSONObject(node[i]); + public String searchContent(String key, boolean quick) throws JSONException { + if (rule.getSearchUrl().isEmpty()) return ""; + String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key)); + String webContent = fetch(webUrl); + List videos = new ArrayList<>(); + if (rule.getSearchVodNode().startsWith("json:")) { + String[] node = rule.getSearchVodNode().substring(5).split(">"); + JSONObject data = new JSONObject(webContent); + for (int i = 0; i < node.length; i++) { + if (i == node.length - 1) { + JSONArray vodArray = data.getJSONArray(node[i]); + for (int j = 0; j < vodArray.length(); j++) { + JSONObject vod = vodArray.getJSONObject(j); + String name = vod.optString(rule.getSearchVodName()).trim(); + name = rule.getSearchVodNameR(name); + String id = vod.optString(rule.getSearchVodId()).trim(); + id = rule.getSearchVodIdR(id); + String pic = vod.optString(rule.getSearchVodImg()).trim(); + pic = rule.getSearchVodImgR(pic); + pic = Misc.fixUrl(webUrl, pic); + String mark = vod.optString(rule.getSearchVodMark()).trim(); + mark = rule.getSearchVodMarkR(mark); + videos.add(new Vod(id, name, pic, mark)); } - } - } else { - JXDocument doc = JXDocument.create(webContent); - List vodNodes = doc.selN(rule.getSearchVodNode()); - for (int i = 0; i < vodNodes.size(); i++) { - String name = vodNodes.get(i).selOne(rule.getSearchVodName()).asString().trim(); - name = rule.getSearchVodNameR(name); - String id = vodNodes.get(i).selOne(rule.getSearchVodId()).asString().trim(); - id = rule.getSearchVodIdR(id); - String pic = vodNodes.get(i).selOne(rule.getSearchVodImg()).asString().trim(); - pic = rule.getSearchVodImgR(pic); - pic = Misc.fixUrl(webUrl, pic); - String mark = ""; - if (!rule.getCateVodMark().isEmpty()) { - try { - mark = vodNodes.get(i).selOne(rule.getSearchVodMark()).asString().trim(); - mark = rule.getSearchVodMarkR(mark); - } catch (Exception e) { - SpiderDebug.log(e); - } - } - videos.add(new Vod(id, name, pic, mark)); + } else { + data = data.getJSONObject(node[i]); } } - return Result.string(videos); - } catch (Exception e) { - SpiderDebug.log(e); - return ""; - } - } - - protected void fetchRule() { - if (rule == null && ext != null) { - if (ext.startsWith("http")) { - String json = OkHttpUtil.string(ext); - rule = XPathRule.fromJson(json); - loadRuleExt(json); - } else { - rule = XPathRule.fromJson(ext); - loadRuleExt(ext); + } else { + JXDocument doc = JXDocument.create(webContent); + List vodNodes = doc.selN(rule.getSearchVodNode()); + for (int i = 0; i < vodNodes.size(); i++) { + String name = vodNodes.get(i).selOne(rule.getSearchVodName()).asString().trim(); + name = rule.getSearchVodNameR(name); + String id = vodNodes.get(i).selOne(rule.getSearchVodId()).asString().trim(); + id = rule.getSearchVodIdR(id); + String pic = vodNodes.get(i).selOne(rule.getSearchVodImg()).asString().trim(); + pic = rule.getSearchVodImgR(pic); + pic = Misc.fixUrl(webUrl, pic); + String mark = ""; + if (!rule.getCateVodMark().isEmpty()) { + try { + mark = vodNodes.get(i).selOne(rule.getSearchVodMark()).asString().trim(); + mark = rule.getSearchVodMarkR(mark); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + videos.add(new Vod(id, name, pic, mark)); } } + return Result.string(videos); } protected void loadRuleExt(String json) { diff --git a/app/src/main/java/com/github/catvod/spider/XPathMac.java b/app/src/main/java/com/github/catvod/spider/XPathMac.java index af300407..5d0f3759 100644 --- a/app/src/main/java/com/github/catvod/spider/XPathMac.java +++ b/app/src/main/java/com/github/catvod/spider/XPathMac.java @@ -48,61 +48,46 @@ public class XPathMac extends XPath { } playerConfigJs = jsonObj.optString("pCfgJs").trim(); playerConfigJsRegex = jsonObj.optString("pCfgJsR", playerConfigJsRegex).trim(); - } catch (JSONException e) { - SpiderDebug.log(e); - } - } - - @Override - public String homeContent(boolean filter) { - String result = super.homeContent(filter); - if (result.length() > 0 && playerConfigJs.length() > 0) { //嘗試通過playerConfigJs獲取展示和flag匹配關系 - String webContent = fetch(playerConfigJs); - Matcher matcher = Pattern.compile(playerConfigJsRegex).matcher(webContent); - if (!matcher.find()) return result; - try { - JSONObject jsonObject = new JSONObject(matcher.group(1)); - Iterator keys = jsonObject.keys(); - while (keys.hasNext()) { - String key = keys.next(); - JSONObject keyObj = jsonObject.optJSONObject(key); - if (keyObj == null) continue; - String show = keyObj.optString("show").trim(); - if (show.isEmpty()) continue; - show2VipFlag.put(show, key); - } - } catch (Exception e) { - SpiderDebug.log(e); - } - } - return result; - } - - @Override - public String detailContent(List ids) { - String result = super.detailContent(ids); - if (!decodeVipFlag || result.isEmpty()) return result; - try { - JSONObject jsonObject = new JSONObject(result); - String[] playFrom = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$"); - if (playFrom.length > 0) { - for (int i = 0; i < playFrom.length; i++) { - if (show2VipFlag.containsKey(playFrom[i])) { - playFrom[i] = show2VipFlag.get(playFrom[i]); - } - } - jsonObject.optJSONArray("list").getJSONObject(0).put("vod_play_from", TextUtils.join("$$$", playFrom)); - result = jsonObject.toString(); - } } catch (Exception e) { SpiderDebug.log(e); } + } + + @Override + public String homeContent(boolean filter) throws JSONException { + String result = super.homeContent(filter); + if (result.isEmpty() || playerConfigJs.isEmpty()) return result; + //嘗試通過playerConfigJs獲取展示和flag匹配關系 + String webContent = fetch(playerConfigJs); + Matcher matcher = Pattern.compile(playerConfigJsRegex).matcher(webContent); + if (!matcher.find()) return result; + JSONObject jsonObject = new JSONObject(matcher.group(1)); + Iterator keys = jsonObject.keys(); + while (keys.hasNext()) { + String key = keys.next(); + JSONObject keyObj = jsonObject.optJSONObject(key); + if (keyObj == null) continue; + String show = keyObj.optString("show").trim(); + if (show.isEmpty()) continue; + show2VipFlag.put(show, key); + } return result; } + @Override + public String detailContent(List ids) throws JSONException { + String result = super.detailContent(ids); + if (!decodeVipFlag || result.isEmpty()) return result; + JSONObject jsonObject = new JSONObject(result); + String[] playFrom = jsonObject.optJSONArray("list").getJSONObject(0).optString("vod_play_from").split("\\$\\$\\$"); + if (playFrom.length == 0) return result; + for (int i = 0; i < playFrom.length; i++) if (show2VipFlag.containsKey(playFrom[i])) playFrom[i] = show2VipFlag.get(playFrom[i]); + jsonObject.optJSONArray("list").getJSONObject(0).put("vod_play_from", TextUtils.join("$$$", playFrom)); + return jsonObject.toString(); + } + @Override public String playerContent(String flag, String id, List vipFlags) { - fetchRule(); String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id); String videoUrl = null; // 嘗試分析直連