diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index ce1be81d..b6f6c69f 100644 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -11,7 +11,7 @@ -keepattributes Signature -keepattributes *Annotation* -dontwarn sun.misc.** --keep class com.google.gson.examples.android.model.** { ; } +-keep class com.google.gson.**{*;} -keep class * extends com.google.gson.TypeAdapter -keep class * implements com.google.gson.TypeAdapterFactory -keep class * implements com.google.gson.JsonSerializer @@ -21,5 +21,5 @@ -keep,allowobfuscation,allowshrinking class * extends com.google.gson.reflect.TypeToken # OkHttp --dontwarn okhttp3.** --keep class okhttp3.** { *; } +-keep class okio.**{*;} +-keep class okhttp3.**{*;} diff --git a/app/src/main/java/com/github/catvod/bean/Class.java b/app/src/main/java/com/github/catvod/bean/Class.java new file mode 100644 index 00000000..90e643cc --- /dev/null +++ b/app/src/main/java/com/github/catvod/bean/Class.java @@ -0,0 +1,28 @@ +package com.github.catvod.bean; + +import com.google.gson.annotations.SerializedName; + +public class Class { + + @SerializedName("type_id") + private String typeId; + @SerializedName("type_name") + private String typeName; + + public Class(String typeId, String typeName) { + this.typeId = typeId; + this.typeName = typeName; + } + + public String getTypeId() { + return typeId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!(obj instanceof Class)) return false; + Class it = (Class) obj; + return getTypeId().equals(it.getTypeId()); + } +} diff --git a/app/src/main/java/com/github/catvod/bean/Result.java b/app/src/main/java/com/github/catvod/bean/Result.java new file mode 100644 index 00000000..4a3ad254 --- /dev/null +++ b/app/src/main/java/com/github/catvod/bean/Result.java @@ -0,0 +1,37 @@ +package com.github.catvod.bean; + +import com.google.gson.Gson; +import com.google.gson.annotations.SerializedName; + +import org.jetbrains.annotations.NotNull; +import org.json.JSONObject; + +import java.util.List; + +public class Result { + + @SerializedName("class") + private List classes; + @SerializedName("list") + private List list; + @SerializedName("filters") + private JSONObject filters; + + public void setClasses(List classes) { + this.classes = classes; + } + + public void setList(List list) { + this.list = list; + } + + public void setFilters(JSONObject filters) { + this.filters = filters; + } + + @NotNull + @Override + public String toString() { + return new Gson().toJson(this); + } +} diff --git a/app/src/main/java/com/github/catvod/bean/Vod.java b/app/src/main/java/com/github/catvod/bean/Vod.java new file mode 100644 index 00000000..c94cc80d --- /dev/null +++ b/app/src/main/java/com/github/catvod/bean/Vod.java @@ -0,0 +1,22 @@ +package com.github.catvod.bean; + +import com.google.gson.annotations.SerializedName; + +public class Vod { + + @SerializedName("vod_id") + private String vodId; + @SerializedName("vod_name") + private String vodName; + @SerializedName("vod_pic") + private String vodPic; + @SerializedName("vod_remarks") + private String vodRemarks; + + public Vod(String vodId, String vodName, String vodPic, String vodRemarks) { + this.vodId = vodId; + this.vodName = vodName; + this.vodPic = vodPic; + this.vodRemarks = vodRemarks; + } +} diff --git a/app/src/main/java/com/github/catvod/spider/XPath.java b/app/src/main/java/com/github/catvod/spider/XPath.java new file mode 100644 index 00000000..f17c7af0 --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/XPath.java @@ -0,0 +1,439 @@ +package com.github.catvod.spider; + +import android.content.Context; +import android.text.TextUtils; + +import com.github.catvod.bean.Class; +import com.github.catvod.bean.Result; +import com.github.catvod.bean.Vod; +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.net.OkHttpUtil; +import com.github.catvod.utils.Misc; +import com.github.catvod.utils.Trans; +import com.github.catvod.xpath.XPathRule; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.seimicrawler.xpath.JXDocument; +import org.seimicrawler.xpath.JXNode; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Set; + +public class XPath extends Spider { + + protected String ext = null; + protected XPathRule rule = null; + + protected HashMap getHeaders() { + HashMap headers = new HashMap<>(); + headers.put("User-Agent", rule.getUa().isEmpty() ? Misc.CHROME : rule.getUa()); + return headers; + } + + @Override + public void init(Context context, String extend) { + super.init(context, extend); + this.ext = extend; + } + + @Override + public String homeContent(boolean filter) { + try { + fetchRule(); + Result result = new Result(); + List videos = new ArrayList<>(); + List classes = new ArrayList<>(); + if (rule.getCateManual().size() > 0) { + Set keys = rule.getCateManual().keySet(); + for (String k : keys) classes.add(new Class(k, rule.getCateManual().get(k))); + } + try { + String webUrl = rule.getHomeUrl(); + JXDocument doc = JXDocument.create(fetch(webUrl)); + if (rule.getCateManual().size() == 0) { + List navNodes = doc.selN(rule.getCateNode()); + for (int i = 0; i < navNodes.size(); i++) { + String name = navNodes.get(i).selOne(rule.getCateName()).asString().trim(); + name = rule.getCateNameR(name); + String id = navNodes.get(i).selOne(rule.getCateId()).asString().trim(); + id = rule.getCateIdR(id); + classes.add(new Class(id, name)); + } + } + if (!rule.getHomeVodNode().isEmpty()) { + try { + List vodNodes = doc.selN(rule.getHomeVodNode()); + for (int i = 0; i < vodNodes.size(); i++) { + String name = vodNodes.get(i).selOne(rule.getHomeVodName()).asString().trim(); + name = rule.getHomeVodNameR(name); + String id = vodNodes.get(i).selOne(rule.getHomeVodId()).asString().trim(); + id = rule.getHomeVodIdR(id); + String pic = vodNodes.get(i).selOne(rule.getHomeVodImg()).asString().trim(); + pic = rule.getHomeVodImgR(pic); + pic = Misc.fixUrl(webUrl, pic); + String mark = ""; + if (!rule.getHomeVodMark().isEmpty()) { + try { + mark = vodNodes.get(i).selOne(rule.getHomeVodMark()).asString().trim(); + mark = rule.getHomeVodMarkR(mark); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + videos.add(new Vod(id, name, pic, mark)); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + } + } catch (Exception e) { + SpiderDebug.log(e); + } + result.setList(videos); + result.setClasses(classes); + result.setFilters(filter && rule.getFilter() != null ? rule.getFilter() : null); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + return ""; + } + } + + @Override + public String homeVideoContent() { + try { + fetchRule(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected String categoryUrl(String tid, String pg, boolean filter, HashMap extend) { + return rule.getCateUrl().replace("{cateId}", tid).replace("{catePg}", pg); + } + + @Override + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + try { + fetchRule(); + String webUrl = categoryUrl(tid, pg, filter, extend); + JSONArray videos = new JSONArray(); + JXDocument doc = JXDocument.create(fetch(webUrl)); + List vodNodes = doc.selN(rule.getCateVodNode()); + for (int i = 0; i < vodNodes.size(); i++) { + String name = vodNodes.get(i).selOne(rule.getCateVodName()).asString().trim(); + name = rule.getCateVodNameR(name); + String id = vodNodes.get(i).selOne(rule.getCateVodId()).asString().trim(); + id = rule.getCateVodIdR(id); + String pic = vodNodes.get(i).selOne(rule.getCateVodImg()).asString().trim(); + pic = rule.getCateVodImgR(pic); + pic = Misc.fixUrl(webUrl, pic); + String mark = ""; + if (!rule.getCateVodMark().isEmpty()) { + try { + mark = vodNodes.get(i).selOne(rule.getCateVodMark()).asString().trim(); + mark = rule.getCateVodMarkR(mark); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", Trans.get(name)); + v.put("vod_pic", pic); + v.put("vod_remarks", Trans.get(mark)); + videos.put(v); + } + JSONObject result = new JSONObject(); + result.put("page", pg); + result.put("pagecount", Integer.MAX_VALUE); + result.put("limit", 90); + result.put("total", Integer.MAX_VALUE); + result.put("list", videos); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected void detailContentExt(String content, JSONObject vod) { + + } + + @Override + public String detailContent(List ids) { + try { + fetchRule(); + String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0)); + String webContent = fetch(webUrl); + JXDocument doc = JXDocument.create(webContent); + JXNode vodNode = doc.selNOne(rule.getDetailNode()); + + String cover = "", title = "", desc = "", category = "", area = "", year = "", remark = "", director = "", actor = ""; + + title = vodNode.selOne(rule.getDetailName()).asString().trim(); + title = rule.getDetailNameR(title); + + if (!rule.getDetailImg().isEmpty()) { + try { + cover = vodNode.selOne(rule.getDetailImg()).asString().trim(); + cover = rule.getDetailImgR(cover); + cover = Misc.fixUrl(webUrl, cover); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailCate().isEmpty()) { + try { + category = vodNode.selOne(rule.getDetailCate()).asString().trim(); + category = rule.getDetailCateR(category); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailYear().isEmpty()) { + try { + year = vodNode.selOne(rule.getDetailYear()).asString().trim(); + year = rule.getDetailYearR(year); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailArea().isEmpty()) { + try { + area = vodNode.selOne(rule.getDetailArea()).asString().trim(); + area = rule.getDetailAreaR(area); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailMark().isEmpty()) { + try { + remark = vodNode.selOne(rule.getDetailMark()).asString().trim(); + remark = rule.getDetailMarkR(remark); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailActor().isEmpty()) { + try { + actor = vodNode.selOne(rule.getDetailActor()).asString().trim(); + actor = rule.getDetailActorR(actor); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailDirector().isEmpty()) { + try { + director = vodNode.selOne(rule.getDetailDirector()).asString().trim(); + director = rule.getDetailDirectorR(director); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!rule.getDetailDesc().isEmpty()) { + try { + desc = vodNode.selOne(rule.getDetailDesc()).asString().trim(); + desc = rule.getDetailDescR(desc); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + + JSONObject vod = new JSONObject(); + vod.put("vod_id", ids.get(0)); + vod.put("vod_name", Trans.get(title)); + vod.put("vod_pic", cover); + vod.put("type_name", Trans.get(category)); + vod.put("vod_year", year); + vod.put("vod_area", Trans.get(area)); + vod.put("vod_remarks", Trans.get(remark)); + vod.put("vod_actor", Trans.get(actor)); + vod.put("vod_director", Trans.get(director)); + vod.put("vod_content", Trans.get(desc)); + + ArrayList playFrom = new ArrayList<>(); + List fromNodes = doc.selN(rule.getDetailFromNode()); + for (int i = 0; i < fromNodes.size(); i++) { + String name = fromNodes.get(i).selOne(rule.getDetailFromName()).asString().trim(); + name = rule.getDetailFromNameR(name); + playFrom.add(name); + } + + ArrayList playList = new ArrayList<>(); + List urlListNodes = doc.selN(rule.getDetailUrlNode()); + for (int i = 0; i < urlListNodes.size(); i++) { + List urlNodes = urlListNodes.get(i).sel(rule.getDetailUrlSubNode()); + List vodItems = new ArrayList<>(); + for (int j = 0; j < urlNodes.size(); j++) { + String name = urlNodes.get(j).selOne(rule.getDetailUrlName()).asString().trim(); + name = rule.getDetailUrlNameR(name); + String id = urlNodes.get(j).selOne(rule.getDetailUrlId()).asString().trim(); + id = rule.getDetailUrlIdR(id); + vodItems.add(Trans.get(name) + "$" + id); + } + // 排除播放列表为空的播放源 + if (vodItems.size() == 0 && playFrom.size() > i) { + playFrom.set(i, ""); + } + playList.add(TextUtils.join("#", vodItems)); + } + // 排除播放列表为空的播放源 + for (int i = playFrom.size() - 1; i >= 0; i--) { + if (playFrom.get(i).isEmpty()) + playFrom.remove(i); + } + for (int i = playList.size() - 1; i >= 0; i--) { + if (playList.get(i).isEmpty()) + playList.remove(i); + } + for (int i = playList.size() - 1; i >= 0; i--) { + if (i >= playFrom.size()) + playList.remove(i); + } + String vod_play_from = TextUtils.join("$$$", playFrom); + String vod_play_url = TextUtils.join("$$$", playList); + vod.put("vod_play_from", Trans.get(vod_play_from)); + vod.put("vod_play_url", vod_play_url); + detailContentExt(webContent, vod); + JSONObject result = new JSONObject(); + JSONArray list = new JSONArray(); + list.put(vod); + result.put("list", list); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e.getMessage()); + for (StackTraceElement traceEl : e.getStackTrace()) { + SpiderDebug.log(traceEl.toString()); + } + } + return ""; + } + + @Override + public String playerContent(String flag, String id, List vipFlags) { + try { + fetchRule(); + String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id); + SpiderDebug.log(webUrl); + JSONObject result = new JSONObject(); + result.put("parse", "1"); + if (!rule.getPlayUa().isEmpty()) { + result.put("ua", rule.getPlayUa()); + } + result.put("url", webUrl); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + @Override + public String searchContent(String key, boolean quick) { + try { + fetchRule(); + if (rule.getSearchUrl().isEmpty()) { + return ""; + } + String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key)); + String webContent = fetch(webUrl); + JSONObject result = new JSONObject(); + JSONArray videos = new JSONArray(); + // add maccms suggest search api support + if (rule.getSearchVodNode().startsWith("json:")) { + String[] node = rule.getSearchVodNode().substring(5).split(">"); + JSONObject data = new JSONObject(webContent); + for (int i = 0; i < node.length; i++) { + if (i == node.length - 1) { + JSONArray vodArray = data.getJSONArray(node[i]); + for (int j = 0; j < vodArray.length(); j++) { + JSONObject vod = vodArray.getJSONObject(j); + String name = vod.optString(rule.getSearchVodName()).trim(); + name = rule.getSearchVodNameR(name); + String id = vod.optString(rule.getSearchVodId()).trim(); + id = rule.getSearchVodIdR(id); + String pic = vod.optString(rule.getSearchVodImg()).trim(); + pic = rule.getSearchVodImgR(pic); + pic = Misc.fixUrl(webUrl, pic); + String mark = vod.optString(rule.getSearchVodMark()).trim(); + mark = rule.getSearchVodMarkR(mark); + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", Trans.get(name)); + v.put("vod_pic", pic); + v.put("vod_remarks", Trans.get(mark)); + videos.put(v); + } + } else { + data = data.getJSONObject(node[i]); + } + } + } else { + JXDocument doc = JXDocument.create(webContent); + List vodNodes = doc.selN(rule.getSearchVodNode()); + for (int i = 0; i < vodNodes.size(); i++) { + String name = vodNodes.get(i).selOne(rule.getSearchVodName()).asString().trim(); + name = rule.getSearchVodNameR(name); + String id = vodNodes.get(i).selOne(rule.getSearchVodId()).asString().trim(); + id = rule.getSearchVodIdR(id); + String pic = vodNodes.get(i).selOne(rule.getSearchVodImg()).asString().trim(); + pic = rule.getSearchVodImgR(pic); + pic = Misc.fixUrl(webUrl, pic); + String mark = ""; + if (!rule.getCateVodMark().isEmpty()) { + try { + mark = vodNodes.get(i).selOne(rule.getSearchVodMark()).asString().trim(); + mark = rule.getSearchVodMarkR(mark); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", Trans.get(name)); + v.put("vod_pic", pic); + v.put("vod_remarks", Trans.get(mark)); + videos.put(v); + } + } + result.put("list", videos); + return result.toString(); + } catch ( + Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected void fetchRule() { + if (rule == null) { + if (ext != null) { + if (ext.startsWith("http")) { + String json = OkHttpUtil.string(ext); + rule = XPathRule.fromJson(json); + loadRuleExt(json); + } else { + rule = XPathRule.fromJson(ext); + loadRuleExt(ext); + } + } + } + } + + protected void loadRuleExt(String json) { + + } + + protected String fetch(String webUrl) { + SpiderDebug.log(webUrl); + return OkHttpUtil.string(webUrl, getHeaders()); + } +} diff --git a/jar/custom_spider.jar b/jar/custom_spider.jar index 982fb153..14aa764b 100644 Binary files a/jar/custom_spider.jar and b/jar/custom_spider.jar differ diff --git a/jar/custom_spider.jar.md5 b/jar/custom_spider.jar.md5 index 51d124aa..5383e0ef 100644 --- a/jar/custom_spider.jar.md5 +++ b/jar/custom_spider.jar.md5 @@ -1 +1 @@ -e79ee1c89df18b03332b1f342ca44d2f +1012798d28ebd672ba6007e6d6fab951