diff --git a/app/src/main/java/com/github/catvod/spider/Bdys01.java b/app/src/main/java/com/github/catvod/spider/Bdys01.java new file mode 100644 index 00000000..21008690 --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/Bdys01.java @@ -0,0 +1,559 @@ +package com.github.catvod.spider; + +import android.annotation.TargetApi; +import android.content.Context; +import android.os.Build; +import android.text.TextUtils; +import android.util.Base64; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.utils.Misc; +import com.github.catvod.utils.gZip; +import com.github.catvod.utils.okhttp.OKCallBack; +import com.github.catvod.utils.okhttp.OkHttpUtil; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; + +import javax.crypto.Cipher; +import javax.crypto.spec.SecretKeySpec; + +import okhttp3.Call; +import okhttp3.Response; + + +public class Bdys01 extends Spider { + private static final String siteUrl = "https://www.bdys01.com"; + private static final String siteHost = "www.bdys01.com"; + private String cookie=""; + private String referer=""; + + /** + * 筛选配置 + */ + private JSONObject filterConfig; + + + @Override + public void init(Context context) { + super.init(context); + try { + filterConfig = new JSONObject("{\"0\":[{\"key\":\"s\",\"name\":\"类型\",\"value\":[{\"n\":\"全部\",\"v\":\"all\"},{\"n\":\"动作\",\"v\":\"dongzuo\"},{\"n\":\"爱情\",\"v\":\"aiqing\"},{\"n\":\"喜剧\",\"v\":\"xiju\"},{\"n\":\"科幻\",\"v\":\"kehuan\"},{\"n\":\"恐怖\",\"v\":\"kongbu\"},{\"n\":\"战争\",\"v\":\"zhanzheng\"},{\"n\":\"武侠\",\"v\":\"wuxia\"},{\"n\":\"魔幻\",\"v\":\"mohuan\"},{\"n\":\"剧情\",\"v\":\"juqing\"},{\"n\":\"动画\",\"v\":\"donghua\"},{\"n\":\"惊悚\",\"v\":\"jingsong\"},{\"n\":\"3D\",\"v\":\"3D\"},{\"n\":\"灾难\",\"v\":\"zainan\"},{\"n\":\"悬疑\",\"v\":\"xuanyi\"},{\"n\":\"警匪\",\"v\":\"jingfei\"},{\"n\":\"文艺\",\"v\":\"wenyi\"},{\"n\":\"青春\",\"v\":\"qingchun\"},{\"n\":\"冒险\",\"v\":\"maoxian\"},{\"n\":\"犯罪\",\"v\":\"fanzui\"},{\"n\":\"纪录\",\"v\":\"jilu\"},{\"n\":\"古装\",\"v\":\"guzhuang\"},{\"n\":\"奇幻\",\"v\":\"qihuan\"},{\"n\":\"国语\",\"v\":\"guoyu\"},{\"n\":\"综艺\",\"v\":\"zongyi\"},{\"n\":\"历史\",\"v\":\"lishi\"},{\"n\":\"运动\",\"v\":\"yundong\"},{\"n\":\"原创压制\",\"v\":\"yuanchuang\"},{\"n\":\"美剧\",\"v\":\"meiju\"},{\"n\":\"韩剧\",\"v\":\"hanju\"},{\"n\":\"国产电视剧\",\"v\":\"guoju\"},{\"n\":\"日剧\",\"v\":\"riju\"},{\"n\":\"英剧\",\"v\":\"yingju\"},{\"n\":\"德剧\",\"v\":\"deju\"},{\"n\":\"俄剧\",\"v\":\"eju\"},{\"n\":\"巴剧\",\"v\":\"baju\"},{\"n\":\"加剧\",\"v\":\"jiaju\"},{\"n\":\"西剧\",\"v\":\"anish\"},{\"n\":\"意大利剧\",\"v\":\"yidaliju\"},{\"n\":\"泰剧\",\"v\":\"taiju\"},{\"n\":\"港台剧\",\"v\":\"gangtaiju\"},{\"n\":\"法剧\",\"v\":\"faju\"},{\"n\":\"澳剧\",\"v\":\"aoju\"}]},{\"key\":\"area\",\"name\":\"地区\",\"value\":[{\"n\":\"不限\",\"v\":\"\"},{\"n\":\"中国大陆\",\"v\":\"中国大陆\"},{\"n\":\"中国香港\",\"v\":\"中国香港\"},{\"n\":\"中国台湾\",\"v\":\"中国台湾\"},{\"n\":\"美国\",\"v\":\"美国\"},{\"n\":\"英国\",\"v\":\"英国\"},{\"n\":\"日本\",\"v\":\"日本\"},{\"n\":\"韩国\",\"v\":\"韩国\"},{\"n\":\"法国\",\"v\":\"法国\"},{\"n\":\"印度\",\"v\":\"印度\"},{\"n\":\"德国\",\"v\":\"德国\"},{\"n\":\"西班牙\",\"v\":\"西班牙\"},{\"n\":\"意大利\",\"v\":\"意大利\"},{\"n\":\"澳大利亚\",\"v\":\"澳大利亚\"},{\"n\":\"比利时\",\"v\":\"比利时\"},{\"n\":\"瑞典\",\"v\":\"瑞典\"},{\"n\":\"荷兰\",\"v\":\"荷兰\"},{\"n\":\"丹麦\",\"v\":\"丹麦\"},{\"n\":\"加拿大\",\"v\":\"加拿大\"},{\"n\":\"俄罗斯\",\"v\":\"俄罗斯\"}]},{\"key\":\"year\",\"name\":\"年份\",\"value\":[{\"n\":\"全部\",\"v\":\"\"},{\"n\":\"2023\",\"v\":\"2023\"},{\"n\":\"2022\",\"v\":\"2022\"},{\"n\":\"2021\",\"v\":\"2021\"},{\"n\":\"2020\",\"v\":\"2020\"},{\"n\":\"2019\",\"v\":\"2019\"},{\"n\":\"2018\",\"v\":\"2018\"},{\"n\":\"2017\",\"v\":\"2017\"},{\"n\":\"2016\",\"v\":\"2016\"},{\"n\":\"2015\",\"v\":\"2015\"},{\"n\":\"2014\",\"v\":\"2014\"},{\"n\":\"2013\",\"v\":\"2013\"},{\"n\":\"2012\",\"v\":\"2012\"},{\"n\":\"2011\",\"v\":\"2011\"},{\"n\":\"2010\",\"v\":\"2010\"},{\"n\":\"2009\",\"v\":\"2009\"},{\"n\":\"2008\",\"v\":\"2008\"},{\"n\":\"2007\",\"v\":\"2007\"},{\"n\":\"2006\",\"v\":\"2006\"},{\"n\":\"2005\",\"v\":\"2005\"},{\"n\":\"2004\",\"v\":\"2004\"},{\"n\":\"2003\",\"v\":\"2003\"},{\"n\":\"2002\",\"v\":\"2002\"},{\"n\":\"2001\",\"v\":\"2001\"},{\"n\":\"2000\",\"v\":\"2000\"}]},{\"key\":\"order\",\"name\":\"排序\",\"value\":[{\"n\":\"更新时间\",\"v\":\"0\"},{\"n\":\"豆瓣评分\",\"v\":\"1\"}]}],\"1\":[{\"key\":\"s\",\"name\":\"类型\",\"value\":[{\"n\":\"全部\",\"v\":\"all\"},{\"n\":\"动作\",\"v\":\"dongzuo\"},{\"n\":\"爱情\",\"v\":\"aiqing\"},{\"n\":\"喜剧\",\"v\":\"xiju\"},{\"n\":\"科幻\",\"v\":\"kehuan\"},{\"n\":\"恐怖\",\"v\":\"kongbu\"},{\"n\":\"战争\",\"v\":\"zhanzheng\"},{\"n\":\"武侠\",\"v\":\"wuxia\"},{\"n\":\"魔幻\",\"v\":\"mohuan\"},{\"n\":\"剧情\",\"v\":\"juqing\"},{\"n\":\"动画\",\"v\":\"donghua\"},{\"n\":\"惊悚\",\"v\":\"jingsong\"},{\"n\":\"3D\",\"v\":\"3D\"},{\"n\":\"灾难\",\"v\":\"zainan\"},{\"n\":\"悬疑\",\"v\":\"xuanyi\"},{\"n\":\"警匪\",\"v\":\"jingfei\"},{\"n\":\"文艺\",\"v\":\"wenyi\"},{\"n\":\"青春\",\"v\":\"qingchun\"},{\"n\":\"冒险\",\"v\":\"maoxian\"},{\"n\":\"犯罪\",\"v\":\"fanzui\"},{\"n\":\"纪录\",\"v\":\"jilu\"},{\"n\":\"古装\",\"v\":\"guzhuang\"},{\"n\":\"奇幻\",\"v\":\"qihuan\"},{\"n\":\"国语\",\"v\":\"guoyu\"},{\"n\":\"综艺\",\"v\":\"zongyi\"},{\"n\":\"历史\",\"v\":\"lishi\"},{\"n\":\"运动\",\"v\":\"yundong\"},{\"n\":\"原创压制\",\"v\":\"yuanchuang\"},{\"n\":\"美剧\",\"v\":\"meiju\"},{\"n\":\"韩剧\",\"v\":\"hanju\"},{\"n\":\"国产电视剧\",\"v\":\"guoju\"},{\"n\":\"日剧\",\"v\":\"riju\"},{\"n\":\"英剧\",\"v\":\"yingju\"},{\"n\":\"德剧\",\"v\":\"deju\"},{\"n\":\"俄剧\",\"v\":\"eju\"},{\"n\":\"巴剧\",\"v\":\"baju\"},{\"n\":\"加剧\",\"v\":\"jiaju\"},{\"n\":\"西剧\",\"v\":\"anish\"},{\"n\":\"意大利剧\",\"v\":\"yidaliju\"},{\"n\":\"泰剧\",\"v\":\"taiju\"},{\"n\":\"港台剧\",\"v\":\"gangtaiju\"},{\"n\":\"法剧\",\"v\":\"faju\"},{\"n\":\"澳剧\",\"v\":\"aoju\"}]},{\"key\":\"area\",\"name\":\"地区\",\"value\":[{\"n\":\"不限\",\"v\":\"\"},{\"n\":\"中国大陆\",\"v\":\"中国大陆\"},{\"n\":\"中国香港\",\"v\":\"中国香港\"},{\"n\":\"中国台湾\",\"v\":\"中国台湾\"},{\"n\":\"美国\",\"v\":\"美国\"},{\"n\":\"英国\",\"v\":\"英国\"},{\"n\":\"日本\",\"v\":\"日本\"},{\"n\":\"韩国\",\"v\":\"韩国\"},{\"n\":\"法国\",\"v\":\"法国\"},{\"n\":\"印度\",\"v\":\"印度\"},{\"n\":\"德国\",\"v\":\"德国\"},{\"n\":\"西班牙\",\"v\":\"西班牙\"},{\"n\":\"意大利\",\"v\":\"意大利\"},{\"n\":\"澳大利亚\",\"v\":\"澳大利亚\"},{\"n\":\"比利时\",\"v\":\"比利时\"},{\"n\":\"瑞典\",\"v\":\"瑞典\"},{\"n\":\"荷兰\",\"v\":\"荷兰\"},{\"n\":\"丹麦\",\"v\":\"丹麦\"},{\"n\":\"加拿大\",\"v\":\"加拿大\"},{\"n\":\"俄罗斯\",\"v\":\"俄罗斯\"}]},{\"key\":\"year\",\"name\":\"年份\",\"value\":[{\"n\":\"全部\",\"v\":\"\"},{\"n\":\"2023\",\"v\":\"2023\"},{\"n\":\"2022\",\"v\":\"2022\"},{\"n\":\"2021\",\"v\":\"2021\"},{\"n\":\"2020\",\"v\":\"2020\"},{\"n\":\"2019\",\"v\":\"2019\"},{\"n\":\"2018\",\"v\":\"2018\"},{\"n\":\"2017\",\"v\":\"2017\"},{\"n\":\"2016\",\"v\":\"2016\"},{\"n\":\"2015\",\"v\":\"2015\"},{\"n\":\"2014\",\"v\":\"2014\"},{\"n\":\"2013\",\"v\":\"2013\"},{\"n\":\"2012\",\"v\":\"2012\"},{\"n\":\"2011\",\"v\":\"2011\"},{\"n\":\"2010\",\"v\":\"2010\"},{\"n\":\"2009\",\"v\":\"2009\"},{\"n\":\"2008\",\"v\":\"2008\"},{\"n\":\"2007\",\"v\":\"2007\"},{\"n\":\"2006\",\"v\":\"2006\"},{\"n\":\"2005\",\"v\":\"2005\"},{\"n\":\"2004\",\"v\":\"2004\"},{\"n\":\"2003\",\"v\":\"2003\"},{\"n\":\"2002\",\"v\":\"2002\"},{\"n\":\"2001\",\"v\":\"2001\"},{\"n\":\"2000\",\"v\":\"2000\"}]},{\"key\":\"order\",\"name\":\"排序\",\"value\":[{\"n\":\"更新时间\",\"v\":\"0\"},{\"n\":\"豆瓣评分\",\"v\":\"1\"}]}],\"a\":[{\"key\":\"s\",\"name\":\"类型\",\"value\":[{\"n\":\"全部\",\"v\":\"all\"},{\"n\":\"动作\",\"v\":\"dongzuo\"},{\"n\":\"爱情\",\"v\":\"aiqing\"},{\"n\":\"喜剧\",\"v\":\"xiju\"},{\"n\":\"科幻\",\"v\":\"kehuan\"},{\"n\":\"恐怖\",\"v\":\"kongbu\"},{\"n\":\"战争\",\"v\":\"zhanzheng\"},{\"n\":\"武侠\",\"v\":\"wuxia\"},{\"n\":\"魔幻\",\"v\":\"mohuan\"},{\"n\":\"剧情\",\"v\":\"juqing\"},{\"n\":\"动画\",\"v\":\"donghua\"},{\"n\":\"惊悚\",\"v\":\"jingsong\"},{\"n\":\"3D\",\"v\":\"3D\"},{\"n\":\"灾难\",\"v\":\"zainan\"},{\"n\":\"悬疑\",\"v\":\"xuanyi\"},{\"n\":\"警匪\",\"v\":\"jingfei\"},{\"n\":\"文艺\",\"v\":\"wenyi\"},{\"n\":\"青春\",\"v\":\"qingchun\"},{\"n\":\"冒险\",\"v\":\"maoxian\"},{\"n\":\"犯罪\",\"v\":\"fanzui\"},{\"n\":\"纪录\",\"v\":\"jilu\"},{\"n\":\"古装\",\"v\":\"guzhuang\"},{\"n\":\"奇幻\",\"v\":\"qihuan\"},{\"n\":\"国语\",\"v\":\"guoyu\"},{\"n\":\"综艺\",\"v\":\"zongyi\"},{\"n\":\"历史\",\"v\":\"lishi\"},{\"n\":\"运动\",\"v\":\"yundong\"},{\"n\":\"原创压制\",\"v\":\"yuanchuang\"},{\"n\":\"美剧\",\"v\":\"meiju\"},{\"n\":\"韩剧\",\"v\":\"hanju\"},{\"n\":\"国产电视剧\",\"v\":\"guoju\"},{\"n\":\"日剧\",\"v\":\"riju\"},{\"n\":\"英剧\",\"v\":\"yingju\"},{\"n\":\"德剧\",\"v\":\"deju\"},{\"n\":\"俄剧\",\"v\":\"eju\"},{\"n\":\"巴剧\",\"v\":\"baju\"},{\"n\":\"加剧\",\"v\":\"jiaju\"},{\"n\":\"西剧\",\"v\":\"anish\"},{\"n\":\"意大利剧\",\"v\":\"yidaliju\"},{\"n\":\"泰剧\",\"v\":\"taiju\"},{\"n\":\"港台剧\",\"v\":\"gangtaiju\"},{\"n\":\"法剧\",\"v\":\"faju\"},{\"n\":\"澳剧\",\"v\":\"aoju\"}]},{\"key\":\"area\",\"name\":\"地区\",\"value\":[{\"n\":\"不限\",\"v\":\"\"},{\"n\":\"中国大陆\",\"v\":\"中国大陆\"},{\"n\":\"中国香港\",\"v\":\"中国香港\"},{\"n\":\"中国台湾\",\"v\":\"中国台湾\"},{\"n\":\"美国\",\"v\":\"美国\"},{\"n\":\"英国\",\"v\":\"英国\"},{\"n\":\"日本\",\"v\":\"日本\"},{\"n\":\"韩国\",\"v\":\"韩国\"},{\"n\":\"法国\",\"v\":\"法国\"},{\"n\":\"印度\",\"v\":\"印度\"},{\"n\":\"德国\",\"v\":\"德国\"},{\"n\":\"西班牙\",\"v\":\"西班牙\"},{\"n\":\"意大利\",\"v\":\"意大利\"},{\"n\":\"澳大利亚\",\"v\":\"澳大利亚\"},{\"n\":\"比利时\",\"v\":\"比利时\"},{\"n\":\"瑞典\",\"v\":\"瑞典\"},{\"n\":\"荷兰\",\"v\":\"荷兰\"},{\"n\":\"丹麦\",\"v\":\"丹麦\"},{\"n\":\"加拿大\",\"v\":\"加拿大\"},{\"n\":\"俄罗斯\",\"v\":\"俄罗斯\"}]},{\"key\":\"year\",\"name\":\"年份\",\"value\":[{\"n\":\"全部\",\"v\":\"\"},{\"n\":\"2023\",\"v\":\"2023\"},{\"n\":\"2022\",\"v\":\"2022\"},{\"n\":\"2021\",\"v\":\"2021\"},{\"n\":\"2020\",\"v\":\"2020\"},{\"n\":\"2019\",\"v\":\"2019\"},{\"n\":\"2018\",\"v\":\"2018\"},{\"n\":\"2017\",\"v\":\"2017\"},{\"n\":\"2016\",\"v\":\"2016\"},{\"n\":\"2015\",\"v\":\"2015\"},{\"n\":\"2014\",\"v\":\"2014\"},{\"n\":\"2013\",\"v\":\"2013\"},{\"n\":\"2012\",\"v\":\"2012\"},{\"n\":\"2011\",\"v\":\"2011\"},{\"n\":\"2010\",\"v\":\"2010\"},{\"n\":\"2009\",\"v\":\"2009\"},{\"n\":\"2008\",\"v\":\"2008\"},{\"n\":\"2007\",\"v\":\"2007\"},{\"n\":\"2006\",\"v\":\"2006\"},{\"n\":\"2005\",\"v\":\"2005\"},{\"n\":\"2004\",\"v\":\"2004\"},{\"n\":\"2003\",\"v\":\"2003\"},{\"n\":\"2002\",\"v\":\"2002\"},{\"n\":\"2001\",\"v\":\"2001\"},{\"n\":\"2000\",\"v\":\"2000\"}]},{\"key\":\"order\",\"name\":\"排序\",\"value\":[{\"n\":\"更新时间\",\"v\":\"0\"},{\"n\":\"豆瓣评分\",\"v\":\"1\"}]}]}"); + } catch (JSONException e) { + SpiderDebug.log(e); + } + } + + /** + * 爬虫headers + * + * @param url + * @return + */ + protected HashMap getHeaders(String url,String ref) { + HashMap headers = new HashMap<>(); + headers.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"); + if(!ref.equals("google")){ + headers.put("Authority", "www.bdys01.com"); + if(ref.length()>0){ + headers.put("Referer", ref); + } + if(cookie.length()>0){ + headers.put("Cookie", cookie); + } + } + headers.put("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"); + return headers; + } + + protected HashMap getHeaders2(String url) { + HashMap headers = new HashMap<>(); + String ss = url.replace("https://","").split("/")[0]; + headers.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"); + headers.put("Authority", ss); + headers.put("Origin", "www.bdys01.com"); + headers.put("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"); + return headers; + } + + + protected String getCookie(){ + cookie=""; + String cookieurl="https://www.bdys01.com/zzzzz"; + Map> respHeaders = new HashMap<>(); + OkHttpUtil.stringNoRedirect(cookieurl, getHeaders(cookieurl,referer), respHeaders); + if(respHeaders.containsKey("set-cookie")){ + cookie = respHeaders.get("set-cookie").get(0).split(";")[0]; + } + return cookie; + } + + /** + * 获取分类数据 + 首页最近更新视频列表数据 + * + * @param filter 是否开启筛选 关联的是 软件设置中 首页数据源里的筛选开关 + * @return + */ + @Override + public String homeContent(boolean filter) { + try { + Document doc = Jsoup.parse(OkHttpUtil.string(siteUrl, getHeaders(siteUrl,referer))); + referer=siteUrl+"/"; + // 分类节点 + JSONObject result = new JSONObject(); + JSONArray classes = new JSONArray(); + String catestr ="{\"全部\": \"a\",\"电视剧\": \"1\",\"电影\": \"0\"}"; + JSONObject catedef = new JSONObject(catestr); + Iterator it = catedef.keys(); + while(it.hasNext()){ + JSONObject jsonObject = new JSONObject(); + String key =(String) it.next(); + jsonObject.put("type_name", key); + jsonObject.put("type_id", catedef.getString(key)); + classes.put(jsonObject); + } + result.put("class", classes); + if (filter) { + result.put("filters", filterConfig); + } + try { + // 取首页推荐视频列表 + Element homeList = doc.select("div.row.row-cards").get(0); + Elements list = homeList.select("div.col-4.rows-md-7"); + JSONArray videos = new JSONArray(); + for (int i = 0; i < list.size(); i++) { + Element vod = list.get(i); + String title = vod.selectFirst("h3.card-title").text(); + String cover = vod.selectFirst("img.w-100" ).attr("data-src"); + String remark = vod.selectFirst("p.text-muted").text(); + String id =vod.selectFirst("a.d-block.cover").attr("href"); + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", title); + v.put("vod_pic", cover); + v.put("vod_remarks", remark); + videos.put(v); + } + result.put("list", videos); + } catch (Exception e) { + SpiderDebug.log(e); + } + getCookie(); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + /** + * 获取分类信息数据 + * + * @param tid 分类id + * @param pg 页数 + * @param filter 同homeContent方法中的filter + * @param extend 筛选参数{k:v, k1:v1} + * @return + */ + @Override + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + try { + SpiderDebug.log("Bdys01>>categoryContent"+ filter+"tid:"+tid+"extend:"+extend.toString()); + String url = siteUrl + "/s/" ; + if (extend != null && extend.size() > 0 ) { + String urlformat= ""; + if(tid.equals("a")){ + urlformat= url +"{s}/"+ pg +"?&area={area}&year={year}&order={order}"; + }else { + urlformat= url +"{s}/"+ pg +"?&type="+tid+"&area={area}&year={year}&order={order}"; + } + for (Iterator it = extend.keySet().iterator(); it.hasNext(); ) { + String key = it.next(); + String value = extend.get(key); + if (value.length() > 0) { + urlformat=urlformat.replace("{"+key+"}",URLEncoder.encode(value)); + } + } + for (int i=0 ;i<4;i++) { + if (urlformat.contains("{s}")) { + urlformat = urlformat.replace("{s}", "all"); + } else if (urlformat.contains("{area}")) { + urlformat = urlformat.replace("&area={area}", ""); + } else if (urlformat.contains("{year}")) { + urlformat = urlformat.replace("&year={year}", ""); + } else if (urlformat.contains("{order}")) { + urlformat = urlformat.replace("{order}", "0"); + } + } + url = urlformat; + SpiderDebug.log("Bdys01>>categoryContent"+"url:"+url); + } else { + if(tid.equals("a")){ + url += "all/" + pg + "?&order=0"; + }else { + url += "all/" + pg + "?&type=" + tid + "&order=0"; + } + } + referer=siteUrl+"/"; + String html = OkHttpUtil.string(url, getHeaders(url,referer)); + referer = url; + Document doc = Jsoup.parse(html); + JSONObject result = new JSONObject(); + int pageCount = 0; + int page = -1; + Elements pageInfo = doc.select("a.page-link"); + if (pageInfo.size() == 0) { + page = Integer.parseInt(pg); + pageCount = page; + } else { + for (int i = 0; i < pageInfo.size(); i++) { + Element a = pageInfo.get(i); + String name = a.text(); + if (name.equals("尾页")) { + String gg =a.attr("href"); + String hf =""; + if(gg.contains("JSESSIONID")){ + int start = gg.lastIndexOf("/")+1; + int end = gg.indexOf(";"); + hf =gg.substring(start,end); + }else{ + int start = gg.lastIndexOf("/")+1; + int end = gg.indexOf("?"); + hf =gg.substring(start,end); + } + if (!hf.isEmpty()) { + pageCount = Integer.parseInt(hf); + } else { + pageCount = 0; + } + break; + } + } + } + JSONArray videos = new JSONArray(); + if (!html.contains("没有找到您想要的结果哦")) { + // 取当前分类页的视频列表 + Elements list = doc.select("div.col-lg-8"); + for (int i = 0; i < list.size(); i++) { + Element vod = list.get(i); + String title = vod.selectFirst("h3.mb-0").text(); + String cover = vod.selectFirst("img.w-100").attr("src"); + String remark = vod.selectFirst("p.mb-0").text(); + String idtt = vod.selectFirst("a.d-block").attr("href"); + String id =""; + if(idtt.contains("JSESSIONID")) { + int end = idtt.indexOf(";"); + id = idtt.substring(0,end); + }else{ + id =idtt; + } + JSONObject v = new JSONObject(); + v.put("vod_id", id); + v.put("vod_name", title); + v.put("vod_pic", cover); + v.put("vod_remarks", remark); + videos.put(v); + } + } + result.put("page", page); + result.put("pagecount", pageCount); + result.put("limit", 48); + result.put("total", pageCount <= 1 ? videos.length() : pageCount * 48); + result.put("list", videos); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + /** + * 视频详情信息 + * + * @param ids 视频id + * @return + */ + @Override + public String detailContent(List ids) { + try { + String url = siteUrl + ids.get(0); + Document doc = Jsoup.parse(OkHttpUtil.string(url, getHeaders(url,referer))); + referer=url; + JSONObject result = new JSONObject(); + JSONObject vodList = new JSONObject(); + String cover = doc.selectFirst("div.col-md-auto img").attr("src"); + String title = doc.selectFirst("h1.d-none.d-md-block").text(); + String desc = doc.select("div.card.collapse > div.card-body").text(); + String year = doc.select("span.badge.bg-purple-lt").text().replace("更新时间：","").split("-")[0]; + String category = "", area = "", remark = "", director = "", actor = ""; + Element details = doc.select("div.col.mb-2").get(0); + Elements allp = details.select("p"); + for (int i = 0; i < allp.size(); i++) { + Element text = allp.get(i); + String info = text.selectFirst("strong").text(); + if (info.equals("类型：")) { + List categorys = new ArrayList<>(); + Elements aa = text.select("a"); + for (int j = 0; j < aa.size(); j++) { + categorys.add(aa.get(j).text()); + } + category ="类型："+TextUtils.join(",", categorys); + } else if (info.contains("地区")) { + int start= text.text().indexOf("[")+1; + int end = text.text().indexOf("]"); + area = text.text().substring(start,end); + } else if (info.contains("豆瓣")) { + remark = "豆瓣:"+text.text(); + } else if (info.contains("导演：")) { + director = text.selectFirst("a").text(); + } else if (info.contains("主演：")) { + List actors = new ArrayList<>(); + Elements aa = text.select("a"); + for (int j = 0; j < aa.size(); j++) { + actors.add(aa.get(j).text()); + } + actor =TextUtils.join(",", actors); + } + } + String vid = ids.get(0); + vodList.put("vod_id", vid); + vodList.put("vod_name", title); + vodList.put("vod_pic", cover); + vodList.put("type_name", category); + vodList.put("vod_year", year); + vodList.put("vod_area", area); + vodList.put("vod_remarks", remark); + vodList.put("vod_actor", actor); + vodList.put("vod_director", director); + vodList.put("vod_content", desc); + Map vod_play = new TreeMap<>(); + // 取播放列表数据 + Elements playListA = doc.select("a.btn.btn-square"); + String sourceName = "播放列表"; + String playList = ""; + List vodItems = new ArrayList<>(); + for (int j = 0; j < playListA.size(); j++) { + Element vod = playListA.get(j); + String idtt = vod.attr("href"); + String playURL =""; + if(idtt.contains("JSESSIONID")) { + int end = idtt.indexOf(";"); + playURL = idtt.substring(0,end); + }else{ + playURL =idtt; + } + vodItems.add(vod.text() + "$" + playURL); + } + if (vodItems.size() > 0) + playList = TextUtils.join("#", vodItems); + vod_play.put(sourceName, playList); + if (vod_play.size() > 0) { + String vod_play_from = TextUtils.join("$$$", vod_play.keySet()); + String vod_play_url = TextUtils.join("$$$", vod_play.values()); + vodList.put("vod_play_from", vod_play_from); + vodList.put("vod_play_url", vod_play_url); + } + JSONArray list = new JSONArray(); + list.put(vodList); + result.put("list", list); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + /** + * 获取视频播放信息 + * + * @param flag 播放源 + * @param id 视频id + * @param vipFlags 所有可能需要vip解析的源 + * @return + */ + @TargetApi(Build.VERSION_CODES.KITKAT) + @Override + public String playerContent(String flag, String id, List vipFlags) { + try { + String url = siteUrl + id ; + String htmlplay = OkHttpUtil.string(url, getHeaders(url,referer)); + Document doc = Jsoup.parse(htmlplay); + Elements allScript = doc.select("script"); + JSONObject result = new JSONObject(); + String pid=""; + for (int i = 0; i < allScript.size(); i++) { + String scContent = allScript.get(i).toString(); + if (scContent.contains("var pid =")) { + int start = scContent.indexOf("var pid ="); + int end = scContent.lastIndexOf("var time") + 1; + String pidtmp = scContent.substring(start, end); + start = pidtmp.indexOf("=")+2; + end = pidtmp.lastIndexOf(";") ; + pid = pidtmp.substring(start, end); + break; + } + } + long t = System.currentTimeMillis(); + String key = Misc.NewMD5(pid+"-"+t, StandardCharsets.UTF_8).substring(0,16); + String sg = encrypt(pid+"-"+t,key); + String geturl = siteUrl+"/lines?t="+t+"&sg="+sg+"&pid="+pid; + String urlsrc = OkHttpUtil.string(geturl, getHeaders(geturl,"")); + JSONObject urlsrcobj = new JSONObject(urlsrc); + JSONObject urldb = urlsrcobj.getJSONObject("data"); + List urldblist = new ArrayList<>(); + if(!urldb.isNull("url3")){ + String url3 = urldb.optString("url3"); + urldblist.add(url3); + } + if(!urldb.isNull("m3u8")){ + String m3u8 = urldb.optString("m3u8").replace("www.bde4.cc","www.bdys01.com"); + urldblist.add(m3u8); + } + if(!urldb.isNull("m3u8_2")){ + String m3u8_2 =urldb.optString("m3u8_2"); + String[] m2=m3u8_2.split(","); + for(int i=0; i< m2.length ;i++){ + urldblist.add(m2[i].replace("www.bde4.cc","www.bdys01.com")); + } + } + int index =new Random().nextInt(urldblist.size()); + String videourl = urldblist.get(index); + if(videourl.contains("mp4")){ + result.put("parse", 0); + result.put("playUrl", ""); + result.put("url", videourl); + result.put("header", ""); + return result.toString(); + } + Map> respHeaders = new HashMap<>(); + OkHttpUtil.stringNoRedirect(videourl, getHeaders(videourl,""), respHeaders); + String redirect = OkHttpUtil.getRedirectLocation(respHeaders); + String realm3u8=""; + OKCallBack.OKCallBackDefault callBack = new OKCallBack.OKCallBackDefault() { + @Override + protected void onFailure(Call call, Exception e) { + } + @Override + protected void onResponse(Response response) { + } + }; + OkHttpUtil.get(OkHttpUtil.defaultClient(), redirect, null, getHeaders2(redirect), callBack); + if (callBack.getResult().code() == 200) { + InputStream picsrc = callBack.getResult().body().byteStream(); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + int nRead; + byte[] data = new byte[4]; + while ((nRead = picsrc.read(data, 0, data.length)) != -1) { + buffer.write(data, 0, nRead); + } + buffer.flush(); + byte[] targetArray = buffer.toByteArray(); + picsrc.close(); + byte[] dep = Arrays.copyOfRange(targetArray,3354,targetArray.length); + String pp = gZip.KS(dep); + String[] m3u8ori = pp.split("\n"); + for(int i=0;i>searchContent"+"key:"+key); + String url = "https://www.google.com/search?q=site%3A" + siteHost + "+" + URLEncoder.encode(key); + Document doc = Jsoup.parse(OkHttpUtil.string(url,getHeaders(url,"google"))); + SpiderDebug.log("Bdys01>>searchContent"+"url:"+url); + JSONObject result = new JSONObject(); + JSONArray videos = new JSONArray(); + Elements sourceList = doc.select("div.yuRUbf a"); + if(sourceList.size()>0){ + for (int i = 0; i < 1; i++) { + Element sourcess = sourceList.get(i); + String sourceName = sourcess.select("h3.LC20lb.MBeuO.DKV0Md").text(); + String list1 = sourcess.attr("href"); + if(list1.contains("/s/")||list1.contains("play")||list1.contains("performer")||list1.contains("search")||list1.contains("jsessionid")){ + continue; + } + if (sourceName.contains(key)) { + Document ddrklink = Jsoup.parse(OkHttpUtil.string(list1, getHeaders(list1,referer))); + JSONObject v = new JSONObject(); + String cover = ddrklink.selectFirst("div.col-md-auto img").attr("src"); + String title = ddrklink.selectFirst("h2.d-sm-block.d-md-none").text(); + String id =list1.replace("https://www.bdys01.com",""); + v.put("vod_name", title); + v.put("vod_remarks", ""); + v.put("vod_id", id); + v.put("vod_pic", cover); + videos.put(v); + } + } + result.put("list", videos); + return result.toString(); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected String encrypt(String src, String KEY) { + try { + Cipher cipher = Cipher.getInstance("AES/ECB/PKCS7Padding"); + SecretKeySpec keySpec = new SecretKeySpec(KEY.getBytes(), "AES"); + cipher.init(Cipher.ENCRYPT_MODE, keySpec); + return bytesToHexStr(cipher.doFinal(src.getBytes())).toUpperCase(); + } catch (Exception exception) { + SpiderDebug.log(exception); + } + return null; + } + + protected String bytesToHexStr(byte[] bytes) { + StringBuilder hexStr = new StringBuilder(); + for (byte b : bytes) { + String hex = Integer.toHexString(b & 0xFF); + if (hex.length() == 1) { + hex = '0' + hex; + } + hexStr.append(hex); + } + return hexStr.toString(); + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/github/catvod/spider/Bili11.java b/app/src/main/java/com/github/catvod/spider/Bili11.java new file mode 100644 index 00000000..a1ffb61c --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/Bili11.java @@ -0,0 +1,144 @@ +package com.github.catvod.spider; + +import android.content.Context; +import android.text.TextUtils; + +import com.github.catvod.bean.Class; +import com.github.catvod.bean.Result; +import com.github.catvod.bean.Vod; +import com.github.catvod.crawler.Spider; +import com.github.catvod.net.OkHttp; +import com.github.catvod.utils.Misc; +import com.github.catvod.utils.Trans; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.Jsoup; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +/** + * @author ColaMint & FongMi + */ +public class Bili11 extends Spider { + + private static final String url = "https://www.bilibili.com"; + private HashMap header; + private JSONObject ext; + private String extend; + + private String getCookie(String cookie) { + if (TextUtils.isEmpty(cookie)) return "buvid3=84B0395D-C9F2-C490-E92E-A09AB48FE26E71636infoc"; + if (cookie.startsWith("http")) return OkHttp.string(cookie).replace("\n", ""); + return cookie; + } + + private void setHeader() throws Exception { + header.put("cookie", getCookie(ext.getString("cookie"))); + header.put("User-Agent", Misc.CHROME); + header.put("Referer", url); + } + + private void fetchExt() { + String result = OkHttp.string(extend); + if (!TextUtils.isEmpty(result)) extend = result; + } + + private void fetchRule() throws Exception { + if (header.containsKey("cookie") && header.get("cookie").length() > 0) return; + if (extend.startsWith("http")) fetchExt(); + ext = new JSONObject(extend); + setHeader(); + } + + @Override + public void init(Context context, String extend) { + try { + this.extend = extend; + this.header = new HashMap<>(); + fetchRule(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public String homeContent(boolean filter) throws Exception { + fetchRule(); + return Result.string(Class.arrayFrom(ext.getJSONArray("classes").toString()), ext.getJSONObject("filter")); + } + + @Override + public String homeVideoContent() throws Exception { + fetchRule(); + return categoryContent("汽车 数码", "1", true, new HashMap<>()); + } + + @Override + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) throws Exception { + String duration = extend.containsKey("duration") ? extend.get("duration") : "0"; + if (extend.containsKey("tid")) tid = tid + " " + extend.get("tid"); + String url = "https://api.bilibili.com/x/web-interface/search/type?search_type=video&keyword=" + URLEncoder.encode(tid) + "&duration=" + duration + "&page=" + pg; + JSONObject resp = new JSONObject(OkHttp.string(url, header)); + JSONArray result = resp.getJSONObject("data").getJSONArray("result"); + List list = new ArrayList<>(); + for (int i = 0; i < result.length(); ++i) { + JSONObject object = result.getJSONObject(i); + String pic = object.getString("pic"); + Vod vod = new Vod(); + vod.setVodId(object.getString("bvid")); + vod.setVodName(Jsoup.parse(object.getString("title")).text()); + vod.setVodRemarks(object.getString("duration").split(":")[0] + "分鐘"); + vod.setVodPic(pic.startsWith("//") ? "https:" + pic : pic); + list.add(vod); + } + return Result.string(list); + } + + @Override + public String detailContent(List ids) throws Exception { + String bvid = ids.get(0); + String bvid2aidUrl = "https://api.bilibili.com/x/web-interface/archive/stat?bvid=" + bvid; + JSONObject bvid2aidResp = new JSONObject(OkHttp.string(bvid2aidUrl, header)); + String aid = bvid2aidResp.getJSONObject("data").getLong("aid") + ""; + String detailUrl = "https://api.bilibili.com/x/web-interface/view?aid=" + aid; + JSONObject detailResp = new JSONObject(OkHttp.string(detailUrl, header)); + JSONObject detailData = detailResp.getJSONObject("data"); + List playlist = new ArrayList<>(); + JSONArray pages = detailData.getJSONArray("pages"); + for (int i = 0; i < pages.length(); ++i) { + JSONObject page = pages.getJSONObject(i); + String title = page.getString("part").replace("$", "_").replace("#", "_"); + playlist.add(Trans.get(title) + "$" + aid + "+" + page.getLong("cid")); + } + Vod vod = new Vod(); + vod.setVodId(bvid); + vod.setVodName(detailData.getString("title")); + vod.setVodPic(detailData.getString("pic")); + vod.setTypeName(detailData.getString("tname")); + vod.setVodRemarks(detailData.getLong("duration") / 60 + "分鐘"); + vod.setVodContent(detailData.getString("desc")); + vod.setVodPlayFrom("B站"); + vod.setVodPlayUrl(TextUtils.join("#", playlist)); + return Result.string(vod); + } + + @Override + public String searchContent(String key, boolean quick) throws Exception { + return categoryContent(key, "1", true, new HashMap<>()); + } + + @Override + public String playerContent(String flag, String id, List vipFlags) throws Exception { + String[] ids = id.split("\\+"); + String aid = ids[0]; + String cid = ids[1]; + String url = "https://api.bilibili.com/x/player/playurl?avid=" + aid + "&cid=" + cid + "&qn=120&fourk=1"; + JSONObject resp = new JSONObject(OkHttp.string(url, header)); + url = resp.getJSONObject("data").getJSONArray("durl").getJSONObject(0).getString("url"); + return Result.get().url(url).header(header).string(); + } +} diff --git a/app/src/main/java/com/github/catvod/spider/Kunyu77.java b/app/src/main/java/com/github/catvod/spider/Kunyu77.java new file mode 100644 index 00000000..0dee3825 --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/Kunyu77.java @@ -0,0 +1,372 @@ +package com.github.catvod.spider; + +import android.os.Build; +import android.text.TextUtils; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.utils.Misc; +import com.github.catvod.utils.okhttp.OkHttpUtil; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Set; + + +/** + * Author: @SDL + */ +public class Kunyu77 extends Spider { + private static final String siteUrl = "http://api.kunyu77.com"; + + private String uAgent = "Dalvik/2.1.0 (Linux; U; Android " + Build.VERSION.RELEASE + "; " + Build.MODEL + " Build/" + Build.ID + ")"; + + private HashMap getHeaders(String url) { + HashMap headers = new HashMap<>(); + headers.put("user-agent", uAgent); + return headers; + } + + @Override + public String homeContent(boolean filter) { + try { + String url = siteUrl + "/api.php/provide/filter"; + String content = OkHttpUtil.string(url, getHeaders(url)); + JSONObject jsonObject = new JSONObject(decryptResponse(content)).getJSONObject("data"); + Iterator keys = jsonObject.keys(); + JSONArray classes = new JSONArray(); + JSONObject filterConfig = new JSONObject(); + JSONArray extendsAll = null; + while (keys.hasNext()) { + String typeId = keys.next(); + String typeName = jsonObject.getJSONArray(typeId).getJSONObject(0).getString("cat"); + JSONObject newCls = new JSONObject(); + newCls.put("type_id", typeId); + newCls.put("type_name", typeName); + classes.put(newCls); + try { + if (extendsAll == null) { + String filterUrl = siteUrl + "/api.php/provide/searchFilter?type_id=0&pagenum=1&pagesize=1"; + String filterContent = OkHttpUtil.string(filterUrl, getHeaders(filterUrl)); + JSONObject filterObj = new JSONObject(filterContent).getJSONObject("data").getJSONObject("conditions"); + extendsAll = new JSONArray(); + // 年份 + JSONObject newTypeExtend = new JSONObject(); + newTypeExtend.put("key", "year"); + newTypeExtend.put("name", "年份"); + JSONArray newTypeExtendKV = new JSONArray(); + JSONObject kv = new JSONObject(); + kv.put("n", "全部"); + kv.put("v", ""); + newTypeExtendKV.put(kv); + kv = new JSONObject(); + kv.put("n", "2022"); + kv.put("v", "2022"); + newTypeExtendKV.put(kv); + kv = new JSONObject(); + kv.put("n", "2021"); + kv.put("v", "2021"); + newTypeExtendKV.put(kv); + JSONArray years = filterObj.getJSONArray("y"); + for (int j = 0; j < years.length(); j++) { + JSONObject child = years.getJSONObject(j); + kv = new JSONObject(); + kv.put("n", child.getString("name")); + kv.put("v", child.getString("value")); + newTypeExtendKV.put(kv); + } + newTypeExtend.put("value", newTypeExtendKV); + extendsAll.put(newTypeExtend); + // 地区 + newTypeExtend = new JSONObject(); + newTypeExtend.put("key", "area"); + newTypeExtend.put("name", "地区"); + newTypeExtendKV = new JSONArray(); + kv = new JSONObject(); + kv.put("n", "全部"); + kv.put("v", ""); + newTypeExtendKV.put(kv); + JSONArray areas = filterObj.getJSONArray("a"); + for (int j = 0; j < areas.length(); j++) { + JSONObject child = areas.getJSONObject(j); + kv = new JSONObject(); + kv.put("n", child.getString("name")); + kv.put("v", child.getString("value")); + newTypeExtendKV.put(kv); + } + newTypeExtend.put("value", newTypeExtendKV); + extendsAll.put(newTypeExtend); + // 类型 + newTypeExtend = new JSONObject(); + newTypeExtend.put("key", "category"); + newTypeExtend.put("name", "类型"); + newTypeExtendKV = new JSONArray(); + kv = new JSONObject(); + kv.put("n", "全部"); + kv.put("v", ""); + newTypeExtendKV.put(kv); + JSONArray scat = filterObj.getJSONArray("scat"); + for (int j = 0; j < scat.length(); j++) { + JSONObject child = scat.getJSONObject(j); + kv = new JSONObject(); + kv.put("n", child.getString("name")); + kv.put("v", child.getString("value")); + newTypeExtendKV.put(kv); + } + newTypeExtend.put("value", newTypeExtendKV); + extendsAll.put(newTypeExtend); + } + if (extendsAll != null && extendsAll.length() > 0) { + filterConfig.put(typeId, extendsAll); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + } + JSONObject result = new JSONObject(); + result.put("class", classes); + if (filter) { + result.put("filters", filterConfig); + } + return result.toString(); + } catch (Throwable th) { + + } + return ""; + } + + @Override + public String homeVideoContent() { + try { + JSONArray videos = new JSONArray(); + try { + String url = siteUrl + "/api.php/provide/homeBlock?type_id=0"; + String content = OkHttpUtil.string(url, getHeaders(url)); + JSONObject jsonObject = new JSONObject(decryptResponse(content)); + JSONArray jsonArray = jsonObject.getJSONObject("data").getJSONArray("blocks"); + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject blockObj = jsonArray.getJSONObject(i); + String blockName = blockObj.getString("block_name"); + if (!blockName.startsWith("热播")) { + continue; + } + JSONArray contents = blockObj.getJSONArray("contents"); + for (int j = 0; j < contents.length() && j < 3; j++) { + JSONObject vObj = contents.getJSONObject(j); + JSONObject v = new JSONObject(); + v.put("vod_id", vObj.getString("id")); + v.put("vod_name", vObj.getString("title")); + v.put("vod_pic", vObj.getString("videoCover")); + v.put("vod_remarks", vObj.getString("msg")); + videos.put(v); + } + } + } catch (Exception e) { + + } + JSONObject result = new JSONObject(); + result.put("list", videos); + return result.toString(); + } catch (Throwable th) { + + } + return ""; + } + + @Override + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + try { + String url = siteUrl + "/api.php/provide/searchFilter?type_id=" + tid + "&pagenum=" + pg + "&pagesize=24"; + Set keys = extend.keySet(); + for (String key : keys) { + String val = extend.get(key).trim(); + if (val.length() == 0) + continue; + url += "&" + key + "=" + URLEncoder.encode(val); + } + String content = OkHttpUtil.string(url, getHeaders(url)); + JSONObject dataObject = new JSONObject(decryptResponse(content)).getJSONObject("data"); + JSONArray jsonArray = dataObject.getJSONArray("result"); + JSONArray videos = new JSONArray(); + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject vObj = jsonArray.getJSONObject(i); + JSONObject v = new JSONObject(); + v.put("vod_id", vObj.getString("id")); + v.put("vod_name", vObj.getString("title")); + v.put("vod_pic", vObj.getString("videoCover")); + v.put("vod_remarks", vObj.getString("msg")); + videos.put(v); + } + JSONObject result = new JSONObject(); + int limit = 24; + int page = Integer.parseInt(dataObject.getString("page")); + int total = dataObject.getInt("total"); + int pageCount = dataObject.getInt("pagesize"); + result.put("page", page); + result.put("pagecount", pageCount); + result.put("limit", limit); + result.put("total", total); + result.put("list", videos); + return result.toString(); + } catch (Throwable th) { + + } + return ""; + } + + @Override + public String detailContent(List ids) { + try { + String url = siteUrl + "/api.php/provide/videoDetail?ids=" + ids.get(0); + String content = OkHttpUtil.string(url, getHeaders(url)); + JSONObject dataObject = new JSONObject(decryptResponse(content)); + JSONObject vObj = dataObject.getJSONObject("data"); + JSONObject result = new JSONObject(); + JSONArray list = new JSONArray(); + JSONObject vodAtom = new JSONObject(); + String title = vObj.getString("videoName"); + vodAtom.put("vod_id", vObj.getString("id")); + vodAtom.put("vod_name", title); + vodAtom.put("vod_pic", vObj.getString("videoCover")); + vodAtom.put("type_name", vObj.getString("subCategory")); + vodAtom.put("vod_year", vObj.getString("year")); + vodAtom.put("vod_area", vObj.getString("area")); + vodAtom.put("vod_remarks", vObj.getString("msg")); + vodAtom.put("vod_actor", vObj.getString("actor")); + vodAtom.put("vod_director", vObj.getString("director")); + vodAtom.put("vod_content", vObj.getString("brief").trim()); + + url = siteUrl + "/api.php/provide/videoPlaylist?ids=" + ids.get(0); + content = OkHttpUtil.string(url, getHeaders(url)); + JSONArray episodes = new JSONObject(content).getJSONObject("data").getJSONArray("episodes"); + LinkedHashMap> playlist = new LinkedHashMap<>(); + for (int i = 0; i < episodes.length(); i++) { + JSONArray playurls = episodes.getJSONObject(i).getJSONArray("playurls"); + for (int j = 0; j < playurls.length(); j++) { + JSONObject purl = playurls.getJSONObject(j); + String from = purl.getString("playfrom"); + ArrayList urls = playlist.get(from); + if (urls == null) { + urls = new ArrayList<>(); + playlist.put(from, urls); + } + String name = purl.getString("title").replace(title, "").trim(); + if (name.isEmpty()) { + name = (i + 1) + ""; + } + String pid = purl.getString("playurl"); + urls.add(name + "$" + pid); + } + } + String vod_play_from = TextUtils.join("$$$", playlist.keySet()); + StringBuilder sb = new StringBuilder(); + Iterator> iter = playlist.values().iterator(); + short fromSize = (short) playlist.size(); + while (iter.hasNext()) { + fromSize--; + ArrayList urls = iter.next(); + sb.append(TextUtils.join("#", urls)); + if (fromSize > 0) + sb.append("$$$"); + } + vodAtom.put("vod_play_from", vod_play_from); + vodAtom.put("vod_play_url", sb.toString()); + list.put(vodAtom); + result.put("list", list); + return result.toString(); + } catch (Throwable th) { + + } + return ""; + } + + @Override + public String playerContent(String flag, String id, List vipFlags) { + try { + String videoUrl = id; + try { + String url = siteUrl + "/api.php/provide/parserUrl?url=" + id; + String content = OkHttpUtil.string(url, getHeaders(url)); + JSONObject dataObj = new JSONObject(decryptResponse(content)).getJSONObject("data"); + JSONObject playHeader = dataObj.optJSONObject("playHeader"); + String jxUrl = dataObj.getString("url"); + content = OkHttpUtil.string(jxUrl, getHeaders(jxUrl)); + JSONObject result = Misc.jsonParse(jxUrl, content); + if (result != null) { + result.put("parse", 0); + result.put("playUrl", ""); + if (playHeader != null) { + JSONObject header = result.getJSONObject("header"); + Iterator iter = playHeader.keys(); + while (iter.hasNext()) { + String key = iter.next(); + header.put(key, " " + playHeader.getString(key)); + } + result.put("header", header.toString()); + } + return result.toString(); + } + } catch (Throwable th) { + + } + if (Misc.isVip(videoUrl)) { + JSONObject result = new JSONObject(); + result.put("parse", 1); + result.put("jx", "1"); + result.put("url", videoUrl); + return result.toString(); + } + JSONObject result = new JSONObject(); + result.put("parse", 0); + result.put("playUrl", ""); + result.put("url", id); + return result.toString(); + } catch (Throwable th) { + + } + return ""; + } + + @Override + public String searchContent(String key, boolean quick) { + if (quick) + return ""; + try { + String url = siteUrl + "/api.php/provide/searchVideo?searchName=" + URLEncoder.encode(key); + String content = OkHttpUtil.string(url, getHeaders(url)); + JSONObject dataObject = new JSONObject(decryptResponse(content)); + JSONArray jsonArray = dataObject.getJSONArray("data"); + JSONArray videos = new JSONArray(); + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject vObj = jsonArray.getJSONObject(i); + JSONObject v = new JSONObject(); + String title = vObj.getString("videoName"); + if (!title.contains(key)) + continue; + v.put("vod_id", vObj.getString("id")); + v.put("vod_name", title); + v.put("vod_pic", vObj.getString("videoCover")); + v.put("vod_remarks", vObj.getString("msg")); + videos.put(v); + } + JSONObject result = new JSONObject(); + result.put("list", videos); + return result.toString(); + } catch (Throwable th) { + + } + return ""; + } + + protected String decryptResponse(String src) { + return src; + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/github/catvod/spider/Live.java b/app/src/main/java/com/github/catvod/spider/Live.java index 2fc5b683..810a3baf 100644 --- a/app/src/main/java/com/github/catvod/spider/Live.java +++ b/app/src/main/java/com/github/catvod/spider/Live.java @@ -13,7 +13,7 @@ public class Live extends Spider { private int delay; @Override - public void init(Context context, String extend) { + public void init(Context context, String extend) { super.init(context, extend); this.delay = delay(extend); } diff --git a/app/src/main/java/com/github/catvod/spider/MGTV.java b/app/src/main/java/com/github/catvod/spider/MGTV.java new file mode 100644 index 00000000..3cd8658a --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/MGTV.java @@ -0,0 +1,316 @@ +package com.github.catvod.spider; + +import android.content.Context; +import android.net.Uri; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.utils.okhttp.OkHttpUtil; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; + + +public class MGTV extends Spider { + protected JSONObject a; + protected JSONObject q = new JSONObject(); + + private String q(String str, String str2) { + String str3; + try { + if (str2.startsWith("//")) { + Uri parse = Uri.parse(str); + str3 = parse.getScheme() + ":" + str2; + } else if (str2.contains("://")) { + return str2; + } else { + Uri parse2 = Uri.parse(str); + str3 = parse2.getScheme() + "://" + parse2.getHost() + str2; + } + return str3; + } catch (Exception e) { + SpiderDebug.log(e); + return str2; + } + } + + public String categoryContent(String str, String str2, boolean z, HashMap hashMap) { + try { + String str3 = "https://pianku.api.mgtv.com/rider/list/msite/v2?platform=msite&channelId=" + str + "&pn=" + str2 + "&chargeInfo=&sort=c2"; + if (hashMap != null) { + for (String str4 : hashMap.keySet()) { + String trim = hashMap.get(str4).trim(); + if (trim.length() != 0) { + str3 = str3 + "&" + str4 + "=" + URLEncoder.encode(trim); + } + } + } + String content = OkHttpUtil.string(str3, getHeaders(str3)); + JSONObject jSONObject = new JSONObject(); + try { + JSONArray optJSONArray = new JSONObject(content).optJSONObject("data").optJSONArray("hitDocs"); + JSONArray jSONArray = new JSONArray(); + for (int i = 0; i < optJSONArray.length(); i++) { + JSONObject optJSONObject = optJSONArray.optJSONObject(i); + String optString = optJSONObject.optString("title"); + String q = q(str3, optJSONObject.optString("img")); + String optString2 = optJSONObject.optString("updateInfo"); + if (optString2.equals("")) { + optString2 = optJSONObject.optString("subtitle"); + } + String vodId = optJSONObject.optString("playPartId"); + JSONObject jSONObject2 = new JSONObject(); + jSONObject2.put("vod_id", vodId); + jSONObject2.put("vod_name", optString); + jSONObject2.put("vod_pic", q); + jSONObject2.put("vod_remarks", optString2); + jSONArray.put(jSONObject2); + } + jSONObject.put("list", jSONArray); + jSONObject.put("page", str2); + jSONObject.put("pagecount", Integer.MAX_VALUE); + jSONObject.put("limit", 90); + jSONObject.put("total", Integer.MAX_VALUE); + jSONObject.put("list", jSONArray); + } catch (Exception e) { + SpiderDebug.log(e); + } + return jSONObject.toString(4); + } catch (Exception e2) { + SpiderDebug.log(e2); + return ""; + } + } + + public String detailContent(List list) { + try { + String vodId = list.get(0); + String infoUrl = "https://pcweb.api.mgtv.com/player/vinfo?video_id=" + vodId; + String infoJson = OkHttpUtil.string(infoUrl, getHeaders(infoUrl)); + JSONObject info = new JSONObject(infoJson).getJSONObject("data"); + String pic = info.getString("clip_imgurl2"); + + int pageSize = 30; + int page = 1; + String url = "https://pcweb.api.mgtv.com/episode/list?_support=10000000&version=5.5.35&video_id="+vodId+"&page="+page+"&size="+pageSize+"&allowedRC=1&_support=10000000"; + String json = OkHttpUtil.string(url, getHeaders(url)); + JSONObject jSONObject = new JSONObject(json).getJSONObject("data"); + JSONObject jSONObject2 = new JSONObject(); + JSONObject optJSONObject = jSONObject.optJSONObject("info"); + jSONObject2.put("vod_id", vodId); + jSONObject2.put("vod_name", optJSONObject.optString("title")); + jSONObject2.put("vod_pic", pic); + jSONObject2.put("vod_content", optJSONObject.optString("desc")); + JSONArray optJSONArray = jSONObject.optJSONArray("list"); + ArrayList arrayList = new ArrayList(); + + for (int i = 0; i < optJSONArray.length(); i++) { + JSONObject optJSONObject2 = optJSONArray.optJSONObject(i); + String viewName = optJSONObject2.optString("t1") + " " + optJSONObject2.optString("t2"); + arrayList.add(viewName + "$" + optJSONObject2.optString("url")); + } + + int totalPage = jSONObject.getInt("total_page"); + int count = jSONObject.getInt("count"); + int total = jSONObject.getInt("total"); + ArrayList blocks = new ArrayList(); + ArrayList blockList = new ArrayList(); + if (totalPage == 1) { + blocks.add("mgtv"); + } else if (totalPage > 1) { + blocks.add("1-" + pageSize); + } + blockList.add(join("#", arrayList)); + if (totalPage > page) { + for (int curPage = 2; curPage<=totalPage; curPage++) { + String listUrl = "https://pcweb.api.mgtv.com/episode/list?_support=10000000&version=5.5.35&video_id="+vodId+"&page="+curPage+"&size="+pageSize+"&allowedRC=1&_support=10000000"; + String resultJson = OkHttpUtil.string(listUrl, getHeaders(listUrl)); + JSONObject dataObj = new JSONObject(resultJson).getJSONObject("data"); + JSONArray dataList = dataObj.getJSONArray("list"); + int first = (curPage - 1) * pageSize + 1; + int last = 0; + if (totalPage == curPage) { + last = total; + } else { + last = curPage * pageSize; + } + blocks.add(first + "-" + last); + ArrayList curArrayList = new ArrayList(); + for (int i = 0; i < dataList.length(); i++) { + JSONObject item = dataList.optJSONObject(i); + String viewThisName = item.optString("t1") + " " + item.optString("t2"); + curArrayList.add(viewThisName + "$" + item.optString("url")); + } + blockList.add(join("#", curArrayList)); + } + } + + jSONObject2.put("vod_play_from", join("$$$", blocks)); + jSONObject2.put("vod_play_url", join("$$$", blockList)); + JSONObject result = new JSONObject(); + JSONArray jSONArray = new JSONArray(); + jSONArray.put(jSONObject2); + result.put("list", jSONArray); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + return ""; + } + } + + protected HashMap getHeaders(String str) { + HashMap hashMap = new HashMap<>(); + hashMap.put("referer", "https://so.mgtv.com"); + hashMap.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"); + return hashMap; + } + + public String homeContent(boolean z) { + try { + JSONArray optJSONArray = new JSONObject(OkHttpUtil.string("https://pianku.api.mgtv.com/rider/config/platformChannels/v1?platform=msite&abroad=0&_support=10000000", getHeaders("https://pianku.api.mgtv.com/rider/config/platformChannels/v1?platform=msite&abroad=0&_support=10000000"))).optJSONArray("data"); + JSONArray jSONArray = new JSONArray(); + for (int i = 0; i < optJSONArray.length(); i++) { + JSONObject optJSONObject = optJSONArray.optJSONObject(i); + JSONObject jSONObject = new JSONObject(); + jSONObject.put("type_name", optJSONObject.optString("channelName")); + jSONObject.put("type_id", optJSONObject.optString("channelId")); + jSONArray.put(jSONObject); + } + JSONObject jSONObject2 = new JSONObject(); + if (z) { + jSONObject2.put("filters", this.a); + } + jSONObject2.put("class", jSONArray); + try { + JSONArray optJSONArray2 = new JSONObject(OkHttpUtil.string("https://pianku.api.mgtv.com/rider/list/pcweb/v3?platform=pcweb&channelId=2&pn=1&chargeInfo=&sort=c2", getHeaders("https://pianku.api.mgtv.com/rider/list/pcweb/v3?platform=pcweb&channelId=2&pn=1&chargeInfo=&sort=c2"))).optJSONObject("data").optJSONArray("hitDocs"); + JSONArray jSONArray2 = new JSONArray(); + for (int i2 = 0; i2 < optJSONArray2.length(); i2++) { + JSONObject optJSONObject2 = optJSONArray2.optJSONObject(i2); + String optString = optJSONObject2.optString("title"); + String q = optJSONObject2.optString("imgUrlH"); + String optString2 = optJSONObject2.optString("updateInfo"); + JSONObject jSONObject3 = new JSONObject(); + jSONObject3.put("vod_id", optJSONObject2.optString("playPartId")); + jSONObject3.put("vod_name", optString); + jSONObject3.put("vod_pic", q); + jSONObject3.put("vod_remarks", optString2); + jSONArray2.put(jSONObject3); + } + jSONObject2.put("list", jSONArray2); + } catch (Exception e) { + SpiderDebug.log(e); + } + return jSONObject2.toString(4); + } catch (Exception e2) { + SpiderDebug.log(e2); + return ""; + } + } + + public String homeVideoContent() { + try { + JSONArray jSONArray = new JSONObject(OkHttpUtil.string("https://www.mgtv.com/api.php/app/index_video?token=", getHeaders("https://www.mgtv.com/api.php/app/index_video?token="))).getJSONArray("list"); + JSONArray jSONArray2 = new JSONArray(); + for (int i = 0; i < jSONArray.length(); i++) { + JSONArray jSONArray3 = jSONArray.getJSONObject(i).getJSONArray("vlist"); + int i2 = 0; + while (i2 < jSONArray3.length() && i2 < 6) { + JSONObject jSONObject = jSONArray3.getJSONObject(i2); + JSONObject jSONObject2 = new JSONObject(); + jSONObject2.put("vod_id", jSONObject.optString("vod_id")); + jSONObject2.put("vod_name", jSONObject.optString("vod_name")); + jSONObject2.put("vod_pic", jSONObject.optString("vod_pic")); + jSONObject2.put("vod_remarks", jSONObject.optString("vod_remarks")); + jSONArray2.put(jSONObject2); + i2++; + } + } + JSONObject jSONObject3 = new JSONObject(); + jSONObject3.put("list", jSONArray2); + return jSONObject3.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + return ""; + } + } + + public void init(Context context) { + MGTV.super.init(context); + try { + this.a = new JSONObject("{\"1\":[{\"name\":\"类型\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"4\",\"n\":\"王牌综艺\"},{\"v\":\"5\",\"n\":\"大咖秀场\"},{\"v\":\"6\",\"n\":\"搞笑\"},{\"v\":\"7\",\"n\":\"情感\"},{\"v\":\"8\",\"n\":\"音乐\"},{\"v\":\"9\",\"n\":\"生活\"},{\"v\":\"179\",\"n\":\"亲子\"},{\"v\":\"170\",\"n\":\"旅游\"},{\"v\":\"171\",\"n\":\"时尚\"},{\"v\":\"173\",\"n\":\"真人秀\"},{\"v\":\"174\",\"n\":\"竞技\"},{\"v\":\"172\",\"n\":\"访谈\"},{\"v\":\"180\",\"n\":\"脱口秀\"}],\"key\":\"kind\"},{\"name\":\"地区\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"内地\"},{\"v\":\"2\",\"n\":\"港台\"},{\"v\":\"3\",\"n\":\"其他\"}],\"key\":\"area\"},{\"name\":\"排序\",\"value\":[{\"v\":\"c1\",\"n\":\"最新\"},{\"v\":\"c2\",\"n\":\"最热\"},{\"v\":\"c4\",\"n\":\"知乎高分\"}],\"key\":\"sort\"}],\"2\":[{\"name\":\"类型\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"14\",\"n\":\"甜蜜互宠\"},{\"v\":\"15\",\"n\":\"虐恋情深\"},{\"v\":\"16\",\"n\":\"青涩校园\"},{\"v\":\"17\",\"n\":\"仙侠玄幻\"},{\"v\":\"19\",\"n\":\"都市职场\"},{\"v\":\"148\",\"n\":\"古装\"},{\"v\":\"20\",\"n\":\"快意江湖\"},{\"v\":\"147\",\"n\":\"偶像\"},{\"v\":\"21\",\"n\":\"悬疑推理\"},{\"v\":\"22\",\"n\":\"家长里短\"},{\"v\":\"23\",\"n\":\"芒果出品\"},{\"v\":\"24\",\"n\":\"轻松搞笑\"},{\"v\":\"25\",\"n\":\"铁血战争\"},{\"v\":\"26\",\"n\":\"其他\"}],\"key\":\"kind\"},{\"name\":\"地区\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"10\",\"n\":\"内地\"},{\"v\":\"11\",\"n\":\"日韩\"},{\"v\":\"12\",\"n\":\"港台\"}],\"key\":\"area\"},{\"name\":\"排序\",\"value\":[{\"v\":\"c2\",\"n\":\"最热\"},{\"v\":\"c1\",\"n\":\"最新\"},{\"v\":\"c4\",\"n\":\"知乎高分\"}],\"key\":\"sort\"},{\"name\":\"版本\",\"value\":[{\"v\":\"all\",\"n\":\"全部\"},{\"v\":\"2037\",\"n\":\"TV版\"},{\"v\":\"2038\",\"n\":\"特别版\"},{\"v\":\"2040\",\"n\":\"精华版\"},{\"v\":\"2036\",\"n\":\"未删减版\"}],\"key\":\"edition\"}],\"3\":[{\"name\":\"类型\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"34\",\"n\":\"院线大片\"},{\"v\":\"175\",\"n\":\"爱情\"},{\"v\":\"176\",\"n\":\"喜剧\"},{\"v\":\"177\",\"n\":\"动作\"},{\"v\":\"178\",\"n\":\"科幻\"},{\"v\":\"39\",\"n\":\"青春\"},{\"v\":\"43\",\"n\":\"恐怖悬疑\"},{\"v\":\"44\",\"n\":\"战争\"},{\"v\":\"45\",\"n\":\"警匪\"},{\"v\":\"46\",\"n\":\"历史\"},{\"v\":\"47\",\"n\":\"歌舞\"},{\"v\":\"48\",\"n\":\"动画\"},{\"v\":\"50\",\"n\":\"其他\"}],\"key\":\"kind\"},{\"name\":\"资费\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"b1\",\"n\":\"免费\"},{\"v\":\"b2\",\"n\":\"VIP\"},{\"v\":\"b3\",\"n\":\"VIP用券\"},{\"v\":\"b4\",\"n\":\"付费点播\"}],\"key\":\"chargeInfo\"},{\"name\":\"排序\",\"value\":[{\"v\":\"c2\",\"n\":\"最热\"},{\"v\":\"c1\",\"n\":\"最新\"},{\"v\":\"c4\",\"n\":\"知乎高分\"}],\"key\":\"sort\"}],\"106\":[{\"name\":\"类型\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"106\",\"n\":\"时事\"},{\"v\":\"107\",\"n\":\"社会\"},{\"v\":\"108\",\"n\":\"文娱\"},{\"v\":\"109\",\"n\":\"军事\"}],\"key\":\"kind\"},{\"name\":\"排序\",\"value\":[{\"v\":\"c1\",\"n\":\"最新\"},{\"v\":\"c2\",\"n\":\"最热\"}],\"key\":\"sort\"}],\"91\":[{\"name\":\"类型\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"119\",\"n\":\"吐槽\"},{\"v\":\"120\",\"n\":\"恶搞\"},{\"v\":\"121\",\"n\":\"爆笑\"},{\"v\":\"122\",\"n\":\"奇趣\"},{\"v\":\"123\",\"n\":\"饭制\"},{\"v\":\"124\",\"n\":\"其他\"}],\"key\":\"kind\"},{\"name\":\"排序\",\"value\":[{\"v\":\"c2\",\"n\":\"最热\"},{\"v\":\"c1\",\"n\":\"最新\"}],\"key\":\"sort\"}],\"50\":[{\"name\":\"类型\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"60\",\"n\":\"恋爱少女\"},{\"v\":\"86\",\"n\":\"热血\"},{\"v\":\"62\",\"n\":\"搞笑\"},{\"v\":\"63\",\"n\":\"青春\"},{\"v\":\"64\",\"n\":\"魔幻仙侠\"},{\"v\":\"65\",\"n\":\"激燃运动\"},{\"v\":\"66\",\"n\":\"特摄\"},{\"v\":\"67\",\"n\":\"推理\"},{\"v\":\"68\",\"n\":\"亲子幼教\"},{\"v\":\"69\",\"n\":\"芒果出品\"},{\"v\":\"70\",\"n\":\"动漫音乐\"},{\"v\":\"71\",\"n\":\"经典\"},{\"v\":\"72\",\"n\":\"其他\"}],\"key\":\"kind\"},{\"name\":\"地区\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"52\",\"n\":\"内地\"},{\"v\":\"53\",\"n\":\"欧美\"},{\"v\":\"54\",\"n\":\"日韩\"},{\"v\":\"55\",\"n\":\"其他\"}],\"key\":\"area\"},{\"name\":\"版本\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"165\",\"n\":\"剧场版\"},{\"v\":\"57\",\"n\":\"TV版\"},{\"v\":\"166\",\"n\":\"OVA版\"},{\"v\":\"167\",\"n\":\"真人版\"}],\"key\":\"edition\"},{\"name\":\"排序\",\"value\":[{\"v\":\"c2\",\"n\":\"最热\"},{\"v\":\"c1\",\"n\":\"最新\"}],\"key\":\"sort\"}],\"51\":[{\"name\":\"类型\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"97\",\"n\":\"美食\"},{\"v\":\"98\",\"n\":\"文化\"},{\"v\":\"99\",\"n\":\"社会\"},{\"v\":\"100\",\"n\":\"历史\"},{\"v\":\"101\",\"n\":\"军事\"},{\"v\":\"102\",\"n\":\"人物\"},{\"v\":\"103\",\"n\":\"探索\"},{\"v\":\"104\",\"n\":\"自然\"},{\"v\":\"105\",\"n\":\"其他\"}],\"key\":\"kind\"},{\"name\":\"排序\",\"value\":[{\"v\":\"c2\",\"n\":\"最热\"},{\"v\":\"c1\",\"n\":\"最新\"}],\"key\":\"sort\"}],\"20\":[{\"name\":\"类型\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"150\",\"n\":\"MV\"},{\"v\":\"151\",\"n\":\"影视原声\"},{\"v\":\"152\",\"n\":\"音乐节目\"},{\"v\":\"153\",\"n\":\"演唱会\"},{\"v\":\"154\",\"n\":\"颁奖礼\"},{\"v\":\"155\",\"n\":\"音乐现场\"},{\"v\":\"156\",\"n\":\"其他\"}],\"key\":\"kind\"},{\"name\":\"风格\",\"value\":[{\"v\":\"a1\",\"n\":\"全部\"},{\"v\":\"157\",\"n\":\"流行\"},{\"v\":\"158\",\"n\":\"Hip-hop\"},{\"v\":\"159\",\"n\":\"R&B\"},{\"v\":\"160\",\"n\":\"摇滚\"},{\"v\":\"161\",\"n\":\"民谣\"},{\"v\":\"162\",\"n\":\"爵士\"},{\"v\":\"163\",\"n\":\"古典\"},{\"v\":\"164\",\"n\":\"其他\"}],\"key\":\"musicStyle\"},{\"name\":\"排序\",\"value\":[{\"v\":\"c1\",\"n\":\"最新\"},{\"v\":\"c2\",\"n\":\"最热\"}],\"key\":\"sort\"}]}"); + } catch (JSONException e) { + SpiderDebug.log(e); + } + } + + public String join(CharSequence charSequence,Iterable iterable) { + Iterator it = iterable.iterator(); + if (!it.hasNext()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + sb.append(it.next()); + while (it.hasNext()) { + sb.append(charSequence); + sb.append(it.next()); + } + return sb.toString(); + } + + public String playerContent(String str, String str2, List list) { + try { + JSONObject jSONObject = new JSONObject(); + try { + jSONObject.put("parse", 1); + jSONObject.put("jx", "1"); + jSONObject.put("url", "https://www.mgtv.com" + str2); + return jSONObject.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + return jSONObject.toString(); + } + } catch (Exception e2) { + SpiderDebug.log(e2); + return ""; + } + } + + public String searchContent(String str, boolean quick) { + try { + String url = "https://mobileso.bz.mgtv.com/pc/search/v1?allowedRC=1&q=" + str + "&pn=1&pc=10&uid=&corr=1&_support=10000000"; + String json = OkHttpUtil.string(url, getHeaders(url)); + JSONObject data = new JSONObject(json).getJSONObject("data"); + JSONArray contents = data.getJSONArray("contents"); + JSONArray items = new JSONArray(); + for (int i = 0; i < contents.length(); i++) { + JSONObject jSONObject = contents.getJSONObject(i).getJSONObject("data"); + if (!jSONObject.has("sourceList")) { + continue; + } + JSONArray sourceList = jSONObject.getJSONArray("sourceList"); + String vodUrl = sourceList.getJSONObject(0).getString("url"); + String vodId = sourceList.getJSONObject(0).getString("vid"); + String pic = jSONObject.optString("pic"); + JSONObject item = new JSONObject(); + item.put("vod_id", vodId); + item.put("vod_name", jSONObject.optString("title")); + item.put("vod_pic", pic); + item.put("vod_remarks", jSONObject.optString("playTime")); + items.put(item); + } + JSONObject result = new JSONObject(); + result.put("list", items); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + return ""; + } + } +} diff --git a/app/src/main/java/com/github/catvod/spider/QQ.java b/app/src/main/java/com/github/catvod/spider/QQ.java new file mode 100644 index 00000000..26c9a90d --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/QQ.java @@ -0,0 +1,333 @@ +package com.github.catvod.spider; + +import android.content.Context; +import android.net.Uri; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.utils.okhttp.OkHttpUtil; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; + + +public class QQ extends Spider { + protected JSONObject dn; + protected JSONObject q = new JSONObject(); + + private String q(String str, String str2) { + String str3; + try { + if (str2.startsWith("//")) { + Uri parse = Uri.parse(str); + str3 = parse.getScheme() + ":" + str2; + } else if (str2.contains("://")) { + return str2; + } else { + Uri parse2 = Uri.parse(str); + str3 = parse2.getScheme() + "://" + parse2.getHost() + str2; + } + return str3; + } catch (Exception e) { + SpiderDebug.log(e); + return str2; + } + } + + public String categoryContent(String str, String str2, boolean z, HashMap hashMap) { + String str3; + try { + String str4 = "https://v.qq.com/x/bu/pagesheet/list?_all=1&append=1&channel=" + str + "&listpage=1&offset=" + ((Integer.parseInt(str2) - 1) * 21) + "&pagesize=21&sort=18"; + if (hashMap != null) { + for (String str5 : hashMap.keySet()) { + String trim = hashMap.get(str5).trim(); + if (trim.length() != 0) { + str4 = str4 + "&" + str5 + "=" + URLEncoder.encode(trim); + } + } + } + String content = OkHttpUtil.string(str4, getHeaders(str4)); + JSONObject jSONObject = new JSONObject(); + try { + Elements listItems = Jsoup.parse(content).select(".list_item"); + JSONArray jSONArray = new JSONArray(); + for (int i = 0; i < listItems.size(); i++) { + Element item = listItems.get(i); + String Pd = item.select("a").attr("title"); + String q = q(str4, item.select("img").attr("src")); + if (item.select(".figure_caption") == null) { + str3 = ""; + } else { + str3 = item.select(".figure_caption").text(); + } + String Pd2 = item.select("a").attr("data-float"); + JSONObject jSONObject2 = new JSONObject(); + jSONObject2.put("vod_id", Pd2); + jSONObject2.put("vod_name", Pd); + jSONObject2.put("vod_pic", q); + jSONObject2.put("vod_remarks", str3); + jSONArray.put(jSONObject2); + } + jSONObject.put("page", str2); + jSONObject.put("pagecount", Integer.MAX_VALUE); + jSONObject.put("limit", 90); + jSONObject.put("total", Integer.MAX_VALUE); + jSONObject.put("list", jSONArray); + } catch (Exception e) { + SpiderDebug.log(e); + } + return jSONObject.toString(4); + } catch (Exception e2) { + SpiderDebug.log(e2); + return ""; + } + } + + public String detailContent(List list) { + JSONArray jSONArray; + CharSequence charSequence; + JSONArray jSONArray2; + CharSequence charSequence2; + CharSequence charSequence3 = ","; + try { + String str = "https://node.video.qq.com/x/api/float_vinfo2?cid=" + list.get(0); + JSONObject jSONObject = new JSONObject(OkHttpUtil.string(str, getHeaders(str))); + JSONObject optJSONObject = jSONObject.optJSONObject("c"); + if (optJSONObject == null) { + return ""; + } + JSONObject jSONObject2 = new JSONObject(); + jSONObject2.put("vod_id", list.get(0)); + jSONObject2.put("vod_name", optJSONObject.optString("title")); + jSONObject2.put("vod_pic", q(str, optJSONObject.optString("pic"))); + jSONObject2.put("type_name", jSONObject.optJSONArray("typ").opt(0)); + jSONObject2.put("vod_year", optJSONObject.optString("year")); + jSONObject.optJSONObject("people"); + JSONArray optJSONArray = jSONObject.optJSONArray("nam").optJSONArray(0); + ArrayList arrayList = new ArrayList(); + if (optJSONArray != null) { + for (int i = 0; i < optJSONArray.length(); i++) { + arrayList.add(optJSONArray.opt(i)); + } + } + jSONObject2.put("vod_actor", join(charSequence3, arrayList)); + jSONObject2.put("vod_content", optJSONObject.optString("description")); + JSONArray jSONArray3 = optJSONObject.getJSONArray("video_ids"); + ArrayList arrayList2 = new ArrayList(); + ArrayList arrayList3 = new ArrayList(); + int i2 = 1; + while (i2 <= jSONArray3.length()) { + arrayList2.add(jSONArray3.optString(i2 - 1)); + if (!(i2 % 30 == 0 || i2 == jSONArray3.length())) { + charSequence = charSequence3; + jSONArray = jSONArray3; + i2++; + charSequence3 = charSequence; + jSONArray3 = jSONArray; + } + String str2 = "https://union.video.qq.com/fcgi-bin/data?otype=json&tid=682&appid=20001238&appkey=6c03bbe9658448a4&union_platform=1&idlist=" + join(charSequence3, arrayList2); + String h = OkHttpUtil.string(str2, getHeaders(str2)); + JSONArray jSONArray4 = new JSONObject(h.substring(13, h.length() - 1)).getJSONArray("results"); + int i3 = 0; + while (i3 < jSONArray4.length()) { + JSONObject jSONObject3 = jSONArray4.getJSONObject(i3).getJSONObject("fields"); + if (!jSONObject3.optString("title").contains("预告")) { + StringBuilder sb = new StringBuilder(); + String optString = jSONObject3.optString("title"); + charSequence2 = charSequence3; + StringBuilder sb2 = new StringBuilder(); + jSONArray2 = jSONArray3; + sb2.append(optJSONObject.optString("title")); + sb2.append("_"); + sb.append(optString.replace(sb2.toString(), "")); + sb.append("$https://v.qq.com/x/cover/"); + sb.append(list.get(0)); + sb.append("/"); + sb.append(jSONObject3.optString("vid")); + sb.append(".html"); + arrayList3.add(sb.toString()); + } else { + charSequence2 = charSequence3; + jSONArray2 = jSONArray3; + } + i3++; + charSequence3 = charSequence2; + jSONArray3 = jSONArray2; + } + charSequence = charSequence3; + jSONArray = jSONArray3; + arrayList2.clear(); + i2++; + charSequence3 = charSequence; + jSONArray3 = jSONArray; + } + jSONObject2.put("vod_play_from", "qq"); + jSONObject2.put("vod_play_url", join("#", arrayList3)); + JSONObject jSONObject4 = new JSONObject(); + JSONArray jSONArray5 = new JSONArray(); + jSONArray5.put(jSONObject2); + jSONObject4.put("list", jSONArray5); + return jSONObject4.toString(4); + } catch (Exception e) { + SpiderDebug.log(e); + return ""; + } + } + + protected HashMap getHeaders(String str) { + HashMap hashMap = new HashMap<>(); + hashMap.put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"); + return hashMap; + } + + public String homeContent(boolean z) { + try { + Document doc = Jsoup.parse(OkHttpUtil.string("https://v.qq.com/channel/tv?listpage=1&channel=tv&sort=18&_all=1", getHeaders("https://v.qq.com/channel/tv?listpage=1&channel=tv&sort=18&_all=1"))); + JSONArray jSONArray = new JSONArray(); + Iterator it = doc.select(".nav_cell").iterator(); + while (it.hasNext()) { + Element next = it.next(); + JSONObject jSONObject = new JSONObject(); + String Pd = next.select("a").attr("href"); + if (!Pd.contains("/art") && !Pd.contains("feeds_hotspot") && !Pd.contains("wwe") && !Pd.contains("choice") && !Pd.contains("sports_new") && !Pd.contains("games") && !Pd.contains("lols11") && !Pd.contains("ent") && !Pd.contains("news") && !Pd.contains("fashion") && !Pd.contains("tech") && !Pd.contains("auto") && !Pd.contains("house") && !Pd.contains("finance") && !Pd.contains("astro") && !Pd.contains("nba") && !Pd.contains("fun") && !Pd.contains("baby") && !Pd.contains("music") && !Pd.contains("life") && !Pd.contains("travel") && Pd.contains("/channel/")) { + jSONObject.put("type_name", next.select("a").text()); + jSONObject.put("type_id", Pd.split("/channel/")[1]); + jSONArray.put(jSONObject); + } + } + String h = OkHttpUtil.string("https://v.qq.com/x/bu/pagesheet/list?_all=1&append=1&channel=choice", getHeaders("https://v.qq.com/x/bu/pagesheet/list?_all=1&append=1&channel=choice")); + JSONObject jSONObject2 = new JSONObject(); + if (z) { + jSONObject2.put("filters", this.dn); + } + jSONObject2.put("class", jSONArray); + try { + Elements listItem = Jsoup.parse(h).select(".list_item"); + JSONArray jSONArray2 = new JSONArray(); + for (int i = 0; i < listItem.size(); i++) { + Element item = listItem.get(i); + String title = item.select("a").attr("title"); + String pic = q("https://v.qq.com/x/bu/pagesheet/list?_all=1&append=1&channel=choice", item.select("img").attr("src")); + String remark = item.select(".figure_caption").text(); + String id = item.select("a").attr("data-float"); + JSONObject jSONObject3 = new JSONObject(); + jSONObject3.put("vod_id", id); + jSONObject3.put("vod_name", title); + jSONObject3.put("vod_pic", pic); + jSONObject3.put("vod_remarks", remark); + jSONArray2.put(jSONObject3); + } + jSONObject2.put("list", jSONArray2); + } catch (Exception e) { + SpiderDebug.log(e); + } + return jSONObject2.toString(4); + } catch (Exception e2) { + SpiderDebug.log(e2); + return ""; + } + } + + public String homeVideoContent() { + try { + JSONArray jSONArray = new JSONObject(OkHttpUtil.string("https://v.qq.com/api.php/app/index_video?token=", getHeaders("https://v.qq.com/api.php/app/index_video?token="))).getJSONArray("list"); + JSONArray jSONArray2 = new JSONArray(); + for (int i = 0; i < jSONArray.length(); i++) { + JSONArray jSONArray3 = jSONArray.getJSONObject(i).getJSONArray("vlist"); + int i2 = 0; + while (i2 < jSONArray3.length() && i2 < 6) { + JSONObject jSONObject = jSONArray3.getJSONObject(i2); + JSONObject jSONObject2 = new JSONObject(); + jSONObject2.put("vod_id", jSONObject.optString("vod_id")); + jSONObject2.put("vod_name", jSONObject.optString("vod_name")); + jSONObject2.put("vod_pic", jSONObject.optString("vod_pic")); + jSONObject2.put("vod_remarks", jSONObject.optString("vod_remarks")); + jSONArray2.put(jSONObject2); + i2++; + } + } + JSONObject jSONObject3 = new JSONObject(); + jSONObject3.put("list", jSONArray2); + return jSONObject3.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + return ""; + } + } + + public void init(Context context) { + QQ.super.init(context); + try { + this.dn = new JSONObject("{\"tv\":[{\"name\":\"排序\",\"value\":[{\"v\":\"19\",\"n\":\"最新\"},{\"v\":\"18\",\"n\":\"最热\"},{\"v\":\"16\",\"n\":\"好评\"},{\"v\":\"21\",\"n\":\"口碑好剧\"},{\"v\":\"54\",\"n\":\"高分好评\"},{\"v\":\"22\",\"n\":\"知乎高分\"}],\"key\":\"sort\"},{\"name\":\"类型\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"偶像爱情\"},{\"v\":\"2\",\"n\":\"古装历史\"},{\"v\":\"3\",\"n\":\"玄幻史诗\"},{\"v\":\"4\",\"n\":\"都市生活\"},{\"v\":\"14\",\"n\":\"当代主旋律\"},{\"v\":\"5\",\"n\":\"罪案谍战\"},{\"v\":\"6\",\"n\":\"历险科幻\"},{\"v\":\"7\",\"n\":\"军旅抗战\"},{\"v\":\"8\",\"n\":\"喜剧\"},{\"v\":\"9\",\"n\":\"武侠江湖\"},{\"v\":\"10\",\"n\":\"青春校园\"},{\"v\":\"11\",\"n\":\"时代传奇\"},{\"v\":\"12\",\"n\":\"体育电竞\"},{\"v\":\"13\",\"n\":\"真人动漫\"},{\"v\":\"15\",\"n\":\"短剧\"},{\"v\":\"44\",\"n\":\"独播\"}],\"key\":\"feature\"},{\"name\":\"地区\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"814\",\"n\":\"内地\"},{\"v\":\"815\",\"n\":\"美国\"},{\"v\":\"816\",\"n\":\"英国\"},{\"v\":\"818\",\"n\":\"韩国\"},{\"v\":\"9\",\"n\":\"泰国\"},{\"v\":\"10\",\"n\":\"日本\"},{\"v\":\"14\",\"n\":\"中国香港\"},{\"v\":\"817\",\"n\":\"中国台湾\"},{\"v\":\"819\",\"n\":\"其他\"}],\"key\":\"iarea\"},{\"name\":\"年份\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"2022\",\"n\":\"2022\"},{\"v\":\"2021\",\"n\":\"2021\"},{\"v\":\"2020\",\"n\":\"2020\"},{\"v\":\"4061\",\"n\":\"2019\"},{\"v\":\"4060\",\"n\":\"2018\"},{\"v\":\"2017\",\"n\":\"2017\"},{\"v\":\"859\",\"n\":\"2016\"},{\"v\":\"860\",\"n\":\"2015\"},{\"v\":\"861\",\"n\":\"2014\"},{\"v\":\"862\",\"n\":\"2013\"},{\"v\":\"863\",\"n\":\"2012\"},{\"v\":\"864\",\"n\":\"2011\"},{\"v\":\"865\",\"n\":\"2010\"},{\"v\":\"866\",\"n\":\"其他\"}],\"key\":\"year\"},{\"name\":\"资费\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"867\",\"n\":\"免费\"},{\"v\":\"6\",\"n\":\"会员\"}],\"key\":\"pay\"}],\"education\":[{\"name\":\"排序\",\"value\":[{\"v\":\"19\",\"n\":\"按更新\"},{\"v\":\"40\",\"n\":\"最热\"}],\"key\":\"sort\"},{\"name\":\"学段\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"2\",\"n\":\"小学\"},{\"v\":\"3\",\"n\":\"初中\"},{\"v\":\"4\",\"n\":\"高中\"},{\"v\":\"5\",\"n\":\"大学\"},{\"v\":\"6\",\"n\":\"研究生\"},{\"v\":\"7\",\"n\":\"成人\"}],\"key\":\"section\"},{\"name\":\"学级\",\"value\":[{\"v\":\"1\",\"n\":\"全部\"},{\"v\":\"5\",\"n\":\"一年级\"},{\"v\":\"6\",\"n\":\"二年级\"},{\"v\":\"7\",\"n\":\"三年级\"},{\"v\":\"8\",\"n\":\"四年级\"},{\"v\":\"9\",\"n\":\"五年级\"},{\"v\":\"10\",\"n\":\"六年级\"},{\"v\":\"11\",\"n\":\"初一\"},{\"v\":\"12\",\"n\":\"初二\"},{\"v\":\"13\",\"n\":\"初三\"},{\"v\":\"14\",\"n\":\"高一\"},{\"v\":\"15\",\"n\":\"高二\"},{\"v\":\"16\",\"n\":\"高三\"},{\"v\":\"20\",\"n\":\"大四\"}],\"key\":\"grade\"},{\"name\":\"学科\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"41\",\"n\":\"语文\"},{\"v\":\"42\",\"n\":\"数学\"},{\"v\":\"43\",\"n\":\"英语\"},{\"v\":\"44\",\"n\":\"物理\"},{\"v\":\"45\",\"n\":\"化学\"},{\"v\":\"46\",\"n\":\"生物\"},{\"v\":\"47\",\"n\":\"政治\"},{\"v\":\"50\",\"n\":\"其它\"}],\"key\":\"subject\"}],\"movie\":[{\"name\":\"排序\",\"value\":[{\"v\":\"18\",\"n\":\"最近热播\"},{\"v\":\"19\",\"n\":\"最新上架\"},{\"v\":\"21\",\"n\":\"高分好评\"},{\"v\":\"22\",\"n\":\"知乎高分\"}],\"key\":\"sort\"},{\"name\":\"类型\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"100018\",\"n\":\"剧情\"},{\"v\":\"100004\",\"n\":\"喜剧\"},{\"v\":\"100061\",\"n\":\"动作\"},{\"v\":\"100005\",\"n\":\"爱情\"},{\"v\":\"100010\",\"n\":\"惊悚\"},{\"v\":\"4\",\"n\":\"犯罪\"},{\"v\":\"100009\",\"n\":\"悬疑\"},{\"v\":\"100006\",\"n\":\"战争\"},{\"v\":\"100012\",\"n\":\"科幻\"},{\"v\":\"100015\",\"n\":\"动画\"},{\"v\":\"100007\",\"n\":\"恐怖\"},{\"v\":\"100017\",\"n\":\"家庭\"},{\"v\":\"100022\",\"n\":\"传记\"},{\"v\":\"100003\",\"n\":\"冒险\"},{\"v\":\"100016\",\"n\":\"奇幻\"},{\"v\":\"100011\",\"n\":\"武侠\"},{\"v\":\"100021\",\"n\":\"历史\"},{\"v\":\"2\",\"n\":\"运动\"},{\"v\":\"100014\",\"n\":\"歌舞\"},{\"v\":\"100013\",\"n\":\"音乐\"},{\"v\":\"100020\",\"n\":\"纪录\"},{\"v\":\"100019\",\"n\":\"伦理\"},{\"v\":\"3\",\"n\":\"西部\"}],\"key\":\"itype\"},{\"name\":\"地区\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"100024\",\"n\":\"内地\"},{\"v\":\"100025\",\"n\":\"中国香港\"},{\"v\":\"100029\",\"n\":\"美国\"},{\"v\":\"100032\",\"n\":\"欧洲\"},{\"v\":\"100026\",\"n\":\"中国台湾\"},{\"v\":\"100027\",\"n\":\"日本\"},{\"v\":\"100028\",\"n\":\"韩国\"},{\"v\":\"100030\",\"n\":\"印度\"},{\"v\":\"100031\",\"n\":\"泰国\"},{\"v\":\"15\",\"n\":\"英国\"},{\"v\":\"16\",\"n\":\"法国\"},{\"v\":\"17\",\"n\":\"德国\"},{\"v\":\"18\",\"n\":\"加拿大\"},{\"v\":\"19\",\"n\":\"西班牙\"},{\"v\":\"20\",\"n\":\"意大利\"},{\"v\":\"21\",\"n\":\"澳大利亚\"},{\"v\":\"22\",\"n\":\"北欧\"},{\"v\":\"23\",\"n\":\"拉丁美洲\"},{\"v\":\"100033\",\"n\":\"其它\"}],\"key\":\"iarea\"},{\"name\":\"特色\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"院线\"},{\"v\":\"2\",\"n\":\"自制电影\"},{\"v\":\"5\",\"n\":\"独播\"},{\"v\":\"8\",\"n\":\"原声\"},{\"v\":\"9\",\"n\":\"粤语\"},{\"v\":\"3\",\"n\":\"蓝光\"},{\"v\":\"6\",\"n\":\"奥斯卡\"}],\"key\":\"characteristic\"},{\"name\":\"年份\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"2022\",\"n\":\"2022\"},{\"v\":\"2022\",\"n\":\"2022\"},{\"v\":\"2021\",\"n\":\"2021\"},{\"v\":\"2020\",\"n\":\"2020\"},{\"v\":\"20\",\"n\":\"2019\"},{\"v\":\"2018\",\"n\":\"2018\"},{\"v\":\"2017\",\"n\":\"2017\"},{\"v\":\"2016\",\"n\":\"2016\"},{\"v\":\"100063\",\"n\":\"2015\"},{\"v\":\"100034\",\"n\":\"2014\"},{\"v\":\"100035\",\"n\":\"2013-2011\"},{\"v\":\"100036\",\"n\":\"2010-2006\"},{\"v\":\"100037\",\"n\":\"2005-2000\"},{\"v\":\"100038\",\"n\":\"90年代\"},{\"v\":\"100039\",\"n\":\"80年代\"},{\"v\":\"100040\",\"n\":\"其它\"}],\"key\":\"year\"},{\"name\":\"资费\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"免费\"},{\"v\":\"2\",\"n\":\"包月\"},{\"v\":\"3\",\"n\":\"用券\"},{\"v\":\"4\",\"n\":\"付费\"}],\"key\":\"charge\"}],\"variety\":[{\"name\":\"排序\",\"value\":[{\"v\":\"4\",\"n\":\"最热\"},{\"v\":\"5\",\"n\":\"最新\"}],\"key\":\"sort\"},{\"name\":\"独家\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"腾讯自制\"},{\"v\":\"2\",\"n\":\"独播\"}],\"key\":\"exclusive\"},{\"name\":\"地区\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"国内\"},{\"v\":\"2\",\"n\":\"海外\"}],\"key\":\"iarea\"},{\"name\":\"类型\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"10\",\"n\":\"游戏\"},{\"v\":\"11\",\"n\":\"音乐\"},{\"v\":\"24\",\"n\":\"养成\"},{\"v\":\"12\",\"n\":\"情感\"},{\"v\":\"14\",\"n\":\"喜剧\"},{\"v\":\"2\",\"n\":\"脱口秀\"},{\"v\":\"16\",\"n\":\"表演\"},{\"v\":\"25\",\"n\":\"体验\"},{\"v\":\"17\",\"n\":\"亲子\"},{\"v\":\"26\",\"n\":\"文化\"},{\"v\":\"19\",\"n\":\"美食\"},{\"v\":\"20\",\"n\":\"职场\"},{\"v\":\"21\",\"n\":\"体育\"},{\"v\":\"15\",\"n\":\"潮流文化\"},{\"v\":\"3\",\"n\":\"访谈\"},{\"v\":\"22\",\"n\":\"生活服务\"},{\"v\":\"23\",\"n\":\"萌宠\"},{\"v\":\"7\",\"n\":\"资讯\"},{\"v\":\"6\",\"n\":\"晚会\"}],\"key\":\"itype\"},{\"name\":\"年份\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"2022\",\"n\":\"2022\"},{\"v\":\"2021\",\"n\":\"2021\"},{\"v\":\"50\",\"n\":\"2020\"},{\"v\":\"7\",\"n\":\"2019\"},{\"v\":\"1\",\"n\":\"2018\"},{\"v\":\"2\",\"n\":\"2017\"},{\"v\":\"3\",\"n\":\"2016\"},{\"v\":\"4\",\"n\":\"2015\"},{\"v\":\"5\",\"n\":\"2014\"},{\"v\":\"6\",\"n\":\"2013\"},{\"v\":\"2012\",\"n\":\"2012\"},{\"v\":\"2011\",\"n\":\"2011\"},{\"v\":\"2010\",\"n\":\"2010\"},{\"v\":\"99\",\"n\":\"更早\"}],\"key\":\"iyear\"},{\"name\":\"付费\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"会员\"}],\"key\":\"ipay\"}],\"cartoon\":[{\"name\":\"排序\",\"value\":[{\"v\":\"40\",\"n\":\"最热\"},{\"v\":\"23\",\"n\":\"最新\"},{\"v\":\"20\",\"n\":\"好评\"},{\"v\":\"22\",\"n\":\"知乎高分\"}],\"key\":\"sort\"},{\"name\":\"类型\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"2\",\"n\":\"冒险\"},{\"v\":\"5\",\"n\":\"战斗\"},{\"v\":\"1\",\"n\":\"搞笑\"},{\"v\":\"3\",\"n\":\"经典\"},{\"v\":\"4\",\"n\":\"科幻\"},{\"v\":\"9\",\"n\":\"玄幻\"},{\"v\":\"6\",\"n\":\"魔幻\"},{\"v\":\"13\",\"n\":\"武侠\"},{\"v\":\"7\",\"n\":\"恋爱\"},{\"v\":\"14\",\"n\":\"推理\"},{\"v\":\"11\",\"n\":\"腾讯出品\"},{\"v\":\"15\",\"n\":\"日常\"},{\"v\":\"16\",\"n\":\"校园\"},{\"v\":\"17\",\"n\":\"悬疑\"},{\"v\":\"18\",\"n\":\"真人\"},{\"v\":\"19\",\"n\":\"历史\"},{\"v\":\"20\",\"n\":\"竞技\"},{\"v\":\"12\",\"n\":\"其他\"}],\"key\":\"itype\"},{\"name\":\"地区\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"内地\"},{\"v\":\"2\",\"n\":\"日本\"},{\"v\":\"3\",\"n\":\"欧美\"},{\"v\":\"4\",\"n\":\"其他\"}],\"key\":\"iarea\"},{\"name\":\"时间\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"2022\",\"n\":\"2022\"},{\"v\":\"2021\",\"n\":\"2021\"},{\"v\":\"50\",\"n\":\"2020\"},{\"v\":\"11\",\"n\":\"2019\"},{\"v\":\"2018\",\"n\":\"2018\"},{\"v\":\"2017\",\"n\":\"2017\"},{\"v\":\"1\",\"n\":\"2016\"},{\"v\":\"2\",\"n\":\"2015\"},{\"v\":\"3\",\"n\":\"2014\"},{\"v\":\"4\",\"n\":\"2013\"},{\"v\":\"5\",\"n\":\"2012\"},{\"v\":\"6\",\"n\":\"2011\"},{\"v\":\"7\",\"n\":\"00年代\"},{\"v\":\"8\",\"n\":\"90年代\"},{\"v\":\"9\",\"n\":\"80年代\"},{\"v\":\"10\",\"n\":\"更早\"}],\"key\":\"iyear\"},{\"name\":\"资费\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"免费\"},{\"v\":\"2\",\"n\":\"会员\"}],\"key\":\"ipay\"},{\"name\":\"状态\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"46\",\"n\":\"预告片\"},{\"v\":\"44\",\"n\":\"连载\"},{\"v\":\"45\",\"n\":\"完结\"}],\"key\":\"anime_status\"},{\"name\":\"分类\",\"value\":[{\"v\":\"1\",\"n\":\"全部\"},{\"v\":\"2\",\"n\":\"3D动画\"},{\"v\":\"3\",\"n\":\"2D动画\"},{\"v\":\"4\",\"n\":\"特摄\"},{\"v\":\"5\",\"n\":\"其他\"}],\"key\":\"item\"}],\"doco\":[{\"name\":\"排序\",\"value\":[{\"v\":\"19\",\"n\":\"最新\"},{\"v\":\"18\",\"n\":\"最热\"},{\"v\":\"20\",\"n\":\"好评\"},{\"v\":\"22\",\"n\":\"知乎高分\"}],\"key\":\"sort\"},{\"name\":\"出品机构\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"BBC\"},{\"v\":\"4\",\"n\":\"国家地理\"},{\"v\":\"3175\",\"n\":\"HBO\"},{\"v\":\"2\",\"n\":\"NHK\"},{\"v\":\"7\",\"n\":\"历史频道\"},{\"v\":\"3530\",\"n\":\"ITV\"},{\"v\":\"3174\",\"n\":\"探索频道\"},{\"v\":\"3176\",\"n\":\"ZDF\"},{\"v\":\"3172\",\"n\":\"ARTE\"},{\"v\":\"15\",\"n\":\"腾讯自制\"},{\"v\":\"6\",\"n\":\"合作机构\"},{\"v\":\"5\",\"n\":\"其他\"}],\"key\":\"itrailer\"},{\"name\":\"类型\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"4\",\"n\":\"自然\"},{\"v\":\"9\",\"n\":\"美食\"},{\"v\":\"3\",\"n\":\"社会\"},{\"v\":\"5\",\"n\":\"人文\"},{\"v\":\"1\",\"n\":\"历史\"},{\"v\":\"2\",\"n\":\"军事\"},{\"v\":\"7\",\"n\":\"科技\"},{\"v\":\"13\",\"n\":\"财经\"},{\"v\":\"15\",\"n\":\"探险\"},{\"v\":\"6\",\"n\":\"罪案\"},{\"v\":\"11\",\"n\":\"竞技\"},{\"v\":\"10\",\"n\":\"旅游\"}],\"key\":\"itype\"},{\"name\":\"资费\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"免费\"},{\"v\":\"2\",\"n\":\"会员\"}],\"key\":\"pay\"}],\"child\":[{\"name\":\"排序\",\"value\":[{\"v\":\"19\",\"n\":\"最新\"},{\"v\":\"18\",\"n\":\"最热\"},{\"v\":\"20\",\"n\":\"好评\"}],\"key\":\"sort\"},{\"name\":\"地区\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"欧美\"},{\"v\":\"2\",\"n\":\"日韩\"},{\"v\":\"3\",\"n\":\"国内\"}],\"key\":\"iarea\"},{\"name\":\"年龄\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"0-3岁\"},{\"v\":\"2\",\"n\":\"4-6岁\"},{\"v\":\"3\",\"n\":\"7-9岁\"},{\"v\":\"4\",\"n\":\"10岁以上\"}],\"key\":\"iyear\"},{\"name\":\"性别\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"男孩\"},{\"v\":\"2\",\"n\":\"女孩\"}],\"key\":\"gender\"},{\"name\":\"类型\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"儿歌\"},{\"v\":\"2\",\"n\":\"益智\"},{\"v\":\"3\",\"n\":\"手工·绘画\"},{\"v\":\"4\",\"n\":\"玩具\"},{\"v\":\"5\",\"n\":\"英语\"},{\"v\":\"7\",\"n\":\"早教\"},{\"v\":\"6\",\"n\":\"数学\"},{\"v\":\"8\",\"n\":\"国学\"},{\"v\":\"9\",\"n\":\"合家欢\"},{\"v\":\"10\",\"n\":\"冒险\"},{\"v\":\"11\",\"n\":\"交通工具\"},{\"v\":\"12\",\"n\":\"魔幻·科幻\"},{\"v\":\"13\",\"n\":\"动物\"},{\"v\":\"14\",\"n\":\"真人特摄\"},{\"v\":\"15\",\"n\":\"探索\"},{\"v\":\"16\",\"n\":\"其他\"}],\"key\":\"itype\"},{\"name\":\"资费\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"1\",\"n\":\"免费\"},{\"v\":\"2\",\"n\":\"会员\"}],\"key\":\"ipay\"}],\"knowledge\":[{\"name\":\"排序\",\"value\":[{\"v\":\"41\",\"n\":\"最热\"},{\"v\":\"10\",\"n\":\"最新\"}],\"key\":\"sort\"},{\"name\":\"分类\",\"value\":[{\"v\":\"-1\",\"n\":\"全部\"},{\"v\":\"2\",\"n\":\"文化历史\"},{\"v\":\"3\",\"n\":\"亲子育儿\"},{\"v\":\"4\",\"n\":\"职场\"},{\"v\":\"5\",\"n\":\"商业理财\"},{\"v\":\"7\",\"n\":\"生活\"},{\"v\":\"9\",\"n\":\"情感心理\"},{\"v\":\"8\",\"n\":\"运动健身\"},{\"v\":\"6\",\"n\":\"艺术兴趣\"},{\"v\":\"12\",\"n\":\"游戏\"},{\"v\":\"14\",\"n\":\"科学科普\"},{\"v\":\"15\",\"n\":\"健康\"},{\"v\":\"16\",\"n\":\"IT/互联网\"}],\"key\":\"pay_level_one\"}]}"); + } catch (JSONException e) { + SpiderDebug.log(e); + } + } + + public String join(@NonNull CharSequence charSequence, @NonNull Iterable iterable) { + Iterator it = iterable.iterator(); + if (!it.hasNext()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + sb.append(it.next()); + while (it.hasNext()) { + sb.append(charSequence); + sb.append(it.next()); + } + return sb.toString(); + } + + public String playerContent(String str, String str2, List list) { + try { + JSONObject jSONObject = new JSONObject(); + try { + jSONObject.put("parse", 1); + jSONObject.put("jx", "1"); + jSONObject.put("url", str2); + return jSONObject.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + return jSONObject.toString(); + } + } catch (Exception e2) { + SpiderDebug.log(e2); + return ""; + } + } + + public String searchContent(String str, boolean quick) { + try { + String str2 = "http://node.video.qq.com/x/api/msearch?keyWord=" + str; + JSONArray jSONArray = new JSONObject(OkHttpUtil.string(str2, getHeaders(str2))).getJSONArray("uiData"); + JSONArray jSONArray2 = new JSONArray(); + for (int i = 0; i < jSONArray.length(); i++) { + JSONObject jSONObject = jSONArray.getJSONObject(i).getJSONArray("data").getJSONObject(0); + JSONObject jSONObject2 = new JSONObject(); + jSONObject2.put("vod_id", jSONObject.optString("id")); + jSONObject2.put("vod_name", jSONObject.optString("title")); + jSONObject2.put("vod_pic", jSONObject.optString("posterPic")); + jSONObject2.put("vod_remarks", jSONObject.optString("publishDate")); + jSONArray2.put(jSONObject2); + } + JSONObject jSONObject3 = new JSONObject(); + jSONObject3.put("list", jSONArray2); + return jSONObject3.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + return ""; + } + } +} diff --git a/app/src/main/java/com/github/catvod/spider/XBiu.java b/app/src/main/java/com/github/catvod/spider/XBiu.java new file mode 100644 index 00000000..79f864eb --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/XBiu.java @@ -0,0 +1,1935 @@ +package com.github.catvod.spider; + +import android.content.Context; +import android.text.TextUtils; +import android.util.Base64; +import android.util.Pair; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.utils.Misc; +import com.github.catvod.utils.okhttp.OKCallBack; +import com.github.catvod.utils.okhttp.OkHttpUtil; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.net.URL; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import okhttp3.Call; + +public class XBiu extends Spider { + + protected String ext = null; + public JSONObject rule = null; + // 默认的视频类型 + private ArrayList videoFormatList = new ArrayList<>(Arrays.asList(".m3u8", ".mp4", ".mpeg", ".flv", ".mkv")); + + + protected final int base64Flag = Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP; + // 一定是正确的分类名称，用来帮助定位分类列表，猜cateManual + protected final ArrayList cateManuals = new ArrayList<>(Arrays.asList("电影", "剧集", "电视剧", "连续剧", "综艺", "动漫")); + // 无效的分类名，彡来过滤cateManual + protected final ArrayList invalidCateNames = new ArrayList<>(Arrays.asList("更多","下载", "首页", "资讯", "留言", "导航", "专题", "短视频", "热榜", "排行", "追剧","更新","APP", "直播", "label", "Netflix")); + // 详情页 影片信息相关字段，猜详情页信息时用 + protected final ArrayList detailItemNames = new ArrayList<>(Arrays.asList("导演", "主演", "演员", "地区", "类型", "年份", "年代")); + protected final ArrayList detailItemKeys = new ArrayList<>(Arrays.asList("vod_director", "vod_actor", "vod_actor", "vod_area", "type_name", "vod_year", "vod_year")); + protected String splitFlag = ""; // 分段标志,猜cateManual时用 + + // html标签查找时用到的辅助类 + protected class HtmlMatchInfo { + public String group0; // 正则表达式匹配到的字符串 + /** + * 一般用来放href中的内容 + */ + public String group1; // + public String group2; // + public String diff; // 两个匹配结果比较group1得到的不同部分 + public int startPos; // 正则匹配到的起始位置 + public int endPos; // 正则匹配到的结束位置 + public ArrayList uploads; // 祖先结节的索引 + public int matchedUpNodePos = -1; // 与其他HtmlMatchInfo最匹配的祖先节点位置 + public int diffStartIndex; // 不同那部分数据的开始位置 + public int diffEndIndex; // 不同那部分数据的结束位置 + + public void init(Matcher m) { + this.group0 = m.group(0); + if (m.groupCount() > 0) + this.group1 = m.group(1); + if (m.groupCount() > 1) + this.group2 = m.group(2); + this.startPos = m.start(0); + this.endPos = m.end(0); + } + + // 通过比较两个group1的不同部分，不同部分的内容以splitFlag中的字符为开始或结束位置 + public boolean findDiffStr(HtmlMatchInfo rhs, String splitFlag) { + int len = Math.min(group1.length(), rhs.group1.length()); + // 找不同字符的开始位置 + for (int i =0; i < len; ++i){ + char a = group1.charAt(i); + char b = rhs.group1.charAt(i); + if(a== b && splitFlag.indexOf (a) != -1) { + diffStartIndex = i+1; + rhs.diffStartIndex = i+1; + } + if(a != b) break; + } + + // 找不同字符的结束位置 + diffEndIndex = group1.length(); + rhs.diffEndIndex = rhs.group1.length(); + for (int i =1; i < len; ++i){ + char a = group1.charAt(group1.length()-i); + char b = rhs.group1.charAt(rhs.group1.length()-i); + if(a== b && splitFlag.indexOf (a) != -1) { + diffEndIndex = group1.length()-i; + rhs.diffEndIndex = rhs.group1.length()-i; + } + if(a != b) break; + } + if(this.diff == null || this.diff.isEmpty() && diffStartIndex < diffEndIndex) { + diff = group1.substring(diffStartIndex, diffEndIndex); + }else{ + if( diffEndIndex < diffStartIndex || !diff.equals(group1.substring(diffStartIndex, diffEndIndex))){ + return false; + } + } + if(rhs.diffStartIndex < rhs.diffEndIndex) { + rhs.diff = rhs.group1.substring(rhs.diffStartIndex, rhs.diffEndIndex); + } + return true; + } + + // 判断 rhs 与当前对象是有相同的祖先节点 + boolean hasSameUpNode(HtmlMatchInfo rhs) { + if (rhs.uploads.size() != this.uploads.size()) return false; + for (int i = 0; i < uploads.size(); ++i) { + if (uploads.get(i).intValue() != rhs.uploads.get(i).intValue()) continue; + if (matchedUpNodePos == -1 || uploads.get(i).intValue() == matchedUpNodePos) { + matchedUpNodePos = uploads.get(i).intValue(); + rhs.matchedUpNodePos = uploads.get(i).intValue(); + return true; + } + return false; + } + return false; + } + } + + public void init(Context context, String extend) { + super.init(context, extend); + this.ext = extend; + } + + // 初始化抓虫规则 + protected void fetchRule() { + if (rule == null) { + if (ext != null) { + try { + + if (ext.startsWith("http")) { + if(ext.indexOf("{cateId}") != -1 || ext.indexOf("{catePg}") !=-1){ + rule = new JSONObject(); + rule.put("homeUrl", ext); + }else{ + String json = OkHttpUtil.string(ext, null); + rule = new JSONObject(json); + } + + } else { + rule = new JSONObject(ext); + } + + if (!rule.has("list")) { + rule.put("list", new JSONObject()); + } + JSONObject list = rule.getJSONObject("list"); + // 初始化homeUrl,list.url + String homeUrl= rule.getString("homeUrl"); + if(homeUrl.indexOf("{cateId}") != -1){ + URL r = new URL(homeUrl); + String path = r.getPath(); + // 更新解析出来的homeUrl + rule.put("homeUrl", homeUrl.substring(0, homeUrl.indexOf(path))); + if(!list.has("url")){ + list.put("url", homeUrl); + } + } + // 初始化截断标志 + String listUrl = list.getString("url"); + if(listUrl.indexOf("/") !=-1) splitFlag+='/'; + if(listUrl.indexOf(".") !=-1) splitFlag+='.'; + if(listUrl.indexOf("-") !=-1) splitFlag+='-'; + + if (!rule.has("detail")) { + rule.put("detail", new JSONObject()); + } + + if (!rule.has("playlist")) { + rule.put("playlist", new JSONObject()); + } + + // 如果没有search，则生成一个默认的search规则，大部分网站的search规则都一样 + // 省掉一个search json + if (!rule.has("search")) { + String url = addHttpPrefix("index.php/ajax/suggest?mid=1&wd=阿凡达"); + try { + // 尝试访问这个json接口，如果返回了正确的json格式，就认为支持json搜索 + JSONObject result = new JSONObject( OkHttpUtil.string(url, getHeaders(url))); + JSONObject search = new JSONObject(); + search.put("vod_id", "id"); + search.put("vod_name", "name"); + search.put("vod_pic", "pic"); + search.put("url", addHttpPrefix("index.php/ajax/suggest?mid=1&wd={wd}")); + rule.put("search", search); + } + catch (Exception e){ + + } + } + + // 部分网站的播放页上直接就有 播放地址，基本上就是一样的格式，可以尝试在playerContent中直接拿直链 + if (!rule.has("play")) { + JSONObject play = new JSONObject(); + JSONArray region = new JSONArray(); + region.put("var player_aaaa="); + region.put(0); + + JSONArray vod_url = new JSONArray(); + vod_url.put("\"url\":\""); + vod_url.put("\""); + play.put("region", region); + play.put("vod_url", vod_url); + rule.put("play", play); + } + + // play字段中可以填写播放连接的关键字用来帮助识别嗅探结果， + // 一般奇葩的网站会用到 + if (rule.has("play")) { // 自定义嗅探关键字 + JSONObject play = rule.getJSONObject("play"); + JSONArray keywords = play.optJSONArray("keywords"); + if (keywords != null) { + videoFormatList.clear(); + for (int i = 0; i < keywords.length(); ++i) { + videoFormatList.add(keywords.getString(i)); + } + } + } + + // 猜cateManaul + JSONObject cateManual = rule.optJSONObject("cateManual"); + String body = ""; + if (cateManual == null) { + // 重建 cateManaul规则 + body = this.fetchUrl(rule.getString("homeUrl"), rule.optJSONObject("header")); + if(body.length() > 32*1024) { body = body.substring(0, 32 * 1024); } + cateManual = this.guess_rule_cateManual(body); + if(cateManual != null){ + rule.put("cateManual", cateManual); + } + } + + // 猜list.vod_id + if (!list.has("vod_id")) { + if(body.isEmpty()){ + body = this.fetchUrl(rule.getString("homeUrl"), rule.optJSONObject("header")); + if(body.length() > 32*1024) { body = body.substring(0, 32 * 1024); } + } + JSONArray listvodid = this.guess_rule_vod_id(body); + list.put("vod_id", listvodid); + } + + // 如果没有json搜索接口，那么尝试在主页上找search 的接口 url + if (!rule.has("search")) { + if(body.isEmpty()){ + body = this.fetchUrl(rule.getString("homeUrl"), rule.optJSONObject("header")); + if(body.length() > 32*1024) { body = body.substring(0, 32 * 1024); } + } + String url = this.guess_rule_search_url(body); + if(!url.isEmpty()){ + JSONObject search = new JSONObject(); + search.put("url", url); + rule.put("search", search); + } + } + + SpiderDebug.log(String.format("默认rule: %s", rule.toString())); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + + public String addHttpPrefix(String url) { + try { + if (url.isEmpty()) return ""; + if (url.startsWith("http")) return url; + String result = rule.getString("homeUrl"); + if (result.endsWith("/")) { + result = result.substring(0, result.length() - 1); + } + if (url.startsWith("/")) { + result += url; + } else { + result += "/" + url; + } + return result; + } catch (JSONException e) { + e.printStackTrace(); + } + return url; + } + + protected HashMap getHeaders(String url) { + HashMap headers = new HashMap<>(); + try { + if (rule.has("header")) { + JSONObject header = rule.getJSONObject("header"); + Iterator iter = header.keys(); + while (iter.hasNext()) { + String key = iter.next(); + headers.put(key, header.getString(key)); + } + } + } catch (JSONException e) { + e.printStackTrace(); + } + if (!headers.containsKey("User-Agent")) { + headers.put("User-Agent", Misc.CHROME); + } + return headers; + } + + public static class HtmlNodeHlper{ + // 非正常配对的html标签，进行html层级查找时要用到 + protected static ArrayList notPairedTag = new ArrayList<>(Arrays.asList("img", "br", "meta", "!--")); // 为注释 + // 判断当前html标签是否为正常的标签 + public static boolean isPairedHtmlTag(String str, int startPos) { + String tmp = str.substring(startPos, Math.min(str.length(), startPos + 10)); + for (String p : notPairedTag) { + if (tmp.indexOf(p) != -1) { // 找到了 + // 找 > 如果匹配了 /> 则认为是配对的 + for (int i = startPos + 1; i < str.length(); ++i) { + String sm = str.substring(i); + if (str.charAt(i) == '>') { + if (str.charAt(i - 1) == '/') { + return true; + } else { + return false; + } + } + } + return false; + } + } + return true; + } + + // 查找当前标签的html代码 pos必须是标签的开始位置 < + public static String nodeString(String str, int pos) { + if (pos < 0 || pos >= str.length() || str.charAt(pos) != '<') return str; + int isRightNode = 0; + for (int i = pos; i < str.length() - 1; ++i) { +// String sm = str.substring(i, i + 400); + switch (str.charAt(i)) { + // 遇到 / 那么这个位置有可能是xml的结束标识,这种情况下再遇到<则不是当前节点的上级节点 + case '/': { + if (str.charAt(i + 1) == '>') { // "/>" 认为是标签的结束位置 + isRightNode--; + } else if (str.charAt(i - 1) == '<') { // "': { + if (isRightNode == 0) { + return str.substring(pos, i + 1); + } + break; + } + case '<': { + if (str.charAt(i + 1) != '/' && isPairedHtmlTag(str, i)) { // 不是 " findUpNodes(String str, int pos, int lookback) { + ArrayList nodes = new ArrayList<>(); + ArrayList urls = new ArrayList<>(); + if (pos == -1) return nodes; + int isUpNode = 0; + for (int i = pos; i >= 0; --i) { + switch (str.charAt(i)) { + // 遇到 / 那么这个位置有可能是xml的结束标识,这种情况下再遇到<则不是当前节点的上级节点 + case '/': { + if (str.charAt(i + 1) == '>') { + isUpNode++; +// SpiderDebug.log(String.format("not xml %s", str.substring(i, i + 20))); + } else if (str.charAt(i - 1) == '<') { + isUpNode++; + --i; +// SpiderDebug.log(String.format("not xml %s", str.substring(i, i + 20))); + } + break; + } + case '<': { + if (isUpNode == 0) { +// SpiderDebug.log(String.format("find up node %d %s", i, str.substring(i, i + 30))); + urls.add(String.format("%5d", i)); + nodes.add(i); + } else if (isPairedHtmlTag(str, i)) { + isUpNode--; + if (isUpNode < 0) isUpNode = 0; +// SpiderDebug.log(String.format("%s", str.substring(i, i + 30))); + } + + break; + } + default: + break; + } + if (nodes.size() >= lookback) { + break; + } + } + return nodes; + } + // 获取当前节点的所有子节点 + public static ArrayList getChildNodes(String str) { + ArrayList arr = new ArrayList<>(); + int pos = 0; + if (pos < 0 || pos >= str.length() || str.charAt(pos) != '<') return arr; + ++pos; + while (pos > -1 && pos < str.length()) { + pos = str.indexOf('<', pos); + String p = nodeString(str, pos); + if (p.isEmpty()) { + break; + } + arr.add(p); + pos += p.length(); + } + return arr; + } + + // 移除字符串的html标签 + public static String trimHtmlString(String str, String r) { + String ret = str.replace("\r\n", "") + .replace("\n", "") + .replaceAll("<.+?>", r) + // .replace(" ", "") + .replaceAll("\\s+", " ") + .replace(" ", "") + .replace(" ", "") + .trim(); + return ret; + } + + public static String trimHtmlString(String str) { + return trimHtmlString(str, ""); + } + + } + + public static class Utils{ + // 查找列表块的起始位置，取最靠近共同祖先节点的位置 + public static int findBlockPos(ArrayList a, ArrayList b) { + int len = a.size() > b.size() ? b.size() : a.size(); + if(len ==1 ) return b.get(0); + for (int i = 0; i < len; ++i) { + if (a.get(i).intValue() == b.get(i).intValue()) { + return b.get(i - 1); + } + } + return b.get(len - 1); + } + + // 查找两个字符串之间的子串 + // keys 字段说明 + // 0 prefix 1 suffix 2 找到子串后左边index的偏移量 3 找到子串后右边index的偏移量 + public static String findSubString(String str, int startPos, JSONArray keys, String defaultVal) { + try { + if (keys == null) return defaultVal; + String prefix = keys.getString(0); + String suffix = keys.getString(1); + int offsetl = 0; // 左边的偏移量 + int offsetr = 0; // 右边的偏移量 + if (keys.length() > 2) { + offsetl = keys.getInt(2); + } + if (keys.length() > 3) { + offsetr = keys.getInt(3); + } + int a = str.indexOf(prefix, startPos) + prefix.length(); + if (a < prefix.length()) return defaultVal; + int b = str.indexOf(suffix, a); + if (b < a) return defaultVal; + return HtmlNodeHlper.trimHtmlString(str.substring(a + offsetl, b + offsetr)); + } catch (JSONException e) { + e.printStackTrace(); + } + return defaultVal; + } + + public static String findSubString(String str, int startPos, JSONArray keys) { + return findSubString(str, startPos, keys, ""); + } + + // 获取回看层数 + public static int getLookbackCount(JSONArray keys) { + try { + if (keys != null && keys.length() > 4) return keys.getInt(4); + } catch (Exception e) { + //e.printStackTrace(); + } + return 0; + } + + // 遍历JSONObect中的JSONArray查找回看的层数可用的规则 + public static JSONArray getLookbackArray(JSONObject obj) { + try { + Iterator iter = obj.keys(); + while (iter.hasNext()) { + String key = (String) iter.next(); + Object val = obj.get(key); + if (val.getClass().getSimpleName().equals("JSONArray")) { + int c = getLookbackCount((JSONArray) val); + if (c > 0) return (JSONArray) val; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + // 统计子串个数 + public static int getSubStringCount(String str, String sub){ + int pos =0; + int count =0; + while (pos < str.length()){ + pos = str.indexOf(sub, pos); + if(pos == -1) break; + pos += sub.length(); + ++count; + } + return count; + } + + // 获取指定区间的字符串 + public static String getRegion(String str, JSONObject obj) { + try { + if (obj == null) return str; + JSONArray region = obj.optJSONArray("region"); + if (region == null) return str; + String prefix = region.getString(0); + int a = str.indexOf(prefix); + if (a == -1) return str; + int b = str.length(); + if (region.length() > 1) { + b = str.indexOf(region.getString(1), a + prefix.length()); + if (b == -1) b = str.length(); + } + return str.substring(a, b); + } catch (JSONException e) { + e.printStackTrace(); + } + return str; + } + + } + + // 猜测分类列表的html区间代码 + protected String guessCateManualHtmlString(String body) { + String regx = String.format(" list = new ArrayList<>(); + int mcount = 0; + while (m.find() ) { + ++mcount; + if(mcount >30 && !list.isEmpty()){ + break; + } + HtmlMatchInfo cate = new HtmlMatchInfo(); + cate.init(m); + cate.group2 = HtmlNodeHlper.trimHtmlString(HtmlNodeHlper.nodeString(body, cate.startPos)); + if (cate.group2.isEmpty()) continue; + boolean bOk = false; + for (String v : cateManuals){ + if(cate.group2.indexOf(v) !=-1) { + bOk = true; + break; + } + } + if(!bOk) continue; + cate.uploads = HtmlNodeHlper.findUpNodes(body, cate.startPos, 3); + if (!list.isEmpty()) { + boolean b = list.get(0).hasSameUpNode(cate); + if (!b) { // 当前找到的info和list中的匹配 + if (list.size() > 1) { // 如果list中的数据大于1 则认为找到了类型列表 + return HtmlNodeHlper.nodeString(body, list.get(0).matchedUpNodePos); + } + list.clear(); + } + } + list.add(cate); + } + if (list.size() > 1) { // 如果list中的数据大于1 则认为找到了类型列表 + return HtmlNodeHlper.nodeString(body, list.get(0).matchedUpNodePos); + } else { + return ""; + } + } + + // 从html代码中猜测分类名和分类ID cateManual规则 + protected JSONObject guess_rule_cateManual(String body) { + try { + String str = this.guessCateManualHtmlString(body); + if (str.isEmpty()) return new JSONObject(); + + String regx = String.format("](\\s*?\\S+?\\s*?)(\"|<)", TextUtils.join("|", cateManuals)); + Pattern pattern = Pattern.compile(regx, Pattern.CASE_INSENSITIVE); + Matcher m = pattern.matcher(str); + ArrayList list = new ArrayList<>(); + while (m.find()) { + // HtmlMatchInfo 字段映射 + // HtmlMatchInfo.group1 -> href + // HtmlMatchInfo.group2 -> name + // HtmlMatchInfo.diff -> id 分类ID + HtmlMatchInfo cate = new HtmlMatchInfo(); + cate.init(m); + if (cate.group1.length() < 5) continue; + cate.group2 = HtmlNodeHlper.trimHtmlString(HtmlNodeHlper.nodeString(str, cate.startPos)); + if(cate.group2.isEmpty()) continue; + // 判断是否为正常的分类名 + boolean validCateName = true; + for (int j = 0; j < invalidCateNames.size(); ++j) { + if (cate.group2.indexOf(invalidCateNames.get(j)) != -1) { + SpiderDebug.log(String.format("排除无效分类：%s --> %s", cate.group1, cate.group2)); + validCateName = false; + break; + } + } + if(!validCateName) continue; + + if (!list.isEmpty()) { + if (!list.get(0).findDiffStr(cate, splitFlag)) { + SpiderDebug.log(String.format("排除可能无效的分类 %s <--> %s", cate.group1, cate.group2)); + continue; + } + } + list.add(cate); + } + + ArrayList baseInfoIndexs = new ArrayList<>(); + // 找到最可能是正确的导航item + for (int i =0; i < list.size(); ++i){ + list.get(i).diff = null; + for (String v : cateManuals){ + if(list.get(i).group2.indexOf(v) !=-1) { + baseInfoIndexs.add(i); + break; + } + } + } + + // 以找到的导航item为基准重建分类ID + int baseInfoIndex=0; + for (int i =1; i < baseInfoIndexs.size(); ++i){ + baseInfoIndex=baseInfoIndexs.get(0).intValue(); + list.get(baseInfoIndex).findDiffStr(list.get(baseInfoIndexs.get(i).intValue()), splitFlag); + } + + JSONObject cateManual = new JSONObject(); + for (int i = 0; i < list.size(); ++i) { + if(list.get(i).diff == null || list.get(i).diff.isEmpty()) { + if(!list.get(baseInfoIndex).findDiffStr(list.get(i), splitFlag)){ + SpiderDebug.log(String.format("排除可能无效的分类 : %s", list.get(i).group0)); + continue; + } + } + + boolean validCateName = true; + String name = list.get(i).group2; + String id = list.get(i).diff; + if (id == null || id.isEmpty()) continue; + if (name == null || name.isEmpty()) continue; + for (int k =0; k arr = HtmlNodeHlper.findUpNodes(body,m.start(0), i); + String r = HtmlNodeHlper.nodeString(body, arr.get(arr.size()-1)); + String regex2 = "action=\"(.+?)\""; + Pattern pattern2 = Pattern.compile(regex2, Pattern.CASE_INSENSITIVE); + Matcher m2 = pattern2.matcher(r); + if(m2.find()){ + String url = m2.group(1); + char ch = url.indexOf('?') ==-1 ? '?' : '&'; + url = addHttpPrefix(url + ch + wd + "={wd}"); + return url; + } + + } + } + return ""; + } + // 猜测列表数据的 vod_id 规则 + public JSONArray guess_rule_vod_id(String body) { + try { + String regx = " founds = new HashMap<>(); + ArrayList list = new ArrayList<>(); + while (m.find()) { + HtmlMatchInfo cate = new HtmlMatchInfo(); + cate.init(m); + cate.uploads = HtmlNodeHlper.findUpNodes(body, cate.startPos, 4); +// String ms = this.findNodeString(body, cate.uploads.get(cate.uploads.size()-1)); + if (!list.isEmpty()) { + + if(cate.group1.equals( list.get(list.size()-1).group1)) continue; + boolean b = list.get(list.size()-1).hasSameUpNode(cate); + if (!b) { // 当前找到的info和list中的匹配 + if (list.size() > 1) { + HtmlMatchInfo info = list.get(0); + info.findDiffStr(list.get(1), splitFlag); + int id = 0; + boolean isNumberID = false; + try { id = Integer.valueOf(info.diff).intValue(); isNumberID = true; }catch (Exception e){} + + if(id > 100 ){ // cateID一般都是小于100的 + String url = (info.group1.replace(list.get(0).diff, "{vid}")); + JSONArray arr = new JSONArray(); + String prefix = url.substring(0, url.indexOf("{vid}")); + String suffix = url.substring(prefix.length() + "{vid}".length()); + int lookback = info.uploads.indexOf(info.matchedUpNodePos) - 1; + if (lookback < 1) lookback = 1; + arr.put(prefix); + arr.put(suffix); + arr.put(0); + arr.put(0); + arr.put(lookback); + arr.put(list.size()); + + if (!founds.containsKey(url)) { + founds.put(url, arr); + } else { + int nlen = founds.get(url).getInt(5) + list.size(); + arr.put(5, nlen); + founds.put(url, arr); + if(nlen >= 30){ + list.clear(); + break; + } + } + } + + } + list.clear(); + } + } + list.add(cate); + if(list.size()>30){ + break; + } + } + + + if (list.size() > 5 || (list.size()>1 && founds.isEmpty())) { // 如果list中的数据大于1 则认为找到了类型列表 + HtmlMatchInfo info = list.get(0); + info.findDiffStr(list.get(1), splitFlag); + int id = 0; + boolean isNumberID = false; + try { id = Integer.valueOf(info.diff).intValue(); isNumberID = true; }catch (Exception e){} + + if(id > 100 ){ // cateID一般都是小于100的 + + String url = (info.group1.replace(list.get(0).diff, "{vid}")); + JSONArray arr = new JSONArray(); + String prefix = url.substring(0, url.indexOf("{vid}")); + String suffix = url.substring(prefix.length() + "{vid}".length()); + int lookback = info.uploads.indexOf(info.matchedUpNodePos) - 1; + if (lookback < 1) lookback = 1; + arr.put(prefix); + arr.put(suffix); + arr.put(0); + arr.put(0); + arr.put(lookback); + arr.put(list.size()); + + if (!founds.containsKey(url)) { + founds.put(url, arr); + } else { + int nlen = founds.get(url).getInt(5) + list.size(); + arr.put(5, nlen); + founds.put(url, arr); + } + } + } + + + JSONArray c = null; + for (String key : founds.keySet()) { + JSONArray v = founds.get(key); + if(c == null || c.getInt(5) < v.getInt(5)) c = v; + } + return c; + + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + // 猜播放列表 + public JSONArray guess_rule_vod_play_url(String str, String vid) { + String regex = "href=\"(/.+?)\""; + Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + Matcher m = pattern.matcher(str); + HtmlMatchInfo info = new HtmlMatchInfo(); + ArrayList vec = new ArrayList<>(); + boolean p0__ = false; + while (m.find()){ + String sb = m.group(1); + // 太长的url认为是错误的播放地址 + if(sb.length() > 100) continue; + if(sb.indexOf(vid) == -1) continue; + // 如果当前url的长度比上一个url的长度短也认为是无效的播放地址（带上了vod_id一般短一点的可能是详情页的地址） + // 一般来讲 999-1-1.html 这种格式是播放页的地址，不排除这种地址 + boolean is__html = (sb.indexOf(vid+"-") != -1); + if(!is__html && vec.size() > 0 && vec.get(vec.size()-1).length() > sb.length()) continue; + if(is__html && !p0__ ){ // 找到了准确度最高的播放连接格式，如果检查到列表头不是这种格式的话，清空列表从头开始 + vec.clear(); + } + // 如果列表里面装了标准的连接格式，不标准的就不要了 + if(p0__ && !is__html) continue; + + info.init(m); + if(vec.isEmpty()) p0__ = is__html; + vec.add(m.group(1)); + if(vec.size() > 10 && vec.get(vec.size()-2).length() == sb.length()) { + break; + } + } + if(info.group0 != null){ + for (int i =1;i < 4; ++i){ + ArrayList nodes = HtmlNodeHlper.findUpNodes(str, info.startPos, i); + int startPos = nodes.get(nodes.size()-1).intValue(); + //String smd = str.substring(startPos, startPos+10); + String smd = HtmlNodeHlper.nodeString(str, startPos); + + if(smd.indexOf(" map = new HashMap(); + + for (int i = 0; i < words.length; ++i) { + words[i] =words[i].trim(); + if (!words[i].isEmpty() && words[i].indexOf("更新") ==-1) { + + int c = 1; + if (map.containsKey(words[i])) { + c = 1 + map.get(words[i]).intValue(); + ;//[words[i]] + } + map.put(words[i], Integer.valueOf(c)); + } + } + String s = ""; + int c = 0; + for (String key : map.keySet()) { + int v = map.get(key).intValue(); + if (v > c) { + c = v; + s = key; + } + } + val = s; + } + return val.replace("在线", "") + .replace("立即", "") + .replace("观看", "") + .replace("点播", "") + .replace("影片", "") + .replace("信息", "") + .replace("播放", "") + .trim(); + //return val; + } catch (Exception e) { + + } + return ""; + } + + public String guess_value_vod_remarks(String nd, int startPos, String vod_name) { + try { + String all = HtmlNodeHlper.trimHtmlString(nd, "!!!!"); + String[] words = all.split("!!!!"); + String val = ""; + for (int i = 0; i < words.length; ++i) { + String wd = words[i].trim(); + if (!wd.isEmpty() && wd.indexOf(vod_name) == -1) { + String dot = (!val.isEmpty()) ? "," : "";// val += ","; + String tmp = val + dot + wd; + if (tmp.length() > 20) { + break; + } + val = tmp; + } + } + return val; + } catch (Exception e) { + + } + return ""; + + } + + public String guess_value_vod_pic(String nd, int startPos) { + try { + JSONArray vec = new JSONArray(); + vec.put("data-original=\""); + vec.put("\""); + String val = Utils.findSubString(nd, startPos, vec); + if (val.isEmpty()) { + vec.put(0, "data-src=\""); + val = Utils.findSubString(nd, startPos, vec); + } + if (val.isEmpty()) { + vec.put(0, "src=\""); + val = Utils.findSubString(nd, startPos, vec); + } + if (val.isEmpty()) { + vec.put(0, "data-bg=\""); + val = Utils.findSubString(nd, startPos, vec); + } + if (val.isEmpty()) { + //TODO: 直接在nd中找个.jpg .png 之类的当图片 + } + return addHttpPrefix(val); + } catch (Exception e) { + + } + return ""; + } + + @Override + public boolean isVideoFormat(String url) { + url = url.toLowerCase(); + if (url.contains("=http") || url.contains("=https") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) { + return false; + } + for (String format : videoFormatList) { + if (url.contains(format)) { + return true; + } + } + return false; + } + + // 让当前爬虫自己判断是否为可播放的地址 + @Override + public boolean manualVideoCheck() { + return true; + } + + + @Override + public String homeContent(boolean z) { + try { + fetchRule(); + + JSONObject result = new JSONObject(); + JSONArray classes = new JSONArray(); + JSONObject cateManual = rule.optJSONObject("cateManual"); + + Iterator keys = cateManual.keys(); + while (keys.hasNext()) { + String key = keys.next(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("type_name", key); + jsonObject.put("type_id", cateManual.getString(key)); + classes.put(jsonObject); + } + + result.put("class", classes); + if (z && rule.has("filter")) { + result.put("filters", rule.getJSONObject("filter")); + } + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + // from xpath 加入过滤条件 + protected String categoryUrl(String tid, String pg, boolean filter, HashMap extend) { + try { + JSONObject list = this.rule.getJSONObject("list"); + String cateUrl = list.optString(pg, ""); + if(cateUrl.isEmpty()) + cateUrl = list.getString("url"); + if (filter && extend != null && extend.size() > 0) { + for (Iterator it = extend.keySet().iterator(); it.hasNext(); ) { + String key = it.next(); + String value = extend.get(key); + if (value.length() > 0) { + cateUrl = cateUrl.replace("{" + key + "}", URLEncoder.encode(value)); + } + } + } + cateUrl = cateUrl.replace("{cateId}", tid).replace("{catePg}", pg); + Matcher m = Pattern.compile("\\{(.*?)\\}").matcher(cateUrl); + while (m.find()) { + String n = m.group(0).replace("{", "").replace("}", ""); + cateUrl = cateUrl.replace(m.group(0), "").replace("/" + n + "/", ""); + } + return cateUrl; + } catch (Exception e) { + e.printStackTrace(); + } + return ""; + } + + @Override + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + try { + JSONObject list = this.rule.getJSONObject("list"); + String url = categoryUrl(tid, pg, filter, extend); + String body = fetchUrl(url, list.optJSONObject("header")); + String str = Utils.getRegion(body, list); + JSONArray videos = new JSONArray(); + JSONArray lookback = Utils.getLookbackArray(list); + Set set = new HashSet(); + int pos = 0; + ArrayList urlnodes = null; + int lookup = -1; + while (lookback != null) { + pos = str.indexOf(lookback.getString(0), pos); + if (pos == -1) break; + + ArrayList arr = null; + int blockPos = 0; + String nd =""; + do { + arr = HtmlNodeHlper.findUpNodes(str, pos - 1, lookback.getInt(4)); + if (urlnodes == null) { + urlnodes = arr; + blockPos = arr.get(arr.size() - 1); + } else { + blockPos = Utils.findBlockPos(urlnodes, arr); + } + nd = HtmlNodeHlper.nodeString(str, blockPos); + + // 检查是否回看层数过多，如果回看导数过多会导致加载不出来数据或一页只加载一条数据，需要进行修正 + if(lookup < 0){ + int count = Utils.getSubStringCount(nd, lookback.getString(0)); + if(count > 3 && lookback.getInt(4)>1){ + lookback.put(4, lookback.getInt(4)-1); + urlnodes = null; + blockPos=0; + nd=""; + SpiderDebug.log(String.format("找到过多的url匹配项(%d)，降低匹配层级为%d", count, lookback.getInt(4))); + }else if(lookup == -1){ + String pic = guess_value_vod_pic(nd,0); //尝试找一下图片，如果没找到的话增加一级 + String vName = guess_value_vod_name(nd,0); + if(pic.isEmpty()||vName.isEmpty()){ + lookback.put(4, lookback.getInt(4)+1); + urlnodes = null; + blockPos=0; + nd=""; + lookup = -2; // 只退一次 + SpiderDebug.log(String.format("当前层级未找到(%s)，增加匹配层级为%d", pic.isEmpty()? "图片": "标题", lookback.getInt(4))); + }else{ + lookup = lookback.getInt(4); + } + }else{ + lookup = lookback.getInt(4); + } + } + }while (lookup < 0 ); + + + pos += nd.length(); + blockPos = 0; + String vod_id = Utils.findSubString(nd, blockPos, list.getJSONArray("vod_id")); + if (!set.contains(vod_id)) { // 排除重复数据 + set.add(vod_id); + JSONObject v = new JSONObject(); + v.put("vod_id", vod_id); + v.put("vod_name", Utils.findSubString(nd, blockPos, list.optJSONArray("vod_name"))); + v.put("vod_pic", addHttpPrefix(Utils.findSubString(nd, blockPos, list.optJSONArray("vod_pic")))); + v.put("vod_remarks", Utils.findSubString(nd, blockPos, list.optJSONArray("vod_remarks"))); + + if (v.getString("vod_name").isEmpty()) { + v.put("vod_name", guess_value_vod_name(nd, 0)); + } + + if (v.getString("vod_pic").isEmpty()) { + v.put("vod_pic", guess_value_vod_pic(nd, 0)); + } + + // 随便整点remark + if (v.getString("vod_remarks").isEmpty()) { + String vod_name = v.getString("vod_name"); + v.put("vod_remarks", guess_value_vod_remarks(nd, 0, vod_name)); + } + v.put("vod_id", Base64.encodeToString(v.toString().getBytes(Misc.CharsetUTF8), base64Flag)); + videos.put(v); + } +// pos += vod_id.length(); + } + + JSONObject result = new JSONObject(); + result.put("page", pg); + result.put("pagecount", Integer.MAX_VALUE); + result.put("limit", 90); + result.put("total", Integer.MAX_VALUE); + result.put("list", videos); + return result.toString(); + } catch (Exception e) { + e.printStackTrace(); + return ""; + } + } + + // 生成播放的名称 + public ArrayList makeVodPlayFrom(int sz) { + ArrayList vec = new ArrayList(); + for (int i = 1; i <= sz; ++i) { + vec.add("播放列表" + i); + } + return vec; + } + + // 查找播放列表名 + public ArrayList findVodPlayFrom(String str, int sz) { + try { + ArrayList urlnodes = null; + JSONObject playlist = this.rule.getJSONObject("playlist"); + if (!playlist.has("vod_play_from")) { + return makeVodPlayFrom(sz); + } + ArrayList> vod_play_from = new ArrayList>(); + JSONArray rule_vod_play_from = playlist.getJSONArray("vod_play_from"); + for (int i = 0; i < rule_vod_play_from.length(); ++i) { + String s = rule_vod_play_from.get(i).getClass().getSimpleName(); + String key = ""; + String alias = ""; + if (s.equals("String")) { + key = alias = rule_vod_play_from.getString(i); + } else if (s.equals("JSONArray")) { + JSONArray item = rule_vod_play_from.getJSONArray(i); + key = alias = item.getString(0); + if (item.length() > 1) { + alias = item.getString(1); + } + } else { + return makeVodPlayFrom(sz); + } + + int pos = str.indexOf(key); + if (pos == -1) continue; + vod_play_from.add(new Pair<>(pos, alias)); + } + // 找到的名称与实际需要的数量不匹配，返回默认的名称 + if (vod_play_from.size() != sz) { + return makeVodPlayFrom(sz); + } + // 排序 + Collections.sort(vod_play_from, new Comparator>() { + @Override + public int compare(Pair a, Pair b) { + return a.first.intValue() - b.first.intValue(); + } + }); + + ArrayList vec = new ArrayList(); + for (int i = 0; i < vod_play_from.size(); ++i) { + vec.add(vod_play_from.get(i).second); + } + return vec; + } catch (Exception e) { + e.printStackTrace(); + } + return makeVodPlayFrom(sz); + } + + // 查找播放列表 + public ArrayList findVodPlayUrl(String str) { + ArrayList tmp_vod_play_url = new ArrayList(); + ArrayList vod_play_url = new ArrayList(); + try { + int pos = 0; + ArrayList urlnodes = null; + JSONObject playlist = this.rule.getJSONObject("playlist"); + int sort = playlist.optInt("sort", 0); // 如果这个值是0。表示要倒序播放列表 + HashMap map = new HashMap(); + Set rmset = new HashSet(); +// String tmp = ""; + ArrayList tmp = new ArrayList(); + JSONArray lookback = Utils.getLookbackArray(playlist); + while (lookback != null) { + JSONArray rule_vod_play_url = playlist.getJSONArray("vod_play_url"); + pos = str.indexOf(rule_vod_play_url.getString(0), pos); + if (pos == -1) break; + ArrayList arr = HtmlNodeHlper.findUpNodes(str, pos - 1, lookback.getInt(4)); + + int blockPos = 0; + if (urlnodes == null || arr.size() != urlnodes.size() || arr.get(arr.size() - 1).intValue() != urlnodes.get(urlnodes.size() - 1).intValue()) { + urlnodes = arr; + blockPos = arr.get(Math.max(0, arr.size() - 2)); + // 如果上级节点不同，说明当前播放列表已经结束，可以 + if (!tmp.isEmpty()) { + SpiderDebug.log("change play list "); + if (sort != 0) { + Collections.reverse(tmp); + } + tmp_vod_play_url.add(TextUtils.join("#", tmp)); + tmp = new ArrayList(); + } + } else { + blockPos = Utils.findBlockPos(urlnodes, arr); + } +// String bs = str.substring(blockPos, blockPos + 1000); +// String b = this.findNodeString(str, blockPos); + // 处理play_url 为空的情况 + String play_url = addHttpPrefix(Utils.findSubString(str, blockPos, playlist.getJSONArray("vod_play_url"))); + + if (map.containsKey(play_url)) { // 如果以经找过当前播放的url,那么认为之前找到的都是垃圾数据，清空之间的成果 + SpiderDebug.log("发现重复播放连接，清空已解析到的播放列表"); + rmset.add(map.get(play_url)); // 添加移除标志 + } + map.put(play_url, tmp_vod_play_url.size()); + String play_url_title = Utils.findSubString(str, blockPos, playlist.optJSONArray("vod_play_url_title")); + if (play_url_title.isEmpty()) { + play_url_title = HtmlNodeHlper.trimHtmlString(HtmlNodeHlper.nodeString(str, blockPos)); + int dd = 3; + } + tmp.add(play_url_title + "$" + play_url); +// SpiderDebug.log(String.format("%s$%s", play_url_title, play_url)); + pos += play_url.length(); + } + + if (!tmp.isEmpty()) { + if (sort != 0) { + Collections.reverse(tmp); + } + tmp_vod_play_url.add(TextUtils.join("#", tmp)); + } + + for (int i = 0; i < tmp_vod_play_url.size(); ++i) { + if (!rmset.contains(i)) { + vod_play_url.add(tmp_vod_play_url.get(i)); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return vod_play_url; + } + + // 猜测详情数据的html区间 + protected String guessDetailContentRegion(String body) { + String regx = String.format(">\\s*?(%s)|(%s)", TextUtils.join("|", detailItemNames), TextUtils.join("：|", detailItemNames)); + Pattern pattern = Pattern.compile(regx, Pattern.CASE_INSENSITIVE); + Matcher m = pattern.matcher(body); + ArrayList list = new ArrayList<>(); + while (m.find()) { + HtmlMatchInfo cate = new HtmlMatchInfo(); + cate.init(m); + cate.uploads = HtmlNodeHlper.findUpNodes(body, cate.startPos, 5); + if (!list.isEmpty()) { + boolean b = list.get(0).hasSameUpNode(cate); + if (!b) { // 当前找到的info和list中的匹配 + if (list.size() > 1) { + boolean found = false; + for (int i = 0; i < list.size(); ++i) { + if (list.get(i).group0.indexOf("导演") != -1) { + found = true; + } + } + if (found) { + return HtmlNodeHlper.nodeString(body, list.get(0).matchedUpNodePos); + } else { + list.clear(); + } + } + list.clear(); + } + } + list.add(cate); + } + if (list.size() > 1) { // 如果list中的数据大于1 则认为找到了类型列表 + return HtmlNodeHlper.nodeString(body, list.get(0).matchedUpNodePos); + } else { + return ""; + } + } + + @Override + public String detailContent(List ids) { + try { + fetchRule(); + JSONObject vinfo = new JSONObject(new String(Base64.decode(ids.get(0), base64Flag), "UTF-8")); + + JSONObject detail = rule.optJSONObject("detail"); + if (detail == null) return ""; + // 如果不存在url,则使用list中的vod_id来生成url + if (!detail.has("url")) { + JSONObject list = rule.getJSONObject("list"); + JSONArray tmp = list.getJSONArray("vod_id"); + String u = addHttpPrefix(tmp.getString(0) + "{vid}" + tmp.getString(1)); + detail.put("url", u); + } + String url = detail.getString("url").replace("{vid}", vinfo.getString("vod_id")); + String body = fetchUrl(url, detail.optJSONObject("header")); + String str = Utils.getRegion(body, detail); + int startPos = 0; + + String nodeString = ""; + // 圈定 详情数据的范围 + JSONArray lookback = Utils.getLookbackArray(detail); + if (lookback != null) { + int pos = str.indexOf(lookback.getString(0), 0); + if (pos != -1) { + ArrayList arr = HtmlNodeHlper.findUpNodes(str, pos - 1, lookback.getInt(4)); + if (arr.size() > 0) { + startPos = arr.get(arr.size() - 1); + nodeString = HtmlNodeHlper.nodeString(str, startPos); // 精确详情数据的范围 + } + } + } + // 没有指定详情数据范围则猜一个出来 + if (nodeString.isEmpty()) { + nodeString = this.guessDetailContentRegion(body); + } + + if (nodeString.length() != str.length()) { + str = nodeString; + startPos = 0; + } + /////////////////////////////////////////////////////////////////////////////////////// + + JSONObject vod = new JSONObject(); + vod.put("vod_id", ids.get(0)); + vod.put("vod_name", Utils.findSubString(str, startPos, detail.optJSONArray("vod_name"))); + vod.put("vod_pic", addHttpPrefix(Utils.findSubString(str, startPos, detail.optJSONArray("vod_pic")))); + vod.put("type_name", Utils.findSubString(str, startPos, detail.optJSONArray("type_name"))); + vod.put("vod_year", Utils.findSubString(str, startPos, detail.optJSONArray("vod_year"))); + vod.put("vod_area", Utils.findSubString(str, startPos, detail.optJSONArray("vod_area"))); + vod.put("vod_remarks", Utils.findSubString(str, startPos, detail.optJSONArray("vod_remarks"))); + vod.put("vod_actor", Utils.findSubString(str, startPos, detail.optJSONArray("vod_actor"))); + vod.put("vod_director", Utils.findSubString(str, startPos, detail.optJSONArray("vod_director"))); + vod.put("vod_content", Utils.findSubString(str, startPos, detail.optJSONArray("vod_content"))); + + + //////////////////////////////////////////////////////////////////////////////////////// + if (vod.getString("vod_name").isEmpty()) { + vod.put("vod_name", vinfo.optString("vod_name", "")); + } + // 从页面中猜个视频名称出来 + if (vod.getString("vod_name").isEmpty()) { + vod.put("vod_name", guess_value_vod_name(str, startPos)); + } + + //////////////////////////////////////////////////////////////////////////////////////// + if (vod.getString("vod_pic").isEmpty()) { + vod.put("vod_pic", vinfo.optString("vod_pic", "")); + } + if (vod.getString("vod_pic").isEmpty()) { + vod.put("vod_pic", guess_value_vod_pic(str, startPos)); + } + + //////////////////////////////////////////////////////////////////////////////////////// + if (lookback != null && lookback.length() > 1) { + JSONArray key = new JSONArray(); + String name = lookback.getString(0); + String skey = lookback.getString(0); + + ArrayList detailItems = new ArrayList<>(Arrays.asList("导演", "演员", "类型", "年份")); + for (String p : detailItems) { + if (name.indexOf(p) != -1) { + skey = p; + break; + } + } + key.put(name); + key.put(lookback.getString(1)); + if (vod.getString("vod_director").isEmpty()) { + key.put(0, name.replace(skey, "导演")); + vod.put("vod_director", Utils.findSubString(str, startPos, key)); + } + if (vod.getString("vod_actor").isEmpty()) { + key.put(0, name.replace(skey, "主演")); + vod.put("vod_actor", Utils.findSubString(str, startPos, key)); + } + + if (vod.getString("vod_content").isEmpty()) { + String all = HtmlNodeHlper.trimHtmlString(str, "!!!!"); + String[] words = all.split("!!!!"); + String v = ""; + for (int i = 0; i < words.length; ++i) { + if (words[i].length() > v.length()) { + v = words[i]; + } + } + vod.put("vod_content", HtmlNodeHlper.trimHtmlString(v)); + } + + } else { // 猜一下详情数据 + if (vod.getString("vod_director").isEmpty()) { + ArrayList arr = HtmlNodeHlper.getChildNodes(nodeString); + String content = ""; + String f = TextUtils.join("|", detailItemNames); + String regex = String.format("%s",f); + Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + for (int i = 0; i < arr.size(); ++i) { + String p = HtmlNodeHlper.trimHtmlString(arr.get(i), " ").replace("：", ""); + if (p.length() > content.length()) { content = p; } + String[] all = p.split(regex); + // split出来的可能存在空字符串，去除掉 + ArrayList items = new ArrayList<>(); + for(String c: all){ + if(c.isEmpty()) continue; + items.add(c); + } + Matcher m = pattern.matcher(p); + int index = 0; + while (m.find() && index < items.size()){ + String s = m.group(0); + for (int j = 0; j < detailItemNames.size(); ++j) { + String name = detailItemNames.get(j); + String key = detailItemKeys.get(j); + if (s.indexOf(name) != -1) { + if (vod.getString(key).isEmpty()) { + vod.put(key, items.get(index).trim()); + } + break; + } + } + ++index; + } + } + + if (vod.getString("vod_content").isEmpty()) { + vod.put("vod_content", content); + } + } + } + + + playlistContent(ids, vod, body);// 获取播放列表 + + + JSONObject result = new JSONObject(); + JSONArray list = new JSONArray(); + list.put(vod); + result.put("list", list); + return result.toString(); + } catch (Exception e) { + e.printStackTrace(); + return ""; + } + + } + + // 播放页 str 为 detailContent 函数中http返回值 + protected void playlistContent(List ids, JSONObject vod, String body) { + try { + fetchRule(); + JSONObject vinfo = new JSONObject(new String(Base64.decode(ids.get(0), base64Flag), "UTF-8")); + + JSONObject playlist = rule.optJSONObject("playlist"); + if (playlist == null) return; + if (playlist.has("url")) { + String detailUrl = rule.getJSONObject("detail").optString("url"); + String playListUrl = playlist.getString("url"); + if(!detailUrl.equals(playListUrl)){ + String url = playlist.getString("url").replace("{vid}", vinfo.getString("vod_id")); + body = fetchUrl(url, playlist.optJSONObject("header")); + } + } + String str = Utils.getRegion(body, playlist); + + ArrayList vod_play_url = null; + if (!playlist.has("vod_play_url")) { + // 猜vod_play_url的查找规则 + JSONArray vod_play_url_rule = this.guess_rule_vod_play_url(str, vinfo.getString("vod_id")); + if(vod_play_url_rule != null){ + playlist.put("vod_play_url", vod_play_url_rule); + } + } + vod_play_url = this.findVodPlayUrl(str); + ArrayList vod_play_from = this.findVodPlayFrom(str, vod_play_url.size()); + + // 如果有说明播放源的名称，那么对播放源进行排序 + String f1 = TextUtils.join("$$$", vod_play_from); + String f2 = TextUtils.join("$$$", makeVodPlayFrom(vod_play_url.size())); + + if (!f1.equals(f2)) { + ArrayList urls = new ArrayList<>(); + ArrayList froms = new ArrayList<>(); + + JSONArray rule_vod_play_from = playlist.getJSONArray("vod_play_from"); + for (int i = 0; i < rule_vod_play_from.length(); ++i) { + String s = rule_vod_play_from.get(i).getClass().getSimpleName(); + String alias = ""; + if (s.equals("String")) { + alias = rule_vod_play_from.getString(i); + } else if (s.equals("JSONArray")) { + JSONArray item = rule_vod_play_from.getJSONArray(i); + alias = item.getString(0); + if (item.length() > 1) { + alias = item.getString(1); + } + } + + for (int j = 0; j < vod_play_from.size(); ++j) { + if (vod_play_from.get(j).equals(alias)) { + urls.add(vod_play_url.get(j)); + froms.add(vod_play_from.get(j)); + } + } + + } + vod_play_url = urls; + vod_play_from = froms; + } + vod.put("vod_play_url", TextUtils.join("$$$", vod_play_url)); + vod.put("vod_play_from", TextUtils.join("$$$", vod_play_from)); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + // 尝试从播放页中找播放url + protected String parsePlayUrl(String str, String str2, List list) { + try { + JSONObject play = rule.optJSONObject("play"); + if (play == null) { + return ""; + } + String tmp = fetchUrl(str2, play.optJSONObject("header")); + String body = Utils.getRegion(tmp, play); + int startPos = 0; + JSONArray lookback = Utils.getLookbackArray(play); + if (lookback != null) { + int pos = body.indexOf(lookback.getString(0), 0); + if (pos != -1) { + ArrayList arr = HtmlNodeHlper.findUpNodes(body, pos - 1, lookback.getInt(4)); + if (arr.size() > 0) { + startPos = arr.get(arr.size() - 1); + } else { + startPos = pos; + } + } + } + + String vod_url = Utils.findSubString(body, startPos, play.optJSONArray("vod_url")); + vod_url = vod_url.replace("\\/", "/"); + if (vod_url.isEmpty() || !isVideoFormat(vod_url)) return ""; + JSONObject result = new JSONObject(); + result.put("parse", 0); + result.put("playUrl", ""); + result.put("url", vod_url); + return result.toString(); + + } catch (Exception e) { + e.printStackTrace(); + } + + return ""; + } + + @Override + public String playerContent(String str, String str2, List list) { + try { + fetchRule(); + // 先判断是否可以拿到直链 + String ret = parsePlayUrl(str, str2, list); + if (!ret.isEmpty()) return ret; + // 直接将网页地址返回回去进行嗅探 + JSONObject result = new JSONObject(); + result.put("parse", 1); + result.put("playUrl", ""); + result.put("url", str2); + return result.toString(); + } catch (Exception e) { + e.printStackTrace(); + } + return ""; + } + + protected Object parseJsonSearchResult(Object obj) { + try { + if (obj == null) return null; + JSONObject search = rule.optJSONObject("search"); + if (search == null) return null; + String key_vod_id = search.getString("vod_id"); + String key_vod_name = search.getString("vod_name"); + String type = obj.getClass().getSimpleName(); + if (type.equals("JSONObject")) { + JSONObject object = (JSONObject) obj; + if (object.has(key_vod_id) && object.has(key_vod_name)) return object; + for (Iterator iter = object.keys(); iter.hasNext(); ) { + String k = iter.next(); + Object r = parseJsonSearchResult(object.get(k)); + if (r != null) { + return r; + } + } + } else if (type.equals("JSONArray")) { + JSONArray array = (JSONArray) obj; + for (int i = 0; i < array.length(); ++i) { + if (parseJsonSearchResult(array.get(i)) != null) { + return array; + } + } + } + + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + protected String parseSearchResult(String body) { + try { + JSONObject obj = new JSONObject(body); + Object info = parseJsonSearchResult(obj); + if (info == null) return ""; + JSONArray arr = new JSONArray(); + if (info.getClass().getSimpleName().equals("JSONObject")) { + arr.put(info); + } else { + arr = (JSONArray) info; + } + JSONObject search = rule.optJSONObject("search"); + JSONArray videos = new JSONArray(); + for (int i = 0; i < arr.length(); ++i) { + JSONObject v = new JSONObject(); + JSONObject o = arr.getJSONObject(i); + if (search.has("vod_id") && o.has(search.getString("vod_id"))) { + v.put("vod_id", o.get(search.getString("vod_id")).toString()); + } else { + continue; + } + if (search.has("vod_name") && o.has(search.getString("vod_name"))) { + v.put("vod_name", o.get(search.getString("vod_name")).toString()); + } else { + v.put("vod_name", "未知"); + } + + if (search.has("vod_pic") && o.has(search.getString("vod_pic"))) { + v.put("vod_pic", o.get(search.getString("vod_pic")).toString()); + } else { + v.put("vod_pic", ""); + } + + if (search.has("vod_remarks") && o.has(search.getString("vod_remarks"))) { + v.put("vod_remarks", o.get(search.getString("vod_remarks")).toString()); + } else { + v.put("vod_remarks", ""); + } + v.put("vod_id", Base64.encodeToString(v.toString().getBytes(Misc.CharsetUTF8), base64Flag)); + videos.put(v); + } + JSONObject result = new JSONObject(); + result.put("list", videos); + return result.toString(); + + } catch (Exception e) { +// e.printStackTrace(); + } + return ""; + } + + // post 搜索，只做了简单的支持 + protected String postSearch(String wd, boolean z) { + try { + JSONObject search = rule.optJSONObject("search"); + if (search == null) return ""; + String url = search.getString("url"); + JSONObject params = search.getJSONObject("post"); // 查询参数 + HashMap reqpayload = new HashMap<>(); + Iterator iter = params.keys(); + while (iter.hasNext()) { + String key = iter.next(); + String value = params.getString(key).replace("{wd}", wd); + reqpayload.put(key, value); + } + final String[] retval = {""}; + HashMap header = getHeaders(url); + header.put("content-type", "application/x-www-form-urlencoded"); + OkHttpUtil.post(OkHttpUtil.defaultClient(), url, reqpayload, header, new OKCallBack.OKCallBackString() { + @Override + protected void onFailure(Call call, Exception e) { + + } + + @Override + protected void onResponse(String response) { + retval[0] = response; + } + }); + return retval[0]; + } catch (Exception e) { + + } + return ""; + } + + @Override + public String searchContent(String wd, boolean z) { + try { + fetchRule(); + JSONObject search = rule.optJSONObject("search"); + if (search == null) return ""; + String str = ""; + if (search.has("post")) { + str = postSearch(wd, z); + } else { + String url = search.getString("url").replace("{wd}", wd); + str = fetchUrl(url, search.optJSONObject("header")); + } + str = Utils.getRegion(str, search); + // 先当JSON解析试试 + String r = parseSearchResult(str); + if (r != null && !r.isEmpty()) { + return r; + } + + if (!search.has("vod_id")) { + JSONObject list = rule.getJSONObject("list"); + search.put("vod_id", list.getJSONArray("vod_id")); + } + + JSONArray videos = new JSONArray(); + Set set = new HashSet(); + int pos = 0; + ArrayList urlnodes = null; + + JSONArray lookback = Utils.getLookbackArray(search); + + int lookup = -1; + while (lookback != null) { + pos = str.indexOf(lookback.getString(0), pos); + if (pos == -1) break; + + ArrayList arr = null; + int blockPos = 0; + String nd =""; + do { + arr = HtmlNodeHlper.findUpNodes(str, pos - 1, lookback.getInt(4)); + if (urlnodes == null) { + urlnodes = arr; + blockPos = arr.get(arr.size() - 1); + } else { + blockPos = Utils.findBlockPos(urlnodes, arr); + } + nd = HtmlNodeHlper.nodeString(str, blockPos); + // 检查是否回看层数过多，如果回看导数过多会导致加载不出来数据或一页只加载一条数据，需要进行修正 + if(lookup < 0){ + int count = Utils.getSubStringCount(nd, lookback.getString(0)); + if(count > 3 && lookback.getInt(4)>1){ + lookback.put(4, lookback.getInt(4)-1); + urlnodes = null; + blockPos=0; + nd=""; + SpiderDebug.log(String.format("找到过多的url匹配项(%d)，降低匹配层级为%d", count, lookback.get(4))); + } + else if(lookup == -1){ + String pic = guess_value_vod_pic(nd,0); //尝试找一下图片，如果没找到的话增加一级 + String vName = guess_value_vod_name(nd,0); + if(pic.isEmpty()||vName.isEmpty()){ + lookback.put(4, lookback.getInt(4)+1); + urlnodes = null; + blockPos=0; + nd=""; + lookup = -2; // 只退一次 + SpiderDebug.log(String.format("当前层级未找到(%s)，增加匹配层级为%d", pic.isEmpty()? "图片": "标题", lookback.getInt(4))); + }else{ + lookup = lookback.getInt(4); + } + }else{ + lookup = lookback.getInt(4); + } + } + }while (lookup < 0); + + pos += nd.length(); + blockPos = 0; + String vod_id = Utils.findSubString(nd, blockPos, search.getJSONArray("vod_id")); + if (!set.contains(vod_id)) { + set.add(vod_id); + JSONObject v = new JSONObject(); + v.put("vod_id", vod_id); + v.put("vod_name", Utils.findSubString(nd, blockPos, search.optJSONArray("vod_name"))); + v.put("vod_pic", addHttpPrefix(Utils.findSubString(nd, blockPos, search.optJSONArray("vod_pic")))); + v.put("vod_remarks", Utils.findSubString(nd, blockPos, search.optJSONArray("vod_remarks"))); + + if (v.getString("vod_name").isEmpty()) { + v.put("vod_name", guess_value_vod_name(nd, 0)); + } + + if (v.getString("vod_pic").isEmpty()) { + v.put("vod_pic", guess_value_vod_pic(nd, 0)); + } + // 随便整点remark + if (v.getString("vod_remarks").isEmpty()) { + String vod_name = v.getString("vod_name"); + v.put("vod_remarks", guess_value_vod_remarks(nd, 0, vod_name)); + } + v.put("vod_id", Base64.encodeToString(v.toString().getBytes(Misc.CharsetUTF8), base64Flag)); + videos.put(v); + } + } + + JSONObject result = new JSONObject(); + result.put("list", videos); + return result.toString(); + } catch (Exception e) { + e.printStackTrace(); + return ""; + } + } + + protected String fetchUrl(String url, JSONObject h) { + String html = OkHttpUtil.string(url, getHeaders(url)); + html = this.jumpbtwaf(url, html, h); + return html.replaceAll("", "").replace("\r\n","").replace("\n",""); // 移除注释 + } + + protected String jumpbtwaf(String webUrl, String html, JSONObject h) { + + try { + // 没有配置btwaf不执行下面的代码 + if (!rule.optBoolean("btwaf", false)) { + return html; + } + + if (html.contains("检测中") && html.contains("跳转中") && html.contains("btwaf")) { + JSONArray keys = new JSONArray(); + keys.put("btwaf="); + keys.put("\""); + String btwaf = Utils.findSubString(html, 0, keys); + String bturl = webUrl + "?btwaf=" + btwaf; + + Map> cookies = new HashMap<>(); + OkHttpUtil.string(bturl, getHeaders(webUrl), cookies); + for (Map.Entry> entry : cookies.entrySet()) { + if (entry.getKey().equals("set-cookie") || entry.getKey().equals("Set-Cookie")) { + String btcookie = TextUtils.join(";", entry.getValue()); + if (!rule.has("header")) { + rule.put("header", new JSONObject()); + } + rule.getJSONObject("header").put("cookie", btcookie); + break; + } + } + html = fetchUrl(webUrl, h); + } + if (!html.contains("检测中") && !html.contains("btwaf")) { + return html; + } + + } catch (Exception e) { + e.printStackTrace(); + } + return html; + } + +} diff --git a/app/src/main/java/com/github/catvod/spider/XBiubiu.java b/app/src/main/java/com/github/catvod/spider/XBiubiu.java new file mode 100644 index 00000000..875de9f9 --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/XBiubiu.java @@ -0,0 +1,528 @@ +package com.github.catvod.spider; + +import android.content.Context; +import android.text.TextUtils; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.utils.Misc; +import com.github.catvod.utils.okhttp.OKCallBack; +import com.github.catvod.utils.okhttp.OkHttpUtil; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.jsoup.Jsoup; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import okhttp3.Call; + +public class XBiubiu extends Spider { + + @Override + public void init(Context context) { + super.init(context); + } + + public void init(Context context, String extend) { + super.init(context, extend); + this.ext = extend; + } + + @Override + public String homeContent(boolean filter) { + try { + fetchRule(); + JSONObject result = new JSONObject(); + JSONArray classes = new JSONArray(); + String[] fenleis = getRuleVal("fenlei", "").split("#"); + for (String fenlei : fenleis) { + String[] info = fenlei.split("\\$"); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("type_name", info[0]); + jsonObject.put("type_id", info[1]); + classes.put(jsonObject); + } + result.put("class", classes); + return result.toString(); + } catch ( + Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected HashMap getHeaders(String url) { + HashMap headers = new HashMap<>(); + String ua = getRuleVal("ua", Misc.CHROME).trim(); + if (ua.isEmpty()) + ua = Misc.CHROME; + headers.put("User-Agent", ua); + return headers; + } + + @Override + public String homeVideoContent() { + try { + fetchRule(); + if (getRuleVal("shouye").equals("1")) { + JSONArray videos = new JSONArray(); + String[] fenleis = getRuleVal("fenlei", "").split("#"); + for (String fenlei : fenleis) { + String[] info = fenlei.split("\\$"); + JSONObject data = category(info[1], "1", false, new HashMap<>()); + if (data != null) { + JSONArray vids = data.optJSONArray("list"); + if (vids != null) { + for (int i = 0; i < vids.length() && i < 5; i++) { + videos.put(vids.getJSONObject(i)); + } + } + } + if (videos.length() >= 30) + break; + } + JSONObject result = new JSONObject(); + result.put("list", videos); + return result.toString(); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + private JSONObject category(String tid, String pg, boolean filter, HashMap extend) { + try { + fetchRule(); + if (tid.equals("空")) + tid = ""; + String qishiye = rule.optString("qishiye", "nil"); + if (qishiye.equals("空")) + pg = ""; + else if (!qishiye.equals("nil")) { + pg = String.valueOf(Integer.parseInt(pg) - 1 + Integer.parseInt(qishiye)); + } + String webUrl = getRuleVal("url") + tid + pg + getRuleVal("houzhui"); + String html = fetch(webUrl); + html = removeUnicode(html); + String parseContent = html; + boolean shifouercijiequ = getRuleVal("shifouercijiequ").equals("1"); + if (shifouercijiequ) { + String jiequqian = getRuleVal("jiequqian"); + String jiequhou = getRuleVal("jiequhou"); + parseContent = subContent(html, jiequqian, jiequhou).get(0); + } + String jiequshuzuqian = getRuleVal("jiequshuzuqian"); + String jiequshuzuhou = getRuleVal("jiequshuzuhou"); + JSONArray videos = new JSONArray(); + ArrayList jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou); + for (int i = 0; i < jiequContents.size(); i++) { + try { + String jiequContent = jiequContents.get(i); + String title = removeHtml(subContent(jiequContent, getRuleVal("biaotiqian"), getRuleVal("biaotihou")).get(0)); + String pic = ""; + String tupianqian = getRuleVal("tupianqian").toLowerCase(); + if (tupianqian.startsWith("http://") || tupianqian.startsWith("https://")) { + pic = getRuleVal("tupianqian"); + } else { + pic = subContent(jiequContent, getRuleVal("tupianqian"), getRuleVal("tupianhou")).get(0); + } + pic = Misc.fixUrl(webUrl, pic); + String link = subContent(jiequContent, getRuleVal("lianjieqian"), getRuleVal("lianjiehou")).get(0); + link = getRuleVal("ljqianzhui").isEmpty() ? (link + getRuleVal("ljhouzhui")) : ("x:" + getRuleVal("ljqianzhui")) + link + getRuleVal("ljhouzhui"); + String remark = !getRuleVal("fubiaotiqian").isEmpty() && !getRuleVal("fubiaotihou").isEmpty() ? + removeHtml(subContent(jiequContent, getRuleVal("fubiaotiqian"), getRuleVal("fubiaotihou")).get(0)) : ""; + JSONObject v = new JSONObject(); + v.put("vod_id", title + "$$$" + pic + "$$$" + link); + v.put("vod_name", title); + v.put("vod_pic", pic); + v.put("vod_remarks", remark); + videos.put(v); + } catch (Throwable th) { + th.printStackTrace(); + } + } + JSONObject result = new JSONObject(); + result.put("page", pg); + result.put("pagecount", Integer.MAX_VALUE); + result.put("limit", 90); + result.put("total", Integer.MAX_VALUE); + result.put("list", videos); + return result; + } catch (Exception e) { + SpiderDebug.log(e); + } + return null; + } + + private static String removeUnicode(String str) { + Pattern pattern = Pattern.compile("(\\\\u(\\w{4}))"); + Matcher matcher = pattern.matcher(str); + while (matcher.find()) { + String full = matcher.group(1); + String ucode = matcher.group(2); + char c = (char) Integer.parseInt(ucode, 16); + str = str.replace(full, c + ""); + } + return str; + } + + String removeHtml(String text) { + return Jsoup.parse(text).text(); + } + + @Override + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + JSONObject obj = category(tid, pg, filter, extend); + return obj != null ? obj.toString() : ""; + } + + @Override + public String detailContent(List ids) { + try { + fetchRule(); + String[] idInfo = ids.get(0).split("\\$\\$\\$"); + String webUrl = idInfo[2].startsWith("x:") ? idInfo[2] : getRuleVal("url") + idInfo[2]; + String html = fetch(webUrl.startsWith("x:") ? webUrl.substring(2) : webUrl); + String parseContent = html; + boolean bfshifouercijiequ = getRuleVal("bfshifouercijiequ").equals("1"); + if (bfshifouercijiequ) { + String jiequqian = getRuleVal("bfjiequqian"); + String jiequhou = getRuleVal("bfjiequhou"); + parseContent = subContent(html, jiequqian, jiequhou).get(0); + } + + ArrayList playList = new ArrayList<>(); + boolean playDirect = getRuleVal("直接播放").equals("1"); + if (!playDirect) { + String jiequshuzuqian = getRuleVal("bfjiequshuzuqian"); + String jiequshuzuhou = getRuleVal("bfjiequshuzuhou"); + boolean bfyshifouercijiequ = getRuleVal("bfyshifouercijiequ").equals("1"); + ArrayList jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou); + for (int i = 0; i < jiequContents.size(); i++) { + try { + String jiequContent = jiequContents.get(i); + String parseJqContent = bfyshifouercijiequ ? subContent(jiequContent, getRuleVal("bfyjiequqian"), getRuleVal("bfyjiequhou")).get(0) : jiequContent; + ArrayList lastParseContents = subContent(parseJqContent, getRuleVal("bfyjiequshuzuqian"), getRuleVal("bfyjiequshuzuhou")); + List vodItems = new ArrayList<>(); + for (int j = 0; j < lastParseContents.size(); j++) { + String title = subContent(lastParseContents.get(j), getRuleVal("bfbiaotiqian"), getRuleVal("bfbiaotihou")).get(0); + String link = subContent(lastParseContents.get(j), getRuleVal("bflianjieqian"), getRuleVal("bflianjiehou")).get(0); + String bfqianzhui = getRuleVal("bfqianzhui"); + if (!bfqianzhui.isEmpty()) { + link = bfqianzhui + link; + } + vodItems.add(title + "$" + link); + } + playList.add(TextUtils.join("#", vodItems)); + } catch (Throwable th) { + th.printStackTrace(); + } + } + + } else { + playList.add(idInfo[0] + "$" + idInfo[2]); + } + + String cover = idInfo[1], title = idInfo[0], desc = "", category = "", area = "", year = "", remark = "", director = "", actor = ""; + + if (!getRuleVal("leixinqian").isEmpty() && !getRuleVal("leixinhou").isEmpty()) { + try { + category = subContent(html, getRuleVal("leixinqian"), getRuleVal("leixinhou")).get(0).replaceAll("\\s+", "").replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", ""); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!getRuleVal("niandaiqian").isEmpty() && !getRuleVal("niandaihou").isEmpty()) { + try { + year = subContent(html, getRuleVal("niandaiqian"), getRuleVal("niandaihou")).get(0).replaceAll("\\s+", "").replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", ""); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!getRuleVal("zhuangtaiqian").isEmpty() && !getRuleVal("zhuangtaihou").isEmpty()) { + try { + remark = subContent(html, getRuleVal("zhuangtaiqian"), getRuleVal("zhuangtaihou")).get(0); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!getRuleVal("zhuyanqian").isEmpty() && !getRuleVal("zhuyanhou").isEmpty()) { + try { + actor = subContent(html, getRuleVal("zhuyanqian"), getRuleVal("zhuyanhou")).get(0).replaceAll("\\s+", ""); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!getRuleVal("daoyanqian").isEmpty() && !getRuleVal("daoyanhou").isEmpty()) { + try { + director = subContent(html, getRuleVal("daoyanqian"), getRuleVal("daoyanhou")).get(0).replaceAll("\\s+", ""); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!getRuleVal("juqingqian").isEmpty() && !getRuleVal("juqinghou").isEmpty()) { + try { + desc = subContent(html, getRuleVal("juqingqian"), getRuleVal("juqinghou")).get(0); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + JSONObject vod = new JSONObject(); + vod.put("vod_id", ids.get(0)); + vod.put("vod_name", title); + vod.put("vod_pic", cover); + vod.put("type_name", category); + vod.put("vod_year", year); + vod.put("vod_area", area); + vod.put("vod_remarks", remark); + vod.put("vod_actor", actor); + vod.put("vod_director", director); + vod.put("vod_content", desc); + ArrayList playFrom = new ArrayList<>(); + String xlparseContent = html; + if(getRuleVal("xlbiaotiqian").isEmpty() && getRuleVal("xlbiaotihou").isEmpty()){ + + for (int i = 0; i < playList.size(); i++) { + playFrom.add("播放列表" + (i + 1)); + } + }else{ + + boolean xlshifouercijiequ = getRuleVal("xlshifouercijiequ").equals("1"); + if (xlshifouercijiequ) { + String xljiequqian = getRuleVal("xljiequqian"); + String xljiequhou = getRuleVal("xljiequhou"); + xlparseContent = subContent(html, xljiequqian, xljiequhou).get(0); + } + + String xljiequshuzuqian = getRuleVal("xljiequshuzuqian"); + String xljiequshuzuhou = getRuleVal("xljiequshuzuhou"); + ArrayList xljiequContents = subContent(xlparseContent, xljiequshuzuqian, xljiequshuzuhou); + for (int i = 0; i < playList.size(); i++) { + try { + String xltitle = subContent(xljiequContents.get(i), getRuleVal("xlbiaotiqian"), getRuleVal("xlbiaotihou")).get(0); + playFrom.add(xltitle); + } catch (Throwable th) { + th.printStackTrace(); + break; + } + } + + } + String vod_play_from = TextUtils.join("$$$", playFrom); + String vod_play_url = TextUtils.join("$$$", playList); + vod.put("vod_play_from", vod_play_from); + vod.put("vod_play_url", vod_play_url); + + JSONObject result = new JSONObject(); + JSONArray list = new JSONArray(); + list.put(vod); + result.put("list", list); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + @Override + public String playerContent(String flag, String id, List vipFlags) { + try { + fetchRule(); + String webUrl = id.startsWith("x:") ? id.substring(2) : getRuleVal("url") + id; + JSONObject result = new JSONObject(); + result.put("parse", 1); + result.put("playUrl", ""); + result.put("url", webUrl); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + @Override + public String searchContent(String key, boolean quick) { + try { + fetchRule(); + boolean ssmoshiJson = getRuleVal("ssmoshi").equals("0"); + String webUrlTmp = getRuleVal("url") + getRuleVal("sousuoqian") + key + getRuleVal("sousuohou"); + String webUrl = webUrlTmp.split(";")[0]; + String webContent = webUrlTmp.contains(";post") ? fetchPost(webUrl) : fetch(webUrl); + JSONObject result = new JSONObject(); + JSONArray videos = new JSONArray(); + if (ssmoshiJson) { + JSONObject data = new JSONObject(webContent); + JSONArray vodArray = data.getJSONArray("list"); + for (int j = 0; j < vodArray.length(); j++) { + JSONObject vod = vodArray.getJSONObject(j); + String name = vod.optString(getRuleVal("jsname")).trim(); + String id = vod.optString(getRuleVal("jsid")).trim(); + String pic = vod.optString(getRuleVal("jspic")).trim(); + pic = Misc.fixUrl(webUrl, pic); + String link = getRuleVal("sousuohouzhui") + id; + link = getRuleVal("ssljqianzhui").isEmpty() ? (link + getRuleVal("ssljhouzhui")) : ("x:" + getRuleVal("ssljqianzhui")) + link + getRuleVal("ssljhouzhui"); + JSONObject v = new JSONObject(); + v.put("vod_id", name + "$$$" + pic + "$$$" + link); + v.put("vod_name", name); + v.put("vod_pic", pic); + v.put("vod_remarks", ""); + videos.put(v); + } + } else { + String parseContent = webContent; + boolean shifouercijiequ = getRuleVal("sousuoshifouercijiequ").equals("1"); + if (shifouercijiequ) { + String jiequqian = getRuleVal("ssjiequqian"); + String jiequhou = getRuleVal("ssjiequhou"); + parseContent = subContent(webContent, jiequqian, jiequhou).get(0); + } + String jiequshuzuqian = getRuleVal("ssjiequshuzuqian"); + String jiequshuzuhou = getRuleVal("ssjiequshuzuhou"); + ArrayList jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou); + for (int i = 0; i < jiequContents.size(); i++) { + try { + String jiequContent = jiequContents.get(i); + String title = subContent(jiequContent, getRuleVal("ssbiaotiqian"), getRuleVal("ssbiaotihou")).get(0); + String pic = subContent(jiequContent, getRuleVal("sstupianqian"), getRuleVal("sstupianhou")).get(0); + pic = Misc.fixUrl(webUrl, pic); + String link = subContent(jiequContent, getRuleVal("sslianjieqian"), getRuleVal("sslianjiehou")).get(0); + link = getRuleVal("ssljqianzhui").isEmpty() ? (link + getRuleVal("ssljhouzhui")) : ("x:" + getRuleVal("ssljqianzhui")) + link + getRuleVal("ssljhouzhui"); + String remark = ""; + if (!getRuleVal("ssfubiaotiqian").isEmpty() && !getRuleVal("ssfubiaotihou").isEmpty()) { + try { + remark = subContent(jiequContent, getRuleVal("ssfubiaotiqian"), getRuleVal("ssfubiaotihou")).get(0).replaceAll("\\s+", "").replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", ""); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + JSONObject v = new JSONObject(); + v.put("vod_id", title + "$$$" + pic + "$$$" + link); + v.put("vod_name", title); + v.put("vod_pic", pic); + v.put("vod_remarks", remark); + videos.put(v); + } catch (Throwable th) { + th.printStackTrace(); + break; + } + } + } + result.put("list", videos); + return result.toString(); + } catch ( + Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected String ext = null; + protected JSONObject rule = null; + + protected void fetchRule() { + if (rule == null) { + if (ext != null) { + try { + if (ext.startsWith("http")) { + String json = OkHttpUtil.string(ext, null); + rule = new JSONObject(json); + } else { + rule = new JSONObject(ext); + } + } catch (JSONException e) { + } + } + } + } + + protected String fetch(String webUrl) { + SpiderDebug.log(webUrl); + return OkHttpUtil.string(webUrl, getHeaders(webUrl)).replaceAll("\r|\n", ""); + } + + protected String fetchPost(String webUrl) { + SpiderDebug.log(webUrl); + OKCallBack.OKCallBackString callBack = new OKCallBack.OKCallBackString() { + @Override + protected void onFailure(Call call, Exception e) { + + } + + @Override + protected void onResponse(String response) { + } + }; + OkHttpUtil.post(OkHttpUtil.defaultClient(), webUrl, callBack); + return callBack.getResult().replaceAll("\r|\n", ""); + } + + private String getRuleVal(String key, String defaultVal) { + String v = rule.optString(key); + if (v.isEmpty() || v.equals("空")) + return defaultVal; + return v; + } + + private String getRuleVal(String key) { + return getRuleVal(key, ""); + } + + private ArrayList subContent(String content, String startFlag, String endFlag) { + ArrayList result = new ArrayList<>(); + if (startFlag.isEmpty() && endFlag.isEmpty()) { + result.add(content); + return result; + } + try { + Pattern pattern = Pattern.compile(escapeExprSpecialWord(startFlag) + "(.*?)" + escapeExprSpecialWord(endFlag)); + Matcher matcher = pattern.matcher(content); + while (matcher.find()) { + result.add(matcher.group(1)); + } + } catch (Throwable th) { + th.printStackTrace(); + } + return result; + } + + String escapeExprSpecialWord(String regexStr) { + if (!regexStr.isEmpty()) { + String[] fbsArr = {"\\", "$", "(", ")", "*", "+", ".", "[", "]", "?", "^", "{", "}", "|"}; + for (String key : fbsArr) { + if (regexStr.contains(key)) { + regexStr = regexStr.replace(key, "\\" + key); + } + } + } + return regexStr; + } + + //修复软件不支持的格式无法嗅探的问题 + @Override + public boolean manualVideoCheck() { + return true; + } + + private String[] videoFormatList = new String[]{".m3u8", ".mp4", ".mpeg", ".flv", ".m4a",".mp3",".wma",".wmv"}; + + @Override + public boolean isVideoFormat(String url) { + url = url.toLowerCase(); + if (url.contains("=http") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) { + return false; + } + for (String format : videoFormatList) { + if (url.contains(format)) { + return true; + } + } + return false; + } +} diff --git a/app/src/main/java/com/github/catvod/spider/XYQBiu.java b/app/src/main/java/com/github/catvod/spider/XYQBiu.java new file mode 100644 index 00000000..2a8fdf32 --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/XYQBiu.java @@ -0,0 +1,1019 @@ +package com.github.catvod.spider; + +import android.content.Context; +import android.text.TextUtils; +import android.util.Base64; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.utils.Misc; +import com.github.catvod.utils.okhttp.OKCallBack; +import com.github.catvod.utils.okhttp.OkHttpUtil; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.select.Elements; + +import java.net.InetAddress; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import okhttp3.Call; +import okhttp3.Headers; +import okhttp3.Response; + +public class XYQBiu extends Spider { + +// private String btcookie = ""; + + @Override + public void init(Context context) { + super.init(context); + } + + public void init(Context context, String extend) { + super.init(context, extend); + this.ext = extend; + } + + @Override + public String homeContent(boolean filter) { + try { + fetchRule(); + JSONObject result = new JSONObject(); + JSONArray classes = new JSONArray(); + + String[] fenleis = getRuleVal("class_name", "").split("&"); + String[] fenleisval = getRuleVal("class_value", "").split("&"); + for (int i = 0; i < fenleis.length; i++) { + JSONObject fenjson = new JSONObject(); + fenjson.put("type_id", fenleisval[i].replaceAll("＆＆", "&")); + fenjson.put("type_name", fenleis[i]); + classes.put(fenjson); + } + result.put("class", classes); + String getclan = getRuleVal("filterdata", ""); + InetAddress addr = InetAddress.getLocalHost(); + String locadd = "http://" + addr.getHostAddress() + ":9978/file/"; + String filepath = null; + JSONObject filterdata = null; + if (getclan.startsWith("clan://") || getclan.startsWith("http")) { + if (getclan.startsWith("clan://")) { + if (getclan.startsWith("clan://localhost/")) { + filepath = getclan.replace("clan://localhost/", locadd); + } else { + filepath = getclan.replace("clan://", locadd); + } + } else { + filepath = getclan; + } + try { + String filtejson = OkHttpUtil.string(filepath, null); + filterdata = new JSONObject(filtejson); + } catch ( + Exception e) { + SpiderDebug.log(e); + } + } else { + filterdata = rule.optJSONObject("filterdata"); + } + if (filter && filterdata != null) { + result.put("filters", filterdata); + } + return result.toString(); + } catch ( + Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected HashMap getHeaders(String url) { + HashMap headers = new HashMap<>(); + String ua = getRuleVal("UserAgent", "okhttp/3.12.11").trim(); + if (ua.isEmpty()) { + ua = "okhttp/3.12.11"; + } else if (ua.equals("PC_UA")) { + ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36"; + } else if (ua.equals("MOBILE_UA")) { + ua = "Mozilla/5.0 (Linux; Android 11; Mi 10 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.152 Mobile Safari/537.36"; + } + headers.put("User-Agent", ua); + + if (!getRuleVal("Referer").isEmpty()) { + String webref = getRuleVal("Referer").trim(); + if (webref.startsWith("http")) { + headers.put("Referer", webref); + } + } + return headers; + } + + @Override + public String homeVideoContent() { + try { + fetchRule(); + if (getRuleVal("homeContent").equals("1")) { + JSONArray videos = new JSONArray(); + String[] fenleis = getRuleVal("class_value", "").split("&"); + for (String fenlei : fenleis) { + JSONObject data = category(fenlei.replaceAll("＆＆", "&"), "1", false, new HashMap<>()); + if (data != null) { + JSONArray vids = data.optJSONArray("list"); + if (vids != null) { + for (int i = 0; i < vids.length() && i < 5; i++) { + videos.put(vids.getJSONObject(i)); + } + } + } + if (videos.length() >= 20) + break; + } + JSONObject result = new JSONObject(); + result.put("list", videos); + return result.toString(); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + private JSONObject category(String tid, String pg, boolean filter, HashMap extend) { + try { + fetchRule(); + String fistpg = String.valueOf(Integer.parseInt(getRuleVal("firstpage", "1"))); + ////页码 + if (fistpg.equals("0")) { + pg = String.valueOf(Integer.parseInt(pg) - 1 * 1); + } else { + pg = String.valueOf(Integer.parseInt(pg) - 1 + Integer.parseInt(getRuleVal("firstpage", "1"))); + } + //web链接 + String webUrl = getRuleVal("class_url"); + if (webUrl.contains("firstPage=")) { + if (pg.equals("1")) { + webUrl = webUrl.split("\\[firstPage=")[1].split("\\]")[0]; + } else { + webUrl = webUrl.split("\\[firstPage=")[0]; + } + } + //筛选开始 + if (filter && extend != null && extend.size() > 0) { + for (Iterator it = extend.keySet().iterator(); it.hasNext(); ) { + String key = it.next(); + String value = extend.get(key); + if (value.length() > 0) { + webUrl = webUrl.replace("{" + key + "}", URLEncoder.encode(value)); + } + } + } + webUrl = webUrl.replaceAll("\\{cateId\\}", tid).replaceAll("\\{catePg\\}", pg); + Matcher m = Pattern.compile("\\{(.*?)\\}").matcher(webUrl); + while (m.find()) { + String n = m.group(0).replace("{", "").replace("}", ""); + webUrl = webUrl.replace(m.group(0), "").replace("/" + n + "/", ""); + } + //筛选结束 + String html = null; +// String btwatUrl = ""; + //取网页 + if (webUrl.contains(";post")) { + OKCallBack.OKCallBackString callBack = new OKCallBack.OKCallBackString() { + + public void onResponse(String response) { + } + + @Override + protected void onFailure(Call call, Exception exc) { + } + }; + String posturl = webUrl.split("\\?")[0].replaceAll("？？", "?").trim(); + String postbody = webUrl.split("\\?")[1].split(";")[0].replaceAll("？？", "?").trim(); + if (!postbody.isEmpty() && postbody != null) { + if (postbody.startsWith("{") && postbody.endsWith("}")) { + JSONObject jsbody = new JSONObject(postbody); + OkHttpUtil.postJson(OkHttpUtil.defaultClient(), posturl, jsbody.toString(), getHeaders(posturl), callBack); + } else { + LinkedHashMap params = new LinkedHashMap(); + String[] userbody = postbody.split("&"); + for (String userbd : userbody) { + int loca = userbd.indexOf("="); + params.put(userbd.substring(0, loca), userbd.substring(loca + 1)); + } + OkHttpUtil.post(OkHttpUtil.defaultClient(), posturl, params, getHeaders(posturl), callBack); + } + } else { + OkHttpUtil.post(OkHttpUtil.defaultClient(), posturl, null, getHeaders(posturl), callBack); + } + html = convertUnicodeToCh(callBack.getResult().replaceAll("\r|\n", "")); +// btwatUrl = posturl; + } else { + html = convertUnicodeToCh(fetch(webUrl)); +// btwatUrl = webUrl; + } + +// html = jumpbtwaf(btwatUrl,html);//5秒盾 + String parseContent = html; + String mark = ""; + String pic = ""; + boolean fenleiJson = getRuleVal("cat_mode").equals("0"); + boolean picneetproxy = getRuleVal("PicNeedProxy").equals("1"); + + JSONArray videos = new JSONArray(); + JSONObject result = new JSONObject(); + if (fenleiJson) { + JSONObject data = new JSONObject(parseContent); + JSONArray vodArray = null; + String[] keylen = getRuleVal("catjsonlist", "data").split("\\."); + if (keylen.length == 1) { + vodArray = data.getJSONArray(keylen[0]); + } else if (keylen.length == 2) { + vodArray = data.getJSONObject(keylen[0]).getJSONArray(keylen[1]); + } else if (keylen.length == 3) { + vodArray = data.getJSONObject(keylen[0]).getJSONObject(keylen[1]).getJSONArray(keylen[2]); + } + //JSONArray vodArray = data.getJSONArray(getRuleVal("catjsonlist","list")); + for (int j = 0; j < vodArray.length(); j++) { + try { + JSONObject vod = vodArray.getJSONObject(j); + String name = vod.optString(getRuleVal("catjsonname")).trim(); + String id = vod.optString(getRuleVal("catjsonid")).trim(); + id = getRuleVal("cat_prefix", "") + id + getRuleVal("cat_suffix", ""); + if (!getRuleVal("catjsonpic").isEmpty()) { + try { + pic = vod.optString(getRuleVal("catjsonpic")).trim(); + pic = Misc.fixUrl(webUrl, pic); + if (picneetproxy) { + pic = fixCover(pic, webUrl); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + } + try { + mark = vod.optString(getRuleVal("catjsonstitle")).trim(); + } catch (Exception e) { + SpiderDebug.log(e); + } + JSONObject v = new JSONObject(); + v.put("vod_id", name + "$$$" + pic + "$$$" + id); + v.put("vod_name", name); + v.put("vod_pic", pic); + v.put("vod_remarks", mark); + videos.put(v); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + } else { + boolean shifouercijiequ = getRuleVal("cat_YN_twice").equals("1"); + if (shifouercijiequ) { + String jiequqian = getRuleVal("cat_twice_pre"); + String jiequhou = getRuleVal("cat_twice_suf"); + parseContent = subContent(html, jiequqian, jiequhou).get(0); + } + String jiequshuzuqian = getRuleVal("cat_arr_pre"); + String jiequshuzuhou = getRuleVal("cat_arr_suf"); + + ArrayList jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou); + for (int i = 0; i < jiequContents.size(); i++) { + try { + String jiequContent = jiequContents.get(i); + String title = subContent(jiequContent, getRuleVal("cat_title").split("&&")[0], getRuleVal("cat_title").split("&&")[1]).get(0).replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", ""); + if (!getRuleVal("cat_pic").isEmpty()) { + try { + pic = subContent(jiequContent, getRuleVal("cat_pic").split("&&")[0], getRuleVal("cat_pic").split("&&")[1]).get(0); + pic = Misc.fixUrl(webUrl, pic); + if (picneetproxy) { + pic = fixCover(pic, webUrl); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + } + String link = subContent(jiequContent, getRuleVal("cat_url").split("&&")[0], getRuleVal("cat_url").split("&&")[1]).get(0); + link = getRuleVal("cat_prefix", "") + link + getRuleVal("cat_suffix", ""); + if (!getRuleVal("cat_subtitle").isEmpty()) { + try { + mark = subContent(jiequContent, getRuleVal("cat_subtitle").split("&&")[0], getRuleVal("cat_subtitle").split("&&")[1]).get(0).replaceAll("\\s+", "").replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", ""); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + JSONObject v = new JSONObject(); + v.put("vod_id", title + "$$$" + pic + "$$$" + link); + v.put("vod_name", title); + v.put("vod_pic", pic); + v.put("vod_remarks", mark); + videos.put(v); + } catch (Throwable th) { + th.printStackTrace(); + break; + } + } + } + + result.put("page", pg); + result.put("pagecount", Integer.MAX_VALUE); + result.put("limit", 90); + result.put("total", Integer.MAX_VALUE); + result.put("list", videos); + return result; + } catch (Exception e) { + SpiderDebug.log(e); + } + return null; + } + + @Override + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + JSONObject obj = category(tid, pg, filter, extend); + return obj != null ? obj.toString() : ""; + } + + @Override + public String detailContent(List ids) { + try { + fetchRule(); + String[] idInfo = ids.get(0).split("\\$\\$\\$"); + String webUrl = idInfo[2]; + + String cover = idInfo[1], vodtitle = idInfo[0], area = ""; + String desc = ""; + String director = ""; + String actor = ""; + String remark = ""; + String year = ""; + String category = ""; + + ArrayList playList = new ArrayList<>(); + ArrayList playFrom = new ArrayList<>(); + boolean isMagnet = false; + boolean zhijiebofang = getRuleVal("force_play").equals("1"); + boolean picneetproxy = getRuleVal("PicNeedProxy").equals("1"); + try { + if (picneetproxy) { + cover = fixCover(cover, webUrl); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + if (zhijiebofang) { + //直接播放 + playFrom.add(idInfo[0]); + playList.add(idInfo[0] + "$" + idInfo[2]); + if (idInfo[2].startsWith("magnet")) { + isMagnet = true; + } + } else { + //非直接播放 + String html = fetch(webUrl).trim(); +// html = jumpbtwaf(webUrl,html);//5秒盾 + html = convertUnicodeToCh(html); + String parseContent = html; + boolean bfshifouercijiequ = getRuleVal("list_YN_twice").equals("1"); + if (bfshifouercijiequ) { + String jiequqian = getRuleVal("list_twice_pre"); + String jiequhou = getRuleVal("list_twice_suf"); + parseContent = subContent(html, jiequqian, jiequhou).get(0); + } + + String jiequshuzuqian = getRuleVal("list_arr_pre"); + String jiequshuzuhou = getRuleVal("list_arr_suf"); + boolean bfyshifouercijiequ = getRuleVal("epi_YN_twice").equals("1"); + ArrayList jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou); + for (int i = 0; i < jiequContents.size(); i++) { + try { + String jiequContent = jiequContents.get(i); + String parseJqContent = bfyshifouercijiequ ? subContent(jiequContent, getRuleVal("epi_twice_pre"), getRuleVal("epi_twice_suf")).get(0) : jiequContent; + ArrayList lastParseContents = subContent(parseJqContent, getRuleVal("epi_arr_pre"), getRuleVal("epi_arr_suf")); + List vodItems = new ArrayList<>(); + for (int j = 0; j < lastParseContents.size(); j++) { + String title = subContent(lastParseContents.get(j), getRuleVal("epi_title").split("&&")[0], getRuleVal("epi_title").split("&&")[1]).get(0); + String link = subContent(lastParseContents.get(j), getRuleVal("epi_url").split("&&")[0], getRuleVal("epi_url").split("&&")[1]).get(0); + link = getRuleVal("epiurl_prefix", "") + link + getRuleVal("epiurl_suffix", ""); + vodItems.add(title + "$" + link); + if (link.startsWith("magnet")) { + isMagnet = true; + break; + } + } + playList.add(TextUtils.join("#", vodItems)); + if (isMagnet) { + break; + } + } catch (Throwable th) { + th.printStackTrace(); + break; + } + } + + //线路代码 + String xlparseContent = html; + //没有线路规则代码时 + if (!getRuleVal("tab_title").isEmpty() && !getRuleVal("tab_arr_pre").isEmpty()) { + //代码取线路名 + boolean xlshifouercijiequ = getRuleVal("tab_YN_twice").equals("1"); + if (xlshifouercijiequ) { + String xljiequqian = getRuleVal("tab_twice_pre"); + String xljiequhou = getRuleVal("tab_twice_suf"); + xlparseContent = subContent(html, xljiequqian, xljiequhou).get(0); + } + + String xljiequshuzuqian = getRuleVal("tab_arr_pre"); + String xljiequshuzuhou = getRuleVal("tab_arr_suf"); + ArrayList xljiequContents = subContent(xlparseContent, xljiequshuzuqian, xljiequshuzuhou); + for (int i = 0; i < playList.size(); i++) { + try { + String xltitle = subContent(xljiequContents.get(i), getRuleVal("tab_title").split("&&")[0], getRuleVal("tab_title").split("&&")[1]).get(0).replaceAll("\\s+", "").replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", ""); + playFrom.add(xltitle); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + } else { + for (int i = 0; i < playList.size(); i++) { + playFrom.add("播放列表" + (i + 1)); + } + } + + if (!getRuleVal("proj_actor").isEmpty()) { + try { + actor = subContent(html, getRuleVal("proj_actor").split("&&")[0], getRuleVal("proj_actor").split("&&")[1]).get(0).replaceAll("\\s+", ""); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + if (!getRuleVal("proj_plot").isEmpty()) { + try { + desc = subContent(html, getRuleVal("proj_plot").split("&&")[0], getRuleVal("proj_plot").split("&&")[1]).get(0); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + } + //非直接播放结束 + JSONObject vod = new JSONObject(); + vod.put("vod_id", ids.get(0)); + vod.put("vod_name", vodtitle); + vod.put("vod_pic", cover); + vod.put("type_name", category); + vod.put("vod_year", year); + vod.put("vod_area", area); + vod.put("vod_remarks", remark); + vod.put("vod_actor", actor); + vod.put("vod_director", director); + vod.put("vod_content", desc); + + String vod_play_from = TextUtils.join("$$$", playFrom); + String vod_play_url = TextUtils.join("$$$", playList); + vod.put("vod_play_from", vod_play_from); + vod.put("vod_play_url", vod_play_url); + + JSONObject result = new JSONObject(); + JSONArray list = new JSONArray(); + list.put(vod); + result.put("list", list); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + + @Override + public String playerContent(String flag, String id, List vipFlags) { + try { + fetchRule(); + String webUrl = id; + //WebView嗅探请求头 + JSONObject webheaders = new JSONObject(); + String webua = getRuleVal("UserAgent", "okhttp/3.12.11").trim(); + if (webua.isEmpty()) { + webua = "okhttp/3.12.11"; + } else if (webua.equals("PC_UA")) { + webua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36"; + } else if (webua.equals("MOBILE_UA")) { + webua = "Mozilla/5.0 (Linux; Android 11; Mi 10 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.152 Mobile Safari/537.36"; + } + webheaders.put("User-Agent", webua); + + if (!getRuleVal("Referer").isEmpty()) { + String webref = getRuleVal("Referer").trim(); + if (webref.startsWith("http")) { + webheaders.put("Referer", webref); + } else if (webref.equals("WebView")) { + webheaders.put("Referer", webUrl); + } + } + //嗅探请求头结束 + + JSONObject result = new JSONObject(); + boolean enforceplay = (getRuleVal("force_play").equals("1") || getRuleVal("force_play").equals("2")); + //强制视频链接 + if (enforceplay) { + webUrl = getRuleVal("play_prefix", "") + webUrl + getRuleVal("play_suffix", ""); + //请求头代码 + if (!getRuleVal("play_header").isEmpty()) { + JSONObject uaresult = rule.optJSONObject("play_header"); + if (uaresult != null) { + result.put("header", uaresult.toString()); + } else { + String[] usera = getRuleVal("play_header").split("#"); + JSONObject strua = new JSONObject(); + for (String user : usera) { + String[] head = user.split("\\$"); + strua.put(head[0], " " + head[1]); + } + result.put("header", strua.toString()); + } + } else { + result.put("header", webheaders.toString()); + } + //视频格式识别 + if (webUrl.contains("#isVideo=true#") || Misc.isVideoFormat(webUrl)) { + if (webUrl.contains("#isVideo=true#")) { + webUrl = webUrl.replaceAll("#isVideo=true#", ""); + } + result.put("parse", 0); + result.put("playUrl", ""); + } else if (Misc.isVip(webUrl)) { + result.put("parse", 1); + result.put("jx", "1"); + result.put("url", webUrl); + return result.toString(); + } else { + //不是视频就转嗅探 + result.put("parse", 1); + result.put("playUrl", ""); + } + result.put("url", webUrl); + return result.toString(); + } + //直接播放代码结束 + + //普通链接开始 + //分析mac链接解析 + boolean enMacPlayer = getRuleVal("Anal_MacPlayer").equals("1"); + String videoUrl = null; + String fromflag = null; + + if (enMacPlayer && enforceplay == false) { + try { + Document doc = Jsoup.parse(fetch(webUrl)); + Elements allScript = doc.select("script"); + for (int i = 0; i < allScript.size(); i++) { + String scContent = allScript.get(i).html().trim(); + if (scContent.startsWith("var player_")) { + int start = scContent.indexOf('{'); + int end = scContent.lastIndexOf('}') + 1; + String json = scContent.substring(start, end); + JSONObject player = new JSONObject(json); + String videoUrlTmp = player.getString("url"); + + fromflag = player.getString("from"); + if (player.has("encrypt")) { + int encrypt = player.getInt("encrypt"); + if (encrypt == 1) { + videoUrlTmp = URLDecoder.decode(videoUrlTmp); + } else if (encrypt == 2) { + videoUrlTmp = new String(Base64.decode(videoUrlTmp, Base64.DEFAULT)); + videoUrlTmp = URLDecoder.decode(videoUrlTmp); + } + } + videoUrl = videoUrlTmp; + break; + } + } + } catch (Exception e) { + SpiderDebug.log(e); + } + } + + + //视频不为null时进行处理解析 + if (videoUrl != null) { + if (Misc.isVip(videoUrl)) { // 使用jx:1 + try { + result.put("parse", 1); + result.put("jx", "1"); + result.put("url", videoUrl); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + } else if (fromflag.matches("duoduozy|ddzy")) { + + result.put("header", webheaders.toString()); + result.put("parse", 1); + result.put("playUrl", ""); + result.put("url", webUrl); + return result.toString(); + } else if (Misc.isVideoFormat(videoUrl)) { + try { + result.put("parse", 0); + result.put("playUrl", ""); + result.put("url", videoUrl); + result.put("header", ""); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + } + //视频不为null代码结束 + + result.put("header", webheaders.toString()); + result.put("parse", 1); + result.put("playUrl", ""); + result.put("url", webUrl); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + @Override + public boolean manualVideoCheck() { + fetchRule(); + if (getRuleVal("ManualSniffer").equals("1")) { + return true; + } else { + return false; + } + } + + //private String[] videoFormatList = getRuleVal("videoformat",".m3u8#.mp4#.flv").split("#"); + + @Override + public boolean isVideoFormat(String url) { + fetchRule(); + String[] videoFormatList = getRuleVal("VideoFormat", ".m3u8#.mp4#.flv#.mp3#.m4a").split("#"); + String[] videoFilterList = getRuleVal("VideoFilter", "=http#.html").split("#"); + url = url.toLowerCase(); + + if (url.contains("=http") || url.contains("=https") || url.contains("=https%3a%2f") || url.contains("=http%3a%2f")) { + return false; + } + for (String format : videoFormatList) { + if (url.contains(format)) { + for (String videoFilter : videoFilterList) { + if (url.contains(videoFilter)) { + return false; + } + } + return true; + } + } + return false; + } + + @Override + public String searchContent(String key, boolean quick) { + try { + fetchRule(); + String webUrlTmp = getRuleVal("search_url").replaceAll("\\{wd\\}", key); + String webUrl = webUrlTmp.split(";")[0]; + String webContent = null; + + if (webUrlTmp.contains(";post")) { + OKCallBack.OKCallBackString callBack = new OKCallBack.OKCallBackString() { + + public void onResponse(String response) { + } + + @Override + protected void onFailure(Call call, Exception exc) { + } + }; + + String postbody = getRuleVal("sea_PtBody").replaceAll("\\{wd\\}", key).trim(); + if (!postbody.isEmpty() && postbody != null) { + if (postbody.startsWith("{") && postbody.endsWith("}")) { + JSONObject jsbody = new JSONObject(postbody); + OkHttpUtil.postJson(OkHttpUtil.defaultClient(), webUrl, jsbody.toString(), getHeaders(webUrl), callBack); + } else { + LinkedHashMap params = new LinkedHashMap(); + //params.put(postbody, key); + String[] userbody = postbody.split("&"); + for (String userbd : userbody) { + int loca = userbd.indexOf("="); + //int end = userbd.lastIndexOf('=') + 1; + //String[] bdhead = userbd.split("="); + params.put(userbd.substring(0, loca), userbd.substring(loca + 1)); + } + OkHttpUtil.post(OkHttpUtil.defaultClient(), webUrl, params, getHeaders(webUrl), callBack); + } + } else { + OkHttpUtil.post(OkHttpUtil.defaultClient(), webUrl, null, getHeaders(webUrl), callBack); + } + webContent = convertUnicodeToCh(callBack.getResult().replaceAll("\r|\n", "")); + } else { + webContent = convertUnicodeToCh(fetch(webUrl)); + } + + String mark = ""; + String pic = ""; + boolean ssmoshiJson = getRuleVal("search_mode").equals("0"); + boolean picneetproxy = getRuleVal("PicNeedProxy").equals("1"); + + JSONObject result = new JSONObject(); + JSONArray videos = new JSONArray(); + if (ssmoshiJson) { + JSONObject data = new JSONObject(webContent); + JSONArray vodArray = null; + String[] keylen = getRuleVal("jsonlist", "list").split("\\."); + if (keylen.length == 1) { + vodArray = data.getJSONArray(keylen[0]); + } else if (keylen.length == 2) { + vodArray = data.getJSONObject(keylen[0]).getJSONArray(keylen[1]); + } else if (keylen.length == 3) { + vodArray = data.getJSONObject(keylen[0]).getJSONObject(keylen[1]).getJSONArray(keylen[2]); + } + //JSONArray vodArray = data.getJSONArray(getRuleVal("jsonlist","list")); + for (int j = 0; j < vodArray.length(); j++) { + try { + JSONObject vod = vodArray.getJSONObject(j); + String name = vod.optString(getRuleVal("jsonname")).trim(); + String id = vod.optString(getRuleVal("jsonid")).trim(); + id = getRuleVal("cat_prefix", "") + id + getRuleVal("cat_suffix", ""); + if (!getRuleVal("jsonpic").isEmpty()) { + try { + pic = vod.optString(getRuleVal("jsonpic")).trim(); + pic = Misc.fixUrl(webUrl, pic); + if (picneetproxy) { + pic = fixCover(pic, webUrl); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + } + try { + mark = vod.optString(getRuleVal("jsonstitle")).trim(); + } catch (Exception e) { + SpiderDebug.log(e); + } + JSONObject v = new JSONObject(); + v.put("vod_id", name + "$$$" + pic + "$$$" + id); + v.put("vod_name", name); + v.put("vod_pic", pic); + v.put("vod_remarks", mark); + videos.put(v); + } catch (Exception e) { + SpiderDebug.log(e); + } + } + } else { + String parseContent = webContent; + boolean shifouercijiequ = getRuleVal("sea_YN_twice").equals("1"); + if (shifouercijiequ) { + String jiequqian = getRuleVal("sea_twice_pre"); + String jiequhou = getRuleVal("sea_twice_suf"); + parseContent = subContent(webContent, jiequqian, jiequhou).get(0); + } + String jiequshuzuqian = getRuleVal("sea_arr_pre"); + String jiequshuzuhou = getRuleVal("sea_arr_suf"); + ArrayList jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou); + for (int i = 0; i < jiequContents.size(); i++) { + try { + String jiequContent = jiequContents.get(i); + String title = subContent(jiequContent, getRuleVal("sea_title").split("&&")[0], getRuleVal("sea_title").split("&&")[1]).get(0).replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", ""); + if (!getRuleVal("sea_pic").isEmpty()) { + try { + pic = subContent(jiequContent, getRuleVal("sea_pic").split("&&")[0], getRuleVal("sea_pic").split("&&")[1]).get(0); + pic = Misc.fixUrl(webUrl, pic); + if (picneetproxy) { + pic = fixCover(pic, webUrl); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + } + String link = subContent(jiequContent, getRuleVal("sea_url").split("&&")[0], getRuleVal("sea_url").split("&&")[1]).get(0); + link = getRuleVal("search_prefix") + link + getRuleVal("search_suffix"); + try { + mark = subContent(jiequContent, getRuleVal("sea_subtitle").split("&&")[0], getRuleVal("sea_subtitle").split("&&")[1]).get(0).replaceAll("\\s+", "").replaceAll("\\&[a-zA-Z]{1,10};", "").replaceAll("<[^>]*>", "").replaceAll("[(/>)<]", ""); + } catch (Exception e) { + SpiderDebug.log(e); + } + JSONObject v = new JSONObject(); + v.put("vod_id", title + "$$$" + pic + "$$$" + link); + v.put("vod_name", title); + v.put("vod_pic", pic); + v.put("vod_remarks", mark); + videos.put(v); + } catch (Throwable th) { + th.printStackTrace(); + break; + } + } + } + result.put("list", videos); + return result.toString(); + } catch ( + Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + protected String ext = null; + protected JSONObject rule = null; + + protected void fetchRule() { + if (rule == null) { + if (ext != null) { + try { + if (ext.startsWith("http")) { + String json = OkHttpUtil.string(ext, null); + rule = new JSONObject(json); + } else { + rule = new JSONObject(ext); + } + } catch (JSONException e) { + } + } + } + } + +// protected String fetch(String webUrl) { +// SpiderDebug.log(webUrl); +// return OkHttpUtil.string(webUrl, getHeaders(webUrl)).replaceAll("\r|\n", ""); + +// } + + + private static String convertUnicodeToCh(String str) { + Pattern pattern = Pattern.compile("(\\\\u(\\w{4}))"); + Matcher matcher = pattern.matcher(str); + + // 迭代，将str中的所有unicode转换为正常字符 + while (matcher.find()) { + String unicodeFull = matcher.group(1); // 匹配出的每个字的unicode，比如\u67e5 + String unicodeNum = matcher.group(2); // 匹配出每个字的数字，比如\u67e5，会匹配出67e5 + + // 将匹配出的数字按照16进制转换为10进制，转换为char类型，就是对应的正常字符了 + char singleChar = (char) Integer.parseInt(unicodeNum, 16); + + // 替换原始字符串中的unicode码 + str = str.replace(unicodeFull, singleChar + ""); + } + return str; + } + + private String getRuleVal(String key, String defaultVal) { + String v = rule.optString(key); + if (v.isEmpty() || v.equals("空") || v.equals("&&")) + return defaultVal; + return v; + } + + private String getRuleVal(String key) { + return getRuleVal(key, ""); + } + + private ArrayList subContent(String content, String startFlag, String endFlag) { + ArrayList result = new ArrayList<>(); + if (startFlag.isEmpty() && endFlag.isEmpty()) { + result.add(content); + return result; + } + try { + Pattern pattern = Pattern.compile(escapeExprSpecialWord(startFlag) + "(.*?)" + escapeExprSpecialWord(endFlag)); + Matcher matcher = pattern.matcher(content); + while (matcher.find()) { + result.add(matcher.group(1).trim()); + } + } catch (Throwable th) { + th.printStackTrace(); + } + if (result.isEmpty()) result.add(""); + return result; + } + + public static String escapeExprSpecialWord(String keyword) { + if (!keyword.isEmpty()) { + String[] fbsArr = {"\\", "$", "(", ")", "*", "+", ".", "[", "]", "?", "^", "{", "}", "|"}; + for (String key : fbsArr) { + if (keyword.contains(key)) { + keyword = keyword.replace(key, "\\" + key); + } + } + } + return keyword; + } + + protected String fixCover(String cover, String site) { + try { + return "proxy://do=XYQBiu&site=" + site + "&pic=" + cover; + } catch (Exception e) { + SpiderDebug.log(e); + } + return cover; + } + + private static HashMap XYQPicHeader = null; + + public static Object[] loadPic(Map prmap) { + try { + //pic = new String(Base64.decode(pic, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP), "UTF-8"); + String site = prmap.get("site"); + String pic = prmap.get("pic"); + + if (XYQPicHeader == null) { + XYQPicHeader = new HashMap<>(); + XYQPicHeader.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36"); + XYQPicHeader.put("referer", site); + } + OKCallBack.OKCallBackDefault callBack = new OKCallBack.OKCallBackDefault() { + @Override + protected void onFailure(Call call, Exception e) { + + } + + @Override + protected void onResponse(Response response) { + + } + }; + OkHttpUtil.get(OkHttpUtil.defaultClient(), pic, null, XYQPicHeader, callBack); + if (callBack.getResult().code() == 200) { + Headers headers = callBack.getResult().headers(); + String type = headers.get("Content-Type"); + if (type == null) { + type = "application/octet-stream"; + } + Object[] result = new Object[3]; + result[0] = 200; + result[1] = type; + System.out.println(pic); + System.out.println(type); + result[2] = callBack.getResult().body().byteStream(); + return result; + } + } catch (Throwable th) { + th.printStackTrace(); + } + return null; + } + + + protected String fetch(String webUrl) { + String html = OkHttpUtil.string(webUrl, getHeaders(webUrl)); + html = this.jumpbtwaf(webUrl, html); + return html.replaceAll("", "").replace("\r\n","").replace("\n",""); // 移除注释 + } + + protected String jumpbtwaf(String webUrl, String html) { + + try { + // 没有配置btwaf不执行下面的代码 + if (!rule.optBoolean("btwaf", false)) { + return html; + } + + for (int i = 0; i < 3; i++) { + if (html.contains("检测中") && html.contains("跳转中") && html.contains("btwaf")) { + String btwaf = subContent(html, "btwaf=", "\"").get(0); + String bturl = webUrl + "?btwaf=" + btwaf; + + Map> cookies = new HashMap<>(); + OkHttpUtil.string(bturl, getHeaders(webUrl), cookies); + for (Map.Entry> entry : cookies.entrySet()) { + if (entry.getKey().equals("set-cookie") || entry.getKey().equals("Set-Cookie")) { + String btcookie = TextUtils.join(";", entry.getValue()); + if (!rule.has("header")) { + rule.put("header", new JSONObject()); + } + rule.getJSONObject("header").put("cookie", btcookie); + break; + } + } + html = fetch(webUrl); + } + if (!html.contains("检测中") && !html.contains("btwaf")) { + return html; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return html; + } + + +} \ No newline at end of file diff --git a/app/src/main/java/com/github/catvod/spider/Yj1211.java b/app/src/main/java/com/github/catvod/spider/Yj1211.java new file mode 100644 index 00000000..5ef5e72a --- /dev/null +++ b/app/src/main/java/com/github/catvod/spider/Yj1211.java @@ -0,0 +1,398 @@ +package com.github.catvod.spider; + + +import android.content.Context; +import android.text.TextUtils; + +import com.github.catvod.crawler.Spider; +import com.github.catvod.crawler.SpiderDebug; +import com.github.catvod.net.OkHttp; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public class Yj1211 extends Spider { + + @Override + public void init(Context context) { + super.init(context); + } + + + @Override + public String homeContent(boolean filter) { + try { + + String srcurl = "http://live.yj1211.work/api/live/getRecommend?page=1&size=20"; + String srcOrignstr =""; + for(int i=0 ; i<3 ; i++){ + srcOrignstr = OkHttp.string(srcurl, getHeaders()); + if(srcOrignstr.length()>0){ + break; + }else{ + Thread.sleep(1000); + } + } + + JSONObject srcori = new JSONObject(srcOrignstr); + JSONArray srcoria = srcori.getJSONArray("data"); + + + JSONObject result = new JSONObject(); + JSONArray classes = new JSONArray(); + String catestr ="{\"推荐\":\"?\",\"斗鱼\":\"ByPlatform?platform=douyu&\",\"哔哩哔哩\":\"ByPlatform?platform=bilibili&\",\"虎牙\":\"ByPlatform?platform=huya&\",\"网易CC\":\"ByPlatform?platform=cc&\"}"; + JSONObject catedef = new JSONObject(catestr); + Iterator it = catedef.keys(); + while(it.hasNext()){ + JSONObject jsonObject = new JSONObject(); + String key =(String) it.next(); + jsonObject.put("type_name", key); + jsonObject.put("type_id", catedef.getString(key)); + classes.put(jsonObject); + } + result.put("class", classes); + + JSONObject filterConfig = new JSONObject(); + String geta ="http://live.yj1211.work/api/live/getAllAreas"; + String aaid = ""; + for(int i=0 ; i<3 ; i++){ + aaid = OkHttp.string(geta, getHeaders()); + if(aaid.length()>0){ + break; + }else{ + Thread.sleep(1000); + } + } + + JSONObject aido = new JSONObject(aaid); + JSONArray aidoa = aido.getJSONArray("data"); + JSONArray extendsAll = new JSONArray(); + for (int j=0 ; j< 13;j++){ + JSONObject newTypeExtend = new JSONObject(); + String typeName = aidoa.getJSONArray(j).getJSONObject(0).getString("typeName"); + newTypeExtend.put("key", "typeName" + j); + newTypeExtend.put("name", typeName); + JSONArray newTypeExtendKV = new JSONArray(); + int fg = Math.min(aidoa.getJSONArray(j).length(), 20); + JSONObject kv = new JSONObject(); + kv.put("n", "全部"); + kv.put("v", typeName + "=" + "all"); + newTypeExtendKV.put(kv); + for (int k=0 ; k< fg ; k++){ + kv = new JSONObject(); + String areaName = aidoa.getJSONArray(j).getJSONObject(k).getString("areaName"); + kv.put("n", areaName); + kv.put("v", typeName + "=" + areaName); + newTypeExtendKV.put(kv); + } + newTypeExtend.put("value", newTypeExtendKV); + extendsAll.put(newTypeExtend); + } + for (int i = 0 ; i < 5 ; i++){ + String typeId = classes.getJSONObject(i).getString("type_id"); + filterConfig.put(typeId, extendsAll); + } + + JSONArray videos = new JSONArray(); + int ch = Math.min(srcoria.length(), 10); + for (int i = 0; i < ch; i++) { + JSONObject srchome = new JSONObject(); + String platForm = srcoria.getJSONObject(i).getString("platForm"); + String rd = srcoria.getJSONObject(i).getString("roomId"); + String id = platForm + "&" + rd; + String name = srcoria.getJSONObject(i).getString("ownerName"); + String pic = srcoria.getJSONObject(i).getString("ownerHeadPic"); + String mark = srcoria.getJSONObject(i).getString("categoryName"); + srchome.put("vod_id", id); + srchome.put("vod_name", name); + srchome.put("vod_pic",pic); + srchome.put("vod_remarks", mark); + videos.put(srchome); + } + if (filter) { + result.put("filters", filterConfig); + } + result.put("list", videos); + return result.toString(); + } catch ( + Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + + @Override + public String categoryContent(String tid, String pg, boolean filter, HashMap extend) { + try { + String catt=""; + switch (tid) { + case "?": + catt = "all"; + break; + case "ByPlatform?platform=douyu&": + catt = "douyu"; + break; + case "ByPlatform?platform=bilibili&": + catt = "bilibili"; + break; + case "ByPlatform?platform=huya&": + catt = "huya"; + break; + case "ByPlatform?platform=cc&": + catt = "cc"; + break; + } + + + + + + + extend = extend == null ? new HashMap<>() : extend; + String srcurl=""; + String[] cate= new String[13]; + int pp = 0; + for(int i = 0; i < 13; i++){ + cate[i] = extend.containsKey("typeName"+i) ? extend.get("typeName"+i) : ("typeName"+i + "=" + "all"); + String[] info = cate[i].split("="); + String area = info[1]; + if( !area.contains("all") ){ + pp= pp + 1; + } + } + if(pp == 1){ + for(int i = 0; i < 13; i++){ + String[] info = cate[i].split("="); + String areaType = info[0]; + String area = info[1]; + if(!area.contains("all")) { + String urlft = "http://live.yj1211.work/api/live/getRecommendByAreaAll?areaType={areaType}&area={area}&page={pg}"; + srcurl = urlft.replace("{areaType}", URLEncoder.encode(areaType)).replace("{area}", URLEncoder.encode(area)).replace("{pg}", pg); + break; + } + } + }else if( pp == 0 || pp > 1 ){ + String urlft = "http://live.yj1211.work/api/live/getRecommend{tid}page={pg}&size=20"; + srcurl = urlft.replace("{tid}", tid).replace("{pg}", pg); + } + String srcOrignstr = ""; + for (int i = 0; i < 3; i++) { + srcOrignstr = OkHttp.string(srcurl, getHeaders()); + if (srcOrignstr.length() > 0 ) { + break; + } else { + Thread.sleep(1000); + } + } + JSONObject srcori = new JSONObject(srcOrignstr); + JSONArray srcoria = srcori.getJSONArray("data"); + JSONObject result = new JSONObject(); + JSONArray videos = new JSONArray(); + for (int i = 0; i < srcoria.length(); i++) { + JSONObject srchome = new JSONObject(); + String platForm = srcoria.getJSONObject(i).getString("platForm"); + if(pp == 1 && !catt.equals("all")){ + if(!platForm.equals(catt) ){ + continue; + } + } + + String rd = srcoria.getJSONObject(i).getString("roomId"); + String id = platForm + "&" + rd; + String name = srcoria.getJSONObject(i).getString("ownerName"); + String pic = srcoria.getJSONObject(i).getString("ownerHeadPic"); + String mark = srcoria.getJSONObject(i).getString("categoryName"); + srchome.put("vod_id", id); + srchome.put("vod_name", name); + srchome.put("vod_pic", pic); + srchome.put("vod_remarks", mark); + videos.put(srchome); + } + if(videos.length() == 0 ) { + JSONObject srchome = new JSONObject(); + srchome.put("vod_id", 111); + srchome.put("vod_name", "此页无符合"); + srchome.put("vod_pic",""); + srchome.put("vod_remarks", "nothing"); + videos.put(srchome); + } + if (pp ==1 ) { + result.put("pagecount", 50); + result.put("limit",1 ); + } else{ + result.put("pagecount", Integer.MAX_VALUE); + result.put("limit",90 ); + } + result.put("list", videos); + result.put("page", pg); + result.put("total", Integer.MAX_VALUE); + + + + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + private HashMap getHeaders() { + HashMap headers = new HashMap<>(); + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.134 YaBrowser/22.7.1.755 (beta) Yowser/2.5 Safari/537.36"); + return headers; + } + + @Override + public String detailContent(List ids) { + try { + String id = ids.get(0); + String[] info = id.split("&"); + String platform = info[0]; + String roomId = info[1]; + String srcurl = "http://live.yj1211.work/api/live/getRoomInfo?platform="+platform+"&roomId="+roomId; + String srcplayurl = "http://live.yj1211.work/api/live/getRealUrl?platform="+platform+"&roomId="+roomId; + String srcdtlstr = OkHttp.string(srcurl, getHeaders()); + String srcdtlplay = OkHttp.string(srcplayurl, getHeaders()); + JSONObject srcdtl = new JSONObject(srcdtlstr); + JSONObject srcdtlo = srcdtl.getJSONObject("data"); + JSONObject srcdtplay = new JSONObject(srcdtlplay); + JSONObject srcdtplayo = srcdtplay.getJSONObject("data"); + String title = srcdtlo.getString("roomName"); + String pic = srcdtlo.getString("roomPic"); + String director = srcdtlo.getString("ownerName") + " RoomID:"+ srcdtlo.getString("roomId"); + String content = srcdtlo.getString("categoryName"); + String actor = "观看人数:" + srcdtlo.getString("online"); + String area =srcdtlo.getString("platForm"); + String typechk =srcdtlo.optString("isLive"); + String type = typechk.equals("") ? "录播" : "正在直播中" ; + JSONObject result = new JSONObject(); + JSONObject vodList = new JSONObject(); + vodList.put("vod_id", ids.get(0)); + vodList.put("vod_pic",pic); + vodList.put("vod_name", title); + vodList.put("vod_area",area ); + vodList.put("type_name", type); + vodList.put("vod_actor", actor); + vodList.put("vod_director", director); + vodList.put("vod_content", content); + String playList = ""; + String pl=""; + List vodItems = new ArrayList<>(); + for(int i=0; i<5 ; i++){ + String[] qq = new String[] {"OD", "HD", "SD", "LD", "FD"}; + String qa = srcdtplayo.optString(qq[i]); + if (qa.isEmpty()) { + continue; + } + switch (qq[i]) { + case "OD": + pl = "原画" + "$" + qa; + break; + case "HD": + pl = "超清" + "$" + qa; + break; + case "SD": + pl = "高清" + "$" + qa; + break; + case "LD": + pl = "清晣" + "$" + qa; + break; + case "FD": + pl = "流畅" + "$" + qa; + break; + } + vodItems.add(pl); + } + if (vodItems.size() > 0) + playList = TextUtils.join("#", vodItems); + + if (playList.length() == 0) + playList = "NoStream$nolink"; + + Map vod_play = new TreeMap<>(); + vod_play.put("YJ1211", playList); + String vod_play_from = TextUtils.join("$$$", vod_play.keySet()); + String vod_play_url = TextUtils.join("$$$", vod_play.values()); + vodList.put("vod_play_from", vod_play_from); + vodList.put("vod_play_url", vod_play_url); + + JSONArray list = new JSONArray(); + list.put(vodList); + result.put("list", list); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + @Override + public String playerContent(String flag, String id, List vipFlags) { + try { + JSONObject result = new JSONObject(); + result.put("header", getHeaders()); + result.put("parse", 1); + result.put("playUrl", ""); + result.put("url", id); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } + + @Override + public String searchContent(String key, boolean quick) { + try { + String url = "http://live.yj1211.work/api/live/search?platform=all&keyWords=" + URLEncoder.encode(key) + "&isLive=0"; + String content =""; + for(int i=0 ; i<3 ; i++){ + content =OkHttp.string(url, getHeaders()); + if(content.length() >0){ + break; + }else{ + Thread.sleep(1000); + } + } + JSONObject searchResult = new JSONObject(content); + JSONArray sra = searchResult.getJSONArray("data"); + JSONObject result = new JSONObject(); + JSONArray videos = new JSONArray(); + if (sra.length() > 0) { + int ch = Math.min(sra.length(), 20); + for (int i = 0; i < ch; i++) { + JSONObject srat = new JSONObject(); + String platForm = sra.getJSONObject(i).getString("platform"); + String rd = sra.getJSONObject(i).getString("roomId"); + String id = platForm + "&" + rd; + String name = sra.getJSONObject(i).getString("nickName"); + String pic = sra.getJSONObject(i).getString("headPic"); + String mark =""; + if(!sra.getJSONObject(i).isNull("cateName")){ + mark = sra.getJSONObject(i).getString("cateName"); + } + srat.put("vod_remarks", mark); + srat.put("vod_id", id); + srat.put("vod_name", name); + srat.put("vod_pic", pic); + + videos.put(srat); + } + } + result.put("list", videos); + return result.toString(); + } catch (Exception e) { + SpiderDebug.log(e); + } + return ""; + } +} diff --git a/app/src/main/java/com/github/catvod/utils/Misc.java b/app/src/main/java/com/github/catvod/utils/Misc.java index c2dcac95..90861591 100644 --- a/app/src/main/java/com/github/catvod/utils/Misc.java +++ b/app/src/main/java/com/github/catvod/utils/Misc.java @@ -17,7 +17,9 @@ import org.json.JSONException; import org.json.JSONObject; import java.math.BigInteger; +import java.nio.charset.Charset; import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.List; import java.util.Locale; @@ -25,7 +27,8 @@ import java.util.Locale; public class Misc { public static final String CHROME = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"; - + public static Charset CharsetUTF8 = Charset.forName("UTF-8"); + public static Charset CharsetIOS8859 = Charset.forName("iso-8859-1"); public static boolean isVip(String url) { List hosts = Arrays.asList("iqiyi.com", "v.qq.com", "youku.com", "le.com", "tudou.com", "mgtv.com", "sohu.com", "acfun.cn", "bilibili.com", "baofeng.com", "pptv.com"); for (String host : hosts) if (url.contains(host)) return true; @@ -138,6 +141,26 @@ public class Misc { return ""; } } + public static String NewMD5(String src, Charset charset) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] bytes = md.digest(src.getBytes(charset)); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < bytes.length; i++) { + int v = bytes[i] & 0xFF; + String hv = Integer.toHexString(v); + if (hv.length() < 2) { + sb.append(0); + } + sb.append(hv); + } + return sb.toString().toLowerCase(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } + return ""; + } + public static DisplayMetrics getDisplayMetrics() { return Init.context().getResources().getDisplayMetrics(); diff --git a/app/src/main/java/com/github/catvod/utils/gZip.java b/app/src/main/java/com/github/catvod/utils/gZip.java new file mode 100644 index 00000000..e8f14ea1 --- /dev/null +++ b/app/src/main/java/com/github/catvod/utils/gZip.java @@ -0,0 +1,22 @@ +package com.github.catvod.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPInputStream; + + +public class gZip { + public static String KS(byte[] bArr) throws IOException { + GZIPInputStream gZIPInputStream = new GZIPInputStream(new ByteArrayInputStream(bArr)); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + byte[] bArr2 = new byte[1024]; + while (true) { + int read = gZIPInputStream.read(bArr2); + if (read == -1) { + return byteArrayOutputStream.toString("UTF-8"); + } + byteArrayOutputStream.write(bArr2, 0, read); + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/github/catvod/utils/okhttp/OKCallBack.java b/app/src/main/java/com/github/catvod/utils/okhttp/OKCallBack.java new file mode 100644 index 00000000..9bd5b7c0 --- /dev/null +++ b/app/src/main/java/com/github/catvod/utils/okhttp/OKCallBack.java @@ -0,0 +1,60 @@ +package com.github.catvod.utils.okhttp; + +import java.io.IOException; + +import okhttp3.Call; +import okhttp3.Response; + + +public abstract class OKCallBack { + + private T result = null; + + public T getResult() { + return result; + } + + protected void setResult(T val) { + result = val; + } + + protected void onError(final Call call, final Exception e) { + onFailure(call, e); + } + + protected void onSuccess(Call call, Response response) { + T obj = onParseResponse(call, response); + setResult(obj); + onResponse(obj); + } + + protected abstract T onParseResponse(Call call, Response response); + + protected abstract void onFailure(Call call, Exception e); + + protected abstract void onResponse(T response); + + public static abstract class OKCallBackDefault extends OKCallBack { + @Override + public Response onParseResponse(Call call, Response response) { + return response; + } + } + + public static abstract class OKCallBackString extends OKCallBack { + @Override + public void onError(Call call, Exception e) { + setResult(""); + super.onError(call, e); + } + + @Override + public String onParseResponse(Call call, Response response) { + try { + return response.body().string(); + } catch (IOException e) { + return ""; + } + } + } +} diff --git a/app/src/main/java/com/github/catvod/utils/okhttp/OKRequest.java b/app/src/main/java/com/github/catvod/utils/okhttp/OKRequest.java new file mode 100644 index 00000000..c24d5ba2 --- /dev/null +++ b/app/src/main/java/com/github/catvod/utils/okhttp/OKRequest.java @@ -0,0 +1,132 @@ +package com.github.catvod.utils.okhttp; + +import android.text.TextUtils; + +import java.io.IOException; +import java.util.Map; + +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.FormBody; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.RequestBody; +import okhttp3.Response; + +class OKRequest { + private final String mMethodType; + private String mUrl; + private Object mTag = null; + private final Map mParamsMap; + private final String mJsonStr; + private final Map mHeaderMap; + private final OKCallBack mCallBack; + private okhttp3.Request mOkHttpRequest; + private okhttp3.Request.Builder mRequestBuilder; + + + OKRequest(String methodType, String url, Map paramsMap, Map headerMap, OKCallBack callBack) { + this(methodType, url, null, paramsMap, headerMap, callBack); + } + + OKRequest(String methodType, String url, String jsonStr, Map headerMap, OKCallBack callBack) { + this(methodType, url, jsonStr, null, headerMap, callBack); + } + + private OKRequest(String methodType, String url, String jsonStr, Map paramsMap, Map headerMap, OKCallBack callBack) { + mMethodType = methodType; + mUrl = url; + mJsonStr = jsonStr; + mParamsMap = paramsMap; + mHeaderMap = headerMap; + mCallBack = callBack; + getInstance(); + } + + public void setTag(Object tag) { + mTag = tag; + } + + private void getInstance() { + mRequestBuilder = new okhttp3.Request.Builder(); + switch (mMethodType) { + case OkHttpUtil.METHOD_GET: + setGetParams(); + break; + case OkHttpUtil.METHOD_POST: + mRequestBuilder.post(getRequestBody()); + break; + } + mRequestBuilder.url(mUrl); + if (mTag != null) + mRequestBuilder.tag(mTag); + if (mHeaderMap != null) { + setHeader(); + } + mOkHttpRequest = mRequestBuilder.build(); + } + + private RequestBody getRequestBody() { + if (!TextUtils.isEmpty(mJsonStr)) { + MediaType JSON = MediaType.parse("application/json; charset=utf-8"); + return RequestBody.create(JSON, mJsonStr); + } + FormBody.Builder formBody = new FormBody.Builder(); + if (mParamsMap != null) { + for (String key : mParamsMap.keySet()) { + formBody.add(key, mParamsMap.get(key)); + } + } + return formBody.build(); + } + + private void setGetParams() { + if (mParamsMap != null) { + mUrl = mUrl + "?"; + for (String key : mParamsMap.keySet()) { + mUrl = mUrl + key + "=" + mParamsMap.get(key) + "&"; + } + mUrl = mUrl.substring(0, mUrl.length() - 1); + } + } + + private void setHeader() { + if (mHeaderMap != null) { + for (String key : mHeaderMap.keySet()) { + mRequestBuilder.addHeader(key, mHeaderMap.get(key)); + } + } + } + + void execute(OkHttpClient client) { + Call call = client.newCall(mOkHttpRequest); + try { + Response response = call.execute(); + if (mCallBack != null) { + mCallBack.onSuccess(call, response); + } + } catch (IOException e) { + if (mCallBack != null) { + mCallBack.onError(call, e); + } + } + } + + void call(OkHttpClient client) { + client.newCall(mOkHttpRequest).enqueue(new Callback() { + @Override + public void onFailure(final Call call, final IOException e) { + if (mCallBack != null) { + mCallBack.onError(call, e); + } + } + + @Override + public void onResponse(final Call call, final Response response) throws IOException { + if (mCallBack != null) { + mCallBack.onSuccess(call, response); + } + } + }); + } +} diff --git a/app/src/main/java/com/github/catvod/utils/okhttp/OkHttpUtil.java b/app/src/main/java/com/github/catvod/utils/okhttp/OkHttpUtil.java new file mode 100644 index 00000000..e161c404 --- /dev/null +++ b/app/src/main/java/com/github/catvod/utils/okhttp/OkHttpUtil.java @@ -0,0 +1,200 @@ +package com.github.catvod.utils.okhttp; + +import com.github.catvod.crawler.SpiderDebug; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import okhttp3.Call; +import okhttp3.OkHttpClient; +import okhttp3.Response; + +public class OkHttpUtil { + + public static final String METHOD_GET = "GET"; + public static final String METHOD_POST = "POST"; + + private static final int DEFAULT_TIMEOUT = 15; + + private static final Object lockO = new Object(); + + private static OkHttpClient defaultClient = null; + + /** + * 不自动重定向 + */ + private static OkHttpClient noRedirectClient = null; + + public static OkHttpClient defaultClient() { + synchronized (lockO) { + if (defaultClient == null) { + OkHttpClient.Builder builder = new OkHttpClient.Builder() + .readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) + .writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) + .connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) + .retryOnConnectionFailure(true) + .sslSocketFactory(new SSLSocketFactoryCompat(SSLSocketFactoryCompat.trustAllCert), SSLSocketFactoryCompat.trustAllCert); + defaultClient = builder.build(); + } + return defaultClient; + } + } + + public static OkHttpClient noRedirectClient() { + synchronized (lockO) { + if (noRedirectClient == null) { + OkHttpClient.Builder builder = new OkHttpClient.Builder() + .readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) + .writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) + .connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) + .followRedirects(false) + .followSslRedirects(false) + .retryOnConnectionFailure(true) + .sslSocketFactory(new SSLSocketFactoryCompat(SSLSocketFactoryCompat.trustAllCert), SSLSocketFactoryCompat.trustAllCert); + noRedirectClient = builder.build(); + } + return noRedirectClient; + } + } + + public static String string(OkHttpClient client, String url, String tag, Map paramsMap, Map headerMap, Map> respHeaderMap) { + OKCallBack stringCallback = new OKCallBack() { + @Override + public String onParseResponse(Call call, Response response) { + try { + if (respHeaderMap != null) { + respHeaderMap.clear(); + respHeaderMap.putAll(response.headers().toMultimap()); + } + return response.body().string(); + } catch (IOException e) { + return ""; + } + } + + @Override + public void onFailure(Call call, Exception e) { + setResult(""); + SpiderDebug.log(e); + } + + @Override + public void onResponse(String response) { + } + }; + OKRequest req = new OKRequest(METHOD_GET, url, paramsMap, headerMap, stringCallback); + req.setTag(tag); + req.execute(client); + return stringCallback.getResult(); + } + + public static String stringNoRedirect(String url, Map headerMap, Map> respHeaderMap) { + return string(noRedirectClient(), url, null, null, headerMap, respHeaderMap); + } + + public static String string(String url, Map headerMap, Map> respHeaderMap) { + return string(defaultClient(), url, null, null, headerMap, respHeaderMap); + } + + public static String string(String url, Map headerMap) { + return string(defaultClient(), url, null, null, headerMap, null); + } + + public static String string(String url, String tag, Map headerMap) { + return string(defaultClient(), url, tag, null, headerMap, null); + } + + public static void get(OkHttpClient client, String url, OKCallBack callBack) { + get(client, url, null, null, callBack); + } + + public static void get(OkHttpClient client, String url, Map paramsMap, OKCallBack callBack) { + get(client, url, paramsMap, null, callBack); + } + + public static void get(OkHttpClient client, String url, Map paramsMap, Map headerMap, OKCallBack callBack) { + new OKRequest(METHOD_GET, url, paramsMap, headerMap, callBack).execute(client); + } + + public static void post(OkHttpClient client, String url, OKCallBack callBack) { + post(client, url, null, callBack); + } + + public static void post(OkHttpClient client, String url, Map paramsMap, OKCallBack callBack) { + post(client, url, paramsMap, null, callBack); + } + + public static void post(OkHttpClient client, String url, Map paramsMap, Map headerMap, OKCallBack callBack) { + new OKRequest(METHOD_POST, url, paramsMap, headerMap, callBack).execute(client); + } + + public static void post(OkHttpClient client, String url, String tag, Map paramsMap, Map headerMap, OKCallBack callBack) { + OKRequest req = new OKRequest(METHOD_POST, url, paramsMap, headerMap, callBack); + req.setTag(tag); + req.execute(client); + } + + public static void postJson(OkHttpClient client, String url, String jsonStr, OKCallBack callBack) { + postJson(client, url, jsonStr, null, callBack); + } + + public static void postJson(OkHttpClient client, String url, String jsonStr, Map headerMap, OKCallBack callBack) { + new OKRequest(METHOD_POST, url, jsonStr, headerMap, callBack).execute(client); + } + + /** + * 根据Tag取消请求 + */ + public static void cancel(OkHttpClient client, Object tag) { + if (client == null || tag == null) return; + for (Call call : client.dispatcher().queuedCalls()) { + if (tag.equals(call.request().tag())) { + call.cancel(); + } + } + for (Call call : client.dispatcher().runningCalls()) { + if (tag.equals(call.request().tag())) { + call.cancel(); + } + } + } + + public static void cancel(Object tag) { + cancel(defaultClient(), tag); + } + + public static void cancelAll() { + cancelAll(defaultClient()); + } + + /** + * 取消所有请求请求 + */ + public static void cancelAll(OkHttpClient client) { + if (client == null) return; + for (Call call : client.dispatcher().queuedCalls()) { + call.cancel(); + } + for (Call call : client.dispatcher().runningCalls()) { + call.cancel(); + } + } + + /** + * 获取重定向地址 + * + * @param headers + * @return + */ + public static String getRedirectLocation(Map> headers) { + if (headers == null) + return null; + if (headers.containsKey("location")) + return headers.get("location").get(0); + if (headers.containsKey("Location")) + return headers.get("Location").get(0); + return null; + } +} diff --git a/app/src/main/java/com/github/catvod/utils/okhttp/SSLSocketFactoryCompat.java b/app/src/main/java/com/github/catvod/utils/okhttp/SSLSocketFactoryCompat.java new file mode 100644 index 00000000..f0395e06 --- /dev/null +++ b/app/src/main/java/com/github/catvod/utils/okhttp/SSLSocketFactoryCompat.java @@ -0,0 +1,167 @@ +package com.github.catvod.utils.okhttp; + +import android.os.Build; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.security.GeneralSecurityException; +import java.security.cert.CertificateException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.X509TrustManager; + +public class SSLSocketFactoryCompat extends SSLSocketFactory { + private final SSLSocketFactory defaultFactory; + // Android 5.0+ (API level21) provides reasonable default settings + // but it still allows SSLv3 + // https://developer.android.com/about/versions/android-5.0-changes.html#ssl + static String[] protocols = null; + static String[] cipherSuites = null; + + static { + try { + SSLSocket socket = (SSLSocket) SSLSocketFactory.getDefault().createSocket(); + if (socket != null) { + /* set reasonable protocol versions */ + // - enable all supported protocols (enables TLSv1.1 and TLSv1.2 on Android <5.0) + // - remove all SSL versions (especially SSLv3) because they're insecure now + List protocols = new LinkedList<>(); + for (String protocol : socket.getSupportedProtocols()) + if (!protocol.toUpperCase().contains("SSL")) + protocols.add(protocol); + SSLSocketFactoryCompat.protocols = protocols.toArray(new String[protocols.size()]); + /* set up reasonable cipher suites */ + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + // choose known secure cipher suites + List allowedCiphers = Arrays.asList( + // TLS 1.2 + "TLS_RSA_WITH_AES_256_GCM_SHA384", + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", + "TLS_ECHDE_RSA_WITH_AES_128_GCM_SHA256", + // maximum interoperability + "TLS_RSA_WITH_3DES_EDE_CBC_SHA", + "TLS_RSA_WITH_AES_128_CBC_SHA", + // additionally + "TLS_RSA_WITH_AES_256_CBC_SHA", + "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA", + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", + "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA", + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA"); + List availableCiphers = Arrays.asList(socket.getSupportedCipherSuites()); + // take all allowed ciphers that are available and put them into preferredCiphers + HashSet preferredCiphers = new HashSet<>(allowedCiphers); + preferredCiphers.retainAll(availableCiphers); + /* For maximum security, preferredCiphers should *replace* enabled ciphers (thus disabling + * ciphers which are enabled by default, but have become unsecure), but I guess for + * the security level of DAVdroid and maximum compatibility, disabling of insecure + * ciphers should be a server-side task */ + // add preferred ciphers to enabled ciphers + HashSet enabledCiphers = preferredCiphers; + enabledCiphers.addAll(new HashSet<>(Arrays.asList(socket.getEnabledCipherSuites()))); + SSLSocketFactoryCompat.cipherSuites = enabledCiphers.toArray(new String[enabledCiphers.size()]); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public SSLSocketFactoryCompat(X509TrustManager tm) { + try { + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, (tm != null) ? new X509TrustManager[]{tm} : null, null); + defaultFactory = sslContext.getSocketFactory(); + } catch (GeneralSecurityException e) { + throw new AssertionError(); // The system has no TLS. Just give up. + } + } + + private void upgradeTLS(SSLSocket ssl) { + // Android 5.0+ (API level21) provides reasonable default settings + // but it still allows SSLv3 + // https://developer.android.com/about/versions/android-5.0-changes.html#ssl + if (protocols != null) { + ssl.setEnabledProtocols(protocols); + } + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP && cipherSuites != null) { + ssl.setEnabledCipherSuites(cipherSuites); + } + } + + @Override + public String[] getDefaultCipherSuites() { + return cipherSuites; + } + + @Override + public String[] getSupportedCipherSuites() { + return cipherSuites; + } + + @Override + public Socket createSocket(Socket s, String host, int port, boolean autoClose) throws IOException { + Socket ssl = defaultFactory.createSocket(s, host, port, autoClose); + if (ssl instanceof SSLSocket) + upgradeTLS((SSLSocket) ssl); + return ssl; + } + + @Override + public Socket createSocket(String host, int port) throws IOException { + Socket ssl = defaultFactory.createSocket(host, port); + if (ssl instanceof SSLSocket) + upgradeTLS((SSLSocket) ssl); + return ssl; + } + + @Override + public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException { + Socket ssl = defaultFactory.createSocket(host, port, localHost, localPort); + if (ssl instanceof SSLSocket) + upgradeTLS((SSLSocket) ssl); + return ssl; + } + + @Override + public Socket createSocket(InetAddress host, int port) throws IOException { + Socket ssl = defaultFactory.createSocket(host, port); + if (ssl instanceof SSLSocket) + upgradeTLS((SSLSocket) ssl); + return ssl; + } + + @Override + public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException { + Socket ssl = defaultFactory.createSocket(address, port, localAddress, localPort); + if (ssl instanceof SSLSocket) + upgradeTLS((SSLSocket) ssl); + return ssl; + } + + //定义一个信任所有证书的TrustManager + public static final X509TrustManager trustAllCert = new X509TrustManager() { + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String authType) throws CertificateException { + } + + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String authType) throws CertificateException { + } + + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return new java.security.cert.X509Certificate[]{}; + } + }; +} diff --git a/jar/custom_spider.jar b/jar/custom_spider.jar index b0e01555..1533d3c6 100644 Binary files a/jar/custom_spider.jar and b/jar/custom_spider.jar differ diff --git a/jar/custom_spider.jar.md5 b/jar/custom_spider.jar.md5 index da14f3c3..42dcb4c3 100644 --- a/jar/custom_spider.jar.md5 +++ b/jar/custom_spider.jar.md5 @@ -1 +1 @@ -0a5952d9789ee1a8398598bea20ce488 +f97e3722085c65ff04e5fe63429a7676