diff --git a/app/src/main/java/com/github/catvod/spider/Douban.java b/app/src/main/java/com/github/catvod/spider/Douban.java index 16ff947b..3e400583 100644 --- a/app/src/main/java/com/github/catvod/spider/Douban.java +++ b/app/src/main/java/com/github/catvod/spider/Douban.java @@ -1,22 +1,28 @@ package com.github.catvod.spider; import android.content.Context; + import com.github.catvod.bean.Class; import com.github.catvod.bean.Result; import com.github.catvod.bean.Vod; import com.github.catvod.crawler.Spider; -import com.github.catvod.crawler.SpiderDebug; import com.github.catvod.net.OkHttp; import com.github.catvod.utils.Utils; import com.google.gson.JsonParser; + import org.json.JSONArray; import org.json.JSONObject; import java.net.URLEncoder; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class Douban extends Spider { - private final String hostURL = "https://frodo.douban.com/api/v2"; + + private final String siteUrl = "https://frodo.douban.com/api/v2"; private final String apikey = "?apikey=0ac44ae016490db2204ce0a042db2916"; private String extend; @@ -31,7 +37,6 @@ public class Douban extends Spider { @Override public void init(Context context, String extend) throws Exception { - super.init(context, extend); this.extend = extend; } @@ -41,59 +46,56 @@ public class Douban extends Spider { List typeIds = Arrays.asList("hot_gaia", "tv_hot", "show_hot", "movie", "tv", "rank_list_movie", "rank_list_tv"); List typeNames = Arrays.asList("热门电影", "热播剧集", "热播综艺", "电影筛选", "电视筛选", "电影榜单", "电视剧榜单"); for (int i = 0; i < typeIds.size(); i++) classes.add(new Class(typeIds.get(i), typeNames.get(i))); - String recommendURL = "http://api.douban.com/api/v2/subject_collection/subject_real_time_hotest/items" + apikey; - JSONObject jsonObject = new JSONObject(OkHttp.string(recommendURL, getHeader())); + String recommendUrl = "http://api.douban.com/api/v2/subject_collection/subject_real_time_hotest/items" + apikey; + JSONObject jsonObject = new JSONObject(OkHttp.string(recommendUrl, getHeader())); JSONArray items = jsonObject.optJSONArray("subject_collection_items"); return Result.string(classes, parseVodListFromJSONArray(items), filter ? JsonParser.parseString(OkHttp.string(extend)) : null); } @Override public String categoryContent(String tid, String pg, boolean filter, HashMap extend) throws Exception { - HashMap ext = new HashMap<>(); - if (extend != null && extend.size() > 0) { - ext.putAll(extend); - } - String sort = ext.get("sort") == null ? "T" : ext.get("sort"); - String tags = URLEncoder.encode(getTags(ext)); + String sort = extend.get("sort") == null ? "T" : extend.get("sort"); + String tags = URLEncoder.encode(getTags(extend)); int start = (Integer.parseInt(pg) - 1) * 20; - String cateURL; + String cateUrl; String itemKey = "items"; switch (tid) { case "hot_gaia": - sort = ext.get("sort") == null ? "recommend" : ext.get("sort"); - String area = ext.get("area") == null ? "全部" : ext.get("area"); + sort = extend.get("sort") == null ? "recommend" : extend.get("sort"); + String area = extend.get("area") == null ? "全部" : extend.get("area"); sort = sort + "&area=" + URLEncoder.encode(area); - cateURL = hostURL + "/movie/hot_gaia" + apikey + "&sort=" + sort + "&start=" + start + "&count=20"; + cateUrl = siteUrl + "/movie/hot_gaia" + apikey + "&sort=" + sort + "&start=" + start + "&count=20"; break; case "tv_hot": - String type = ext.get("type") == null ? "tv_hot" : ext.get("type"); - cateURL = hostURL + "/subject_collection/" + type + "/items" + apikey + "&start=" + start + "&count=20"; + String type = extend.get("type") == null ? "tv_hot" : extend.get("type"); + cateUrl = siteUrl + "/subject_collection/" + type + "/items" + apikey + "&start=" + start + "&count=20"; itemKey = "subject_collection_items"; break; case "show_hot": - String showType = ext.get("type") == null ? "show_hot" : ext.get("type"); - cateURL = hostURL + "/subject_collection/" + showType + "/items" + apikey + "&start=" + start + "&count=20"; + String showType = extend.get("type") == null ? "show_hot" : extend.get("type"); + cateUrl = siteUrl + "/subject_collection/" + showType + "/items" + apikey + "&start=" + start + "&count=20"; itemKey = "subject_collection_items"; break; case "tv": - cateURL = hostURL + "/tv/recommend" + apikey + "&sort=" + sort + "&tags=" + tags + "&start=" + start + "&count=20"; + cateUrl = siteUrl + "/tv/recommend" + apikey + "&sort=" + sort + "&tags=" + tags + "&start=" + start + "&count=20"; break; case "rank_list_movie": - String rankMovieType = ext.get("榜单") == null ? "movie_real_time_hotest" : ext.get("榜单"); - cateURL = hostURL + "/subject_collection/" + rankMovieType + "/items" + apikey + "&start=" + start + "&count=20"; + String rankMovieType = extend.get("榜单") == null ? "movie_real_time_hotest" : extend.get("榜单"); + cateUrl = siteUrl + "/subject_collection/" + rankMovieType + "/items" + apikey + "&start=" + start + "&count=20"; itemKey = "subject_collection_items"; break; case "rank_list_tv": - String rankTVType = ext.get("榜单") == null ? "tv_real_time_hotest" : ext.get("榜单"); - cateURL = hostURL + "/subject_collection/" + rankTVType + "/items" + apikey + "&start=" + start + "&count=20"; + String rankTVType = extend.get("榜单") == null ? "tv_real_time_hotest" : extend.get("榜单"); + cateUrl = siteUrl + "/subject_collection/" + rankTVType + "/items" + apikey + "&start=" + start + "&count=20"; itemKey = "subject_collection_items"; break; default: - cateURL = hostURL + "/movie/recommend" + apikey + "&sort=" + sort + "&tags=" + tags + "&start=" + start + "&count=20"; + cateUrl = siteUrl + "/movie/recommend" + apikey + "&sort=" + sort + "&tags=" + tags + "&start=" + start + "&count=20"; + break; } - JSONObject jsonObject = new JSONObject(OkHttp.string(cateURL, getHeader())); - JSONArray items = jsonObject.getJSONArray(itemKey); - List list = parseVodListFromJSONArray(items); + JSONObject object = new JSONObject(OkHttp.string(cateUrl, getHeader())); + JSONArray array = object.getJSONArray(itemKey); + List list = parseVodListFromJSONArray(array); int page = Integer.parseInt(pg), count = Integer.MAX_VALUE, limit = 20, total = Integer.MAX_VALUE; return Result.get().vod(list).page(page, count, limit, total).string(); } @@ -115,31 +117,25 @@ public class Douban extends Spider { try { return "评分:" + item.getJSONObject("rating").optString("value"); } catch (Exception e) { - SpiderDebug.log(e); + return ""; } - return ""; } private String getPic(JSONObject item) { try { return item.getJSONObject("pic").optString("normal") + "@Referer=https://api.douban.com/@User-Agent=" + Utils.CHROME; } catch (Exception e) { - SpiderDebug.log(e); + return ""; } - return ""; } - private String getTags(HashMap ext) { + private String getTags(HashMap extend) { try { StringBuilder tags = new StringBuilder(); - for (String key : ext.keySet()) { - if (key.equals("sort")) continue; - tags.append(ext.get(key)).append(","); - } - return tags.substring(0, tags.lastIndexOf(",")); + for (String key : extend.keySet()) if (!key.equals("sort")) tags.append(extend.get(key)).append(","); + return Utils.substring(tags.toString()); } catch (Exception e) { -// SpiderDebug.log(e); + return ""; } - return ""; } } diff --git a/app/src/main/java/com/github/catvod/spider/Xb6v.java b/app/src/main/java/com/github/catvod/spider/Xb6v.java index 3bd17414..dc756b66 100644 --- a/app/src/main/java/com/github/catvod/spider/Xb6v.java +++ b/app/src/main/java/com/github/catvod/spider/Xb6v.java @@ -1,41 +1,47 @@ package com.github.catvod.spider; import android.text.TextUtils; + import com.github.catvod.bean.Class; import com.github.catvod.bean.Filter; import com.github.catvod.bean.Result; import com.github.catvod.bean.Vod; import com.github.catvod.crawler.Spider; -import com.github.catvod.crawler.SpiderDebug; import com.github.catvod.net.OkHttp; import com.github.catvod.utils.Utils; -import okhttp3.FormBody; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import okhttp3.FormBody; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + /** * @author zhixc * 新版6V电影网 */ public class Xb6v extends Spider { - private final String siteURL = "http://www.xb6v.com"; - private String nextSearchURLPrefix; - private String nextSearchURLSuffix; + private final String siteUrl = "http://www.xb6v.com"; + private String nextSearchUrlPrefix; + private String nextSearchUrlSuffix; private Map getHeader() { Map header = new HashMap<>(); header.put("User-Agent", Utils.CHROME); - header.put("Referer", siteURL + "/"); + header.put("Referer", siteUrl + "/"); return header; } @@ -48,7 +54,7 @@ public class Xb6v extends Spider { @Override public String homeContent(boolean filter) throws Exception { List classes = new ArrayList<>(); - String html = OkHttp.string(siteURL, getHeader()); + String html = OkHttp.string(siteUrl, getHeader()); Document doc = Jsoup.parse(html); Elements elements = doc.select("#menus > li > a"); LinkedHashMap> filters = new LinkedHashMap<>(); @@ -72,6 +78,10 @@ public class Xb6v extends Spider { return Result.string(classes, parseVodListFromDoc(doc), filters); } + private List parseVodListFromDoc(String html) { + return parseVodListFromDoc(Jsoup.parse(html)); + } + private List parseVodListFromDoc(Document doc) { Elements items = doc.select("#post_container .post_hover"); List list = new ArrayList<>(); @@ -88,14 +98,10 @@ public class Xb6v extends Spider { @Override public String categoryContent(String tid, String pg, boolean filter, HashMap extend) throws Exception { - HashMap ext = new HashMap<>(); - if (extend != null && extend.size() > 0) { - ext.putAll(extend); - } - String cateId = ext.get("cateId") == null ? "" : ext.get("cateId"); - String cateURL = siteURL + tid + cateId; - if (!pg.equals("1")) cateURL += "index_" + pg + ".html"; - String html = OkHttp.string(cateURL, getHeader()); + String cateId = extend.get("cateId") == null ? "" : extend.get("cateId"); + String cateUrl = siteUrl + tid + cateId; + if (!pg.equals("1")) cateUrl += "index_" + pg + ".html"; + String html = OkHttp.string(cateUrl, getHeader()); Document doc = Jsoup.parse(html); String href = doc.select(".pagination > a").last().attr("href"); int page = Integer.parseInt(pg); @@ -109,8 +115,8 @@ public class Xb6v extends Spider { @Override public String detailContent(List ids) throws Exception { String vodId = ids.get(0); - String detailURL = siteURL + vodId; - String html = OkHttp.string(detailURL, getDetailHeader()); + String detailUrl = siteUrl + vodId; + String html = OkHttp.string(detailUrl, getDetailHeader()); Document doc = Jsoup.parse(html); Elements sourceList = doc.select("#post_content"); @@ -121,10 +127,10 @@ public class Xb6v extends Spider { Elements aList = source.select("table a"); List vodItems = new ArrayList<>(); for (Element a : aList) { - String episodeURL = a.attr("href"); + String episodeUrl = a.attr("href"); String episodeName = a.text(); - if (!episodeURL.toLowerCase().startsWith("magnet")) continue; - vodItems.add(episodeName + "$" + episodeURL); + if (!episodeUrl.toLowerCase().startsWith("magnet")) continue; + vodItems.add(episodeName + "$" + episodeUrl); } if (vodItems.size() > 0) { i++; @@ -161,22 +167,15 @@ public class Xb6v extends Spider { vod.setVodActor(actor); vod.setVodDirector(director); vod.setVodContent(description); - if (playMap.size() > 0) { - vod.setVodPlayFrom(TextUtils.join("$$$", playMap.keySet())); - vod.setVodPlayUrl(TextUtils.join("$$$", playMap.values())); - } + vod.setVodPlayFrom(TextUtils.join("$$$", playMap.keySet())); + vod.setVodPlayUrl(TextUtils.join("$$$", playMap.values())); + return Result.string(vod); } private String getStrByRegex(Pattern pattern, String str) { - try { - Matcher matcher = pattern.matcher(str); - if (matcher.find()) { - return matcher.group(1).trim(); - } - } catch (Exception e) { - SpiderDebug.log(e); - } + Matcher matcher = pattern.matcher(str); + if (matcher.find()) return matcher.group(1).trim(); return ""; } @@ -208,8 +207,7 @@ public class Xb6v extends Spider { @Override public String searchContent(String key, boolean quick, String pg) throws Exception { - String searchURL = siteURL + "/e/search/index.php"; - String html; + String searchUrl = siteUrl + "/e/search/index.php"; if (pg.equals("1")) { RequestBody formBody = new FormBody.Builder() .add("show", "title") @@ -220,30 +218,22 @@ public class Xb6v extends Spider { .add("submit", "") .addEncoded("keyboard", key) .build(); - Request request = new Request.Builder() - .url(searchURL) + Request request = new Request.Builder().url(searchUrl) .addHeader("User-Agent", Utils.CHROME) - .addHeader("Origin", siteURL) - .addHeader("Referer", siteURL + "/") + .addHeader("Origin", siteUrl) + .addHeader("Referer", siteUrl + "/") .post(formBody) .build(); Response response = OkHttp.client().newCall(request).execute(); - if (response.body() == null) return ""; - initNextSearchURL(response); - html = response.body().string(); - response.close(); + String[] split = String.valueOf(response.request().url()).split("\\?searchid="); + nextSearchUrlPrefix = split[0] + "index.php?page="; + nextSearchUrlSuffix = "&searchid=" + split[1]; + return Result.string(parseVodListFromDoc(response.body().string())); } else { int page = Integer.parseInt(pg) - 1; - searchURL = nextSearchURLPrefix + page + nextSearchURLSuffix; - html = OkHttp.string(searchURL, getHeader()); + searchUrl = nextSearchUrlPrefix + page + nextSearchUrlSuffix; + return Result.string(parseVodListFromDoc(OkHttp.string(searchUrl, getHeader()))); } - return Result.string(parseVodListFromDoc(Jsoup.parse(html))); - } - - private void initNextSearchURL(Response response) { - String[] split = String.valueOf(response.request().url()).split("\\?searchid="); - nextSearchURLPrefix = split[0] + "index.php?page="; - nextSearchURLSuffix = "&searchid=" + split[1]; } @Override diff --git a/jar/custom_spider.jar b/jar/custom_spider.jar index 3b54dfc3..a3c9acba 100644 Binary files a/jar/custom_spider.jar and b/jar/custom_spider.jar differ diff --git a/jar/custom_spider.jar.md5 b/jar/custom_spider.jar.md5 index 13285e0d..39e2c61c 100644 --- a/jar/custom_spider.jar.md5 +++ b/jar/custom_spider.jar.md5 @@ -1 +1 @@ -386aa45681818befd51ff5008f88f4e5 +a4a22ad4042d5ff3444ed4b66f41a3a4