dongge
/
AndroidCatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean code

This commit is contained in:
FongMi 2023-10-28 11:51:28 +08:00
parent 5daa7f4a17
commit 6698eace27
4 changed files with 83 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
app/src/main/java/com/github/catvod/spider/Douban.java
View File

@ -1,22 +1,28 @@
package com.github.catvod.spider; package com.github.catvod.spider;
import android.content.Context; import android.content.Context;
import com.github.catvod.bean.Class; import com.github.catvod.bean.Class;
import com.github.catvod.bean.Result; import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod; import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider; import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp; import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils; import com.github.catvod.utils.Utils;
import com.google.gson.JsonParser; import com.google.gson.JsonParser;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.*; import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class Douban extends Spider { public class Douban extends Spider {
private final String hostURL = "https://frodo.douban.com/api/v2";
private final String siteUrl = "https://frodo.douban.com/api/v2";
private final String apikey = "?apikey=0ac44ae016490db2204ce0a042db2916"; private final String apikey = "?apikey=0ac44ae016490db2204ce0a042db2916";
private String extend; private String extend;
@ -31,7 +37,6 @@ public class Douban extends Spider {
@Override @Override
public void init(Context context, String extend) throws Exception { public void init(Context context, String extend) throws Exception {
super.init(context, extend);
this.extend = extend; this.extend = extend;
} }
@ -41,59 +46,56 @@ public class Douban extends Spider {
List<String> typeIds = Arrays.asList("hot_gaia", "tv_hot", "show_hot", "movie", "tv", "rank_list_movie", "rank_list_tv"); List<String> typeIds = Arrays.asList("hot_gaia", "tv_hot", "show_hot", "movie", "tv", "rank_list_movie", "rank_list_tv");
List<String> typeNames = Arrays.asList("热门电影", "热播剧集", "热播综艺", "电影筛选", "电视筛选", "电影榜单", "电视剧榜单"); List<String> typeNames = Arrays.asList("热门电影", "热播剧集", "热播综艺", "电影筛选", "电视筛选", "电影榜单", "电视剧榜单");
for (int i = 0; i < typeIds.size(); i++) classes.add(new Class(typeIds.get(i), typeNames.get(i))); for (int i = 0; i < typeIds.size(); i++) classes.add(new Class(typeIds.get(i), typeNames.get(i)));
String recommendURL = "http://api.douban.com/api/v2/subject_collection/subject_real_time_hotest/items" + apikey; String recommendUrl = "http://api.douban.com/api/v2/subject_collection/subject_real_time_hotest/items" + apikey;
JSONObject jsonObject = new JSONObject(OkHttp.string(recommendURL, getHeader())); JSONObject jsonObject = new JSONObject(OkHttp.string(recommendUrl, getHeader()));
JSONArray items = jsonObject.optJSONArray("subject_collection_items"); JSONArray items = jsonObject.optJSONArray("subject_collection_items");
return Result.string(classes, parseVodListFromJSONArray(items), filter ? JsonParser.parseString(OkHttp.string(extend)) : null); return Result.string(classes, parseVodListFromJSONArray(items), filter ? JsonParser.parseString(OkHttp.string(extend)) : null);
} }
@Override @Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws Exception { public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws Exception {
HashMap<String, String> ext = new HashMap<>(); String sort = extend.get("sort") == null ? "T" : extend.get("sort");
if (extend != null && extend.size() > 0) { String tags = URLEncoder.encode(getTags(extend));
ext.putAll(extend);
}
String sort = ext.get("sort") == null ? "T" : ext.get("sort");
String tags = URLEncoder.encode(getTags(ext));
int start = (Integer.parseInt(pg) - 1) * 20; int start = (Integer.parseInt(pg) - 1) * 20;
String cateURL; String cateUrl;
String itemKey = "items"; String itemKey = "items";
switch (tid) { switch (tid) {
case "hot_gaia": case "hot_gaia":
sort = ext.get("sort") == null ? "recommend" : ext.get("sort"); sort = extend.get("sort") == null ? "recommend" : extend.get("sort");
String area = ext.get("area") == null ? "全部" : ext.get("area"); String area = extend.get("area") == null ? "全部" : extend.get("area");
sort = sort + "&area=" + URLEncoder.encode(area); sort = sort + "&area=" + URLEncoder.encode(area);
cateURL = hostURL + "/movie/hot_gaia" + apikey + "&sort=" + sort + "&start=" + start + "&count=20"; cateUrl = siteUrl + "/movie/hot_gaia" + apikey + "&sort=" + sort + "&start=" + start + "&count=20";
break; break;
case "tv_hot": case "tv_hot":
String type = ext.get("type") == null ? "tv_hot" : ext.get("type"); String type = extend.get("type") == null ? "tv_hot" : extend.get("type");
cateURL = hostURL + "/subject_collection/" + type + "/items" + apikey + "&start=" + start + "&count=20"; cateUrl = siteUrl + "/subject_collection/" + type + "/items" + apikey + "&start=" + start + "&count=20";
itemKey = "subject_collection_items"; itemKey = "subject_collection_items";
break; break;
case "show_hot": case "show_hot":
String showType = ext.get("type") == null ? "show_hot" : ext.get("type"); String showType = extend.get("type") == null ? "show_hot" : extend.get("type");
cateURL = hostURL + "/subject_collection/" + showType + "/items" + apikey + "&start=" + start + "&count=20"; cateUrl = siteUrl + "/subject_collection/" + showType + "/items" + apikey + "&start=" + start + "&count=20";
itemKey = "subject_collection_items"; itemKey = "subject_collection_items";
break; break;
case "tv": case "tv":
cateURL = hostURL + "/tv/recommend" + apikey + "&sort=" + sort + "&tags=" + tags + "&start=" + start + "&count=20"; cateUrl = siteUrl + "/tv/recommend" + apikey + "&sort=" + sort + "&tags=" + tags + "&start=" + start + "&count=20";
break; break;
case "rank_list_movie": case "rank_list_movie":
String rankMovieType = ext.get("榜单") == null ? "movie_real_time_hotest" : ext.get("榜单"); String rankMovieType = extend.get("榜单") == null ? "movie_real_time_hotest" : extend.get("榜单");
cateURL = hostURL + "/subject_collection/" + rankMovieType + "/items" + apikey + "&start=" + start + "&count=20"; cateUrl = siteUrl + "/subject_collection/" + rankMovieType + "/items" + apikey + "&start=" + start + "&count=20";
itemKey = "subject_collection_items"; itemKey = "subject_collection_items";
break; break;
case "rank_list_tv": case "rank_list_tv":
String rankTVType = ext.get("榜单") == null ? "tv_real_time_hotest" : ext.get("榜单"); String rankTVType = extend.get("榜单") == null ? "tv_real_time_hotest" : extend.get("榜单");
cateURL = hostURL + "/subject_collection/" + rankTVType + "/items" + apikey + "&start=" + start + "&count=20"; cateUrl = siteUrl + "/subject_collection/" + rankTVType + "/items" + apikey + "&start=" + start + "&count=20";
itemKey = "subject_collection_items"; itemKey = "subject_collection_items";
break; break;
default: default:
cateURL = hostURL + "/movie/recommend" + apikey + "&sort=" + sort + "&tags=" + tags + "&start=" + start + "&count=20"; cateUrl = siteUrl + "/movie/recommend" + apikey + "&sort=" + sort + "&tags=" + tags + "&start=" + start + "&count=20";
break;
} }
JSONObject jsonObject = new JSONObject(OkHttp.string(cateURL, getHeader())); JSONObject object = new JSONObject(OkHttp.string(cateUrl, getHeader()));
JSONArray items = jsonObject.getJSONArray(itemKey); JSONArray array = object.getJSONArray(itemKey);
List<Vod> list = parseVodListFromJSONArray(items); List<Vod> list = parseVodListFromJSONArray(array);
int page = Integer.parseInt(pg), count = Integer.MAX_VALUE, limit = 20, total = Integer.MAX_VALUE; int page = Integer.parseInt(pg), count = Integer.MAX_VALUE, limit = 20, total = Integer.MAX_VALUE;
return Result.get().vod(list).page(page, count, limit, total).string(); return Result.get().vod(list).page(page, count, limit, total).string();
} }
@ -115,31 +117,25 @@ public class Douban extends Spider {
try { try {
return "评分：" + item.getJSONObject("rating").optString("value"); return "评分：" + item.getJSONObject("rating").optString("value");
} catch (Exception e) { } catch (Exception e) {
SpiderDebug.log(e);
}
return ""; return "";
} }
}
private String getPic(JSONObject item) { private String getPic(JSONObject item) {
try { try {
return item.getJSONObject("pic").optString("normal") + "@Referer=https://api.douban.com/@User-Agent=" + Utils.CHROME; return item.getJSONObject("pic").optString("normal") + "@Referer=https://api.douban.com/@User-Agent=" + Utils.CHROME;
} catch (Exception e) { } catch (Exception e) {
SpiderDebug.log(e);
}
return ""; return "";
} }
}
private String getTags(HashMap<String, String> ext) { private String getTags(HashMap<String, String> extend) {
try { try {
StringBuilder tags = new StringBuilder(); StringBuilder tags = new StringBuilder();
for (String key : ext.keySet()) { for (String key : extend.keySet()) if (!key.equals("sort")) tags.append(extend.get(key)).append(",");
if (key.equals("sort")) continue; return Utils.substring(tags.toString());
tags.append(ext.get(key)).append(",");
}
return tags.substring(0, tags.lastIndexOf(","));
} catch (Exception e) { } catch (Exception e) {
// SpiderDebug.log(e);
}
return ""; return "";
} }
}
} }

94
app/src/main/java/com/github/catvod/spider/Xb6v.java
View File

@ -1,41 +1,47 @@
package com.github.catvod.spider; package com.github.catvod.spider;
import android.text.TextUtils; import android.text.TextUtils;
import com.github.catvod.bean.Class; import com.github.catvod.bean.Class;
import com.github.catvod.bean.Filter; import com.github.catvod.bean.Filter;
import com.github.catvod.bean.Result; import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod; import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider; import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp; import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils; import com.github.catvod.utils.Utils;
import okhttp3.FormBody;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import java.util.*; import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import okhttp3.FormBody;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
/** /**
* @author zhixc * @author zhixc
* 新版6V电影网 * 新版6V电影网
*/ */
public class Xb6v extends Spider { public class Xb6v extends Spider {
private final String siteURL = "http://www.xb6v.com"; private final String siteUrl = "http://www.xb6v.com";
private String nextSearchURLPrefix; private String nextSearchUrlPrefix;
private String nextSearchURLSuffix; private String nextSearchUrlSuffix;
private Map<String, String> getHeader() { private Map<String, String> getHeader() {
Map<String, String> header = new HashMap<>(); Map<String, String> header = new HashMap<>();
header.put("User-Agent", Utils.CHROME); header.put("User-Agent", Utils.CHROME);
header.put("Referer", siteURL + "/"); header.put("Referer", siteUrl + "/");
return header; return header;
} }
@ -48,7 +54,7 @@ public class Xb6v extends Spider {
@Override @Override
public String homeContent(boolean filter) throws Exception { public String homeContent(boolean filter) throws Exception {
List<Class> classes = new ArrayList<>(); List<Class> classes = new ArrayList<>();
String html = OkHttp.string(siteURL, getHeader()); String html = OkHttp.string(siteUrl, getHeader());
Document doc = Jsoup.parse(html); Document doc = Jsoup.parse(html);
Elements elements = doc.select("#menus > li > a"); Elements elements = doc.select("#menus > li > a");
LinkedHashMap<String, List<Filter>> filters = new LinkedHashMap<>(); LinkedHashMap<String, List<Filter>> filters = new LinkedHashMap<>();
@ -72,6 +78,10 @@ public class Xb6v extends Spider {
return Result.string(classes, parseVodListFromDoc(doc), filters); return Result.string(classes, parseVodListFromDoc(doc), filters);
} }
private List<Vod> parseVodListFromDoc(String html) {
return parseVodListFromDoc(Jsoup.parse(html));
}
private List<Vod> parseVodListFromDoc(Document doc) { private List<Vod> parseVodListFromDoc(Document doc) {
Elements items = doc.select("#post_container .post_hover"); Elements items = doc.select("#post_container .post_hover");
List<Vod> list = new ArrayList<>(); List<Vod> list = new ArrayList<>();
@ -88,14 +98,10 @@ public class Xb6v extends Spider {
@Override @Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws Exception { public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws Exception {
HashMap<String, String> ext = new HashMap<>(); String cateId = extend.get("cateId") == null ? "" : extend.get("cateId");
if (extend != null && extend.size() > 0) { String cateUrl = siteUrl + tid + cateId;
ext.putAll(extend); if (!pg.equals("1")) cateUrl += "index_" + pg + ".html";
} String html = OkHttp.string(cateUrl, getHeader());
String cateId = ext.get("cateId") == null ? "" : ext.get("cateId");
String cateURL = siteURL + tid + cateId;
if (!pg.equals("1")) cateURL += "index_" + pg + ".html";
String html = OkHttp.string(cateURL, getHeader());
Document doc = Jsoup.parse(html); Document doc = Jsoup.parse(html);
String href = doc.select(".pagination > a").last().attr("href"); String href = doc.select(".pagination > a").last().attr("href");
int page = Integer.parseInt(pg); int page = Integer.parseInt(pg);
@ -109,8 +115,8 @@ public class Xb6v extends Spider {
@Override @Override
public String detailContent(List<String> ids) throws Exception { public String detailContent(List<String> ids) throws Exception {
String vodId = ids.get(0); String vodId = ids.get(0);
String detailURL = siteURL + vodId; String detailUrl = siteUrl + vodId;
String html = OkHttp.string(detailURL, getDetailHeader()); String html = OkHttp.string(detailUrl, getDetailHeader());
Document doc = Jsoup.parse(html); Document doc = Jsoup.parse(html);
Elements sourceList = doc.select("#post_content"); Elements sourceList = doc.select("#post_content");
@ -121,10 +127,10 @@ public class Xb6v extends Spider {
Elements aList = source.select("table a"); Elements aList = source.select("table a");
List<String> vodItems = new ArrayList<>(); List<String> vodItems = new ArrayList<>();
for (Element a : aList) { for (Element a : aList) {
String episodeURL = a.attr("href"); String episodeUrl = a.attr("href");
String episodeName = a.text(); String episodeName = a.text();
if (!episodeURL.toLowerCase().startsWith("magnet")) continue; if (!episodeUrl.toLowerCase().startsWith("magnet")) continue;
vodItems.add(episodeName + "$" + episodeURL); vodItems.add(episodeName + "$" + episodeUrl);
} }
if (vodItems.size() > 0) { if (vodItems.size() > 0) {
i++; i++;
@ -161,22 +167,15 @@ public class Xb6v extends Spider {
vod.setVodActor(actor); vod.setVodActor(actor);
vod.setVodDirector(director); vod.setVodDirector(director);
vod.setVodContent(description); vod.setVodContent(description);
if (playMap.size() > 0) {
vod.setVodPlayFrom(TextUtils.join("$$$", playMap.keySet())); vod.setVodPlayFrom(TextUtils.join("$$$", playMap.keySet()));
vod.setVodPlayUrl(TextUtils.join("$$$", playMap.values())); vod.setVodPlayUrl(TextUtils.join("$$$", playMap.values()));
}
return Result.string(vod); return Result.string(vod);
} }
private String getStrByRegex(Pattern pattern, String str) { private String getStrByRegex(Pattern pattern, String str) {
try {
Matcher matcher = pattern.matcher(str); Matcher matcher = pattern.matcher(str);
if (matcher.find()) { if (matcher.find()) return matcher.group(1).trim();
return matcher.group(1).trim();
}
} catch (Exception e) {
SpiderDebug.log(e);
}
return ""; return "";
} }
@ -208,8 +207,7 @@ public class Xb6v extends Spider {
@Override @Override
public String searchContent(String key, boolean quick, String pg) throws Exception { public String searchContent(String key, boolean quick, String pg) throws Exception {
String searchURL = siteURL + "/e/search/index.php"; String searchUrl = siteUrl + "/e/search/index.php";
String html;
if (pg.equals("1")) { if (pg.equals("1")) {
RequestBody formBody = new FormBody.Builder() RequestBody formBody = new FormBody.Builder()
.add("show", "title") .add("show", "title")
@ -220,30 +218,22 @@ public class Xb6v extends Spider {
.add("submit", "") .add("submit", "")
.addEncoded("keyboard", key) .addEncoded("keyboard", key)
.build(); .build();
Request request = new Request.Builder() Request request = new Request.Builder().url(searchUrl)
.url(searchURL)
.addHeader("User-Agent", Utils.CHROME) .addHeader("User-Agent", Utils.CHROME)
.addHeader("Origin", siteURL) .addHeader("Origin", siteUrl)
.addHeader("Referer", siteURL + "/") .addHeader("Referer", siteUrl + "/")
.post(formBody) .post(formBody)
.build(); .build();
Response response = OkHttp.client().newCall(request).execute(); Response response = OkHttp.client().newCall(request).execute();
if (response.body() == null) return ""; String[] split = String.valueOf(response.request().url()).split("\\?searchid=");
initNextSearchURL(response); nextSearchUrlPrefix = split[0] + "index.php?page=";
html = response.body().string(); nextSearchUrlSuffix = "&searchid=" + split[1];
response.close(); return Result.string(parseVodListFromDoc(response.body().string()));
} else { } else {
int page = Integer.parseInt(pg) - 1; int page = Integer.parseInt(pg) - 1;
searchURL = nextSearchURLPrefix + page + nextSearchURLSuffix; searchUrl = nextSearchUrlPrefix + page + nextSearchUrlSuffix;
html = OkHttp.string(searchURL, getHeader()); return Result.string(parseVodListFromDoc(OkHttp.string(searchUrl, getHeader())));
} }
return Result.string(parseVodListFromDoc(Jsoup.parse(html)));
}
private void initNextSearchURL(Response response) {
String[] split = String.valueOf(response.request().url()).split("\\?searchid=");
nextSearchURLPrefix = split[0] + "index.php?page=";
nextSearchURLSuffix = "&searchid=" + split[1];
} }
@Override @Override

BIN
jar/custom_spider.jar
View File

Binary file not shown.

2
jar/custom_spider.jar.md5
View File

@ -1 +1 @@
386aa45681818befd51ff5008f88f4e5 a4a22ad4042d5ff3444ed4b66f41a3a4