dongge
/
CatVodSpider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Bili

This commit is contained in:
FongMi 2022-09-06 11:34:54 +08:00
parent feaecc88c1
commit 59dd71c015
13 changed files with 232 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/src/main/AndroidManifest.xml
View File

@ -2,6 +2,8 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.github.catvod">
<uses-permission android:name="android.permission.INTERNET" />
<application
android:icon="@mipmap/ic_launcher"
android:label="@string/app_name">

10
app/src/main/java/com/github/catvod/bean/Class.java
View File

@ -1,6 +1,11 @@
package com.github.catvod.bean;
import com.google.gson.Gson;
import com.google.gson.annotations.SerializedName;
import com.google.gson.reflect.TypeToken;
import java.lang.reflect.Type;
import java.util.List;
public class Class {
@ -9,6 +14,11 @@ public class Class {
@SerializedName("type_name")
private String typeName;
public static List<Class> arrayFrom(String str) {
Type listType = new TypeToken<List<Class>>() {}.getType();
return new Gson().fromJson(str, listType);
}
public Class(int typeId, String typeName) {
this(String.valueOf(typeId), typeName);
}

4
app/src/main/java/com/github/catvod/bean/Result.java
View File

@ -39,6 +39,10 @@ public class Result {
return Result.get().classes(classes).vod(list).filters(filters).string();
}
public static String string(List<Class> classes, JSONObject filters) {
return Result.get().classes(classes).filters(filters).string();
}
public static String string(List<Class> classes, List<Vod> list) {
return Result.get().classes(classes).vod(list).string();
}

14
app/src/main/java/com/github/catvod/crawler/Spider.java
View File

@ -2,8 +2,6 @@ package com.github.catvod.crawler;
import android.content.Context;
import org.json.JSONException;
import java.util.HashMap;
import java.util.List;
@ -16,27 +14,27 @@ public abstract class Spider {
init(context);
}
public String homeContent(boolean filter) throws JSONException {
public String homeContent(boolean filter) throws Exception {
return "";
}
public String homeVideoContent() throws JSONException {
public String homeVideoContent() throws Exception {
return "";
}
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws JSONException {
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws Exception {
return "";
}
public String detailContent(List<String> ids) throws JSONException {
public String detailContent(List<String> ids) throws Exception {
return "";
}
public String searchContent(String key, boolean quick) throws JSONException {
public String searchContent(String key, boolean quick) throws Exception {
return "";
}
public String playerContent(String flag, String id, List<String> vipFlags) throws JSONException {
public String playerContent(String flag, String id, List<String> vipFlags) throws Exception {
return "";
}
}

130
app/src/main/java/com/github/catvod/spider/Bili.java Normal file
View File

@ -0,0 +1,130 @@
package com.github.catvod.spider;
import android.content.Context;
import android.text.TextUtils;
import com.github.catvod.bean.Class;
import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider;
import com.github.catvod.net.OkHttpUtil;
import com.github.catvod.utils.Misc;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
/**
* @author ColaMint & FongMi
*/
public class Bili extends Spider {
private final String url = "https://www.bilibili.com";
private HashMap<String, String> header;
private JSONObject ext;
private void initHeader() {
header = new HashMap<>();
header.put("User-Agent", Misc.CHROME);
HashMap<String, List<String>> respHeaderMap = new HashMap<>();
OkHttpUtil.string(url, header, respHeaderMap);
for (String text : Objects.requireNonNull(respHeaderMap.get("set-cookie"))) {
if (!text.contains("buvid3")) continue;
header.put("cookie", text.split(";")[0]);
header.put("Referer", url);
break;
}
}
@Override
public void init(Context context, String extend) {
try {
if (extend.startsWith("http")) extend = OkHttpUtil.string(extend);
ext = new JSONObject(extend);
initHeader();
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public String homeContent(boolean filter) throws Exception {
return Result.string(Class.arrayFrom(ext.getJSONArray("classes").toString()), ext.getJSONObject("filter"));
}
@Override
public String homeVideoContent() throws Exception {
return categoryContent("窗 白噪音", "1", true, new HashMap<>());
}
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws Exception {
String duration = extend.containsKey("duration") ? extend.get("duration") : "0";
String url = "https://api.bilibili.com/x/web-interface/search/type?search_type=video&keyword=" + URLEncoder.encode(tid) + "&duration=" + duration + "&page=" + pg;
JSONObject resp = new JSONObject(OkHttpUtil.string(url, header));
System.out.println(resp.toString());
JSONArray result = resp.getJSONObject("data").getJSONArray("result");
List<Vod> list = new ArrayList<>();
for (int i = 0; i < result.length(); ++i) {
JSONObject object = result.getJSONObject(i);
String pic = object.getString("pic");
Vod vod = new Vod();
vod.setVodId(object.getString("bvid"));
vod.setVodName(Jsoup.parse(object.getString("title")).text());
vod.setVodRemarks(object.getString("duration").split(":")[0] + "分鐘");
vod.setVodPic(pic.startsWith("//") ? "https:" + pic : pic);
list.add(vod);
}
return Result.string(list);
}
@Override
public String detailContent(List<String> ids) throws Exception {
String bvid = ids.get(0);
String bvid2aidUrl = "https://api.bilibili.com/x/web-interface/archive/stat?bvid=" + bvid;
JSONObject bvid2aidResp = new JSONObject(OkHttpUtil.string(bvid2aidUrl, header));
String aid = bvid2aidResp.getJSONObject("data").getLong("aid") + "";
String detailUrl = "https://api.bilibili.com/x/web-interface/view?aid=" + aid;
JSONObject detailResp = new JSONObject(OkHttpUtil.string(detailUrl, header));
JSONObject detailData = detailResp.getJSONObject("data");
List<String> playlist = new ArrayList<>();
JSONArray pages = detailData.getJSONArray("pages");
for (int i = 0; i < pages.length(); ++i) {
JSONObject page = pages.getJSONObject(i);
String title = page.getString("part").replace("$", "_").replace("#", "_");
playlist.add(title + "$" + aid + "+ " + page.getLong("cid"));
}
Vod vod = new Vod();
vod.setVodId(bvid);
vod.setVodName(detailData.getString("title"));
vod.setVodPic(detailData.getString("pic"));
vod.setTypeName(detailData.getString("tname"));
vod.setVodRemarks(detailData.getLong("duration") / 60 + "分鐘");
vod.setVodContent(detailData.getString("desc"));
vod.setVodPlayFrom("B站");
vod.setVodPlayUrl(TextUtils.join("#", playlist));
return Result.string(vod);
}
@Override
public String searchContent(String key, boolean quick) throws Exception {
return categoryContent(key, "1", true, new HashMap<>());
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) throws Exception {
String[] ids = id.split("\\+");
String aid = ids[0];
String cid = ids[1];
String url = "https://api.bilibili.com/x/player/playurl?avid=" + aid + "&cid= " + cid + "&qn=112";
JSONObject resp = new JSONObject(OkHttpUtil.string(url, header));
url = resp.getJSONObject("data").getJSONArray("durl").getJSONObject(0).getString("url");
return Result.get().url(url).header(header).string();
}
}

63
app/src/main/java/com/github/catvod/spider/Biubiu.java
View File

@ -21,6 +21,9 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author 小黄瓜
*/
public class Biubiu extends Spider {
private JSONObject rule = null;
@ -62,17 +65,17 @@ public class Biubiu extends Spider {
@Override
public String homeVideoContent() {
if (getRuleVal("shouye").equals("1")) return "";
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
String[] fenleis = getRuleVal("fenlei", "").split("#");
for (String fenlei : fenleis) {
String[] info = fenlei.split("\\$");
Result result = category(info[1], "1");
for (int i = 0; i < result.getList().size(); i++) {
videos.add(result.getList().get(i));
if (videos.size() >= 30) break;
list.add(result.getList().get(i));
if (list.size() >= 30) break;
}
}
return Result.string(videos);
return Result.string(list);
}
private Result category(String tid, String pg) {
@ -87,7 +90,7 @@ public class Biubiu extends Spider {
}
String jiequshuzuqian = getRuleVal("jiequshuzuqian");
String jiequshuzuhou = getRuleVal("jiequshuzuhou");
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
ArrayList<String> jiequContents = subContent(parseContent, jiequshuzuqian, jiequshuzuhou);
for (int i = 0; i < jiequContents.size(); i++) {
try {
@ -100,12 +103,12 @@ public class Biubiu extends Spider {
vod.setVodId(title + "$$$" + pic + "$$$" + link);
vod.setVodName(title);
vod.setVodPic(pic);
videos.add(vod);
list.add(vod);
} catch (Exception e) {
break;
}
}
return Result.get().vod(videos);
return Result.get().vod(list);
}
@Override
@ -152,14 +155,14 @@ public class Biubiu extends Spider {
}
}
Vod video = new Vod();
video.setVodId(ids.get(0));
video.setVodName(idInfo[0]);
video.setVodPic(idInfo[1]);
Vod vod = new Vod();
vod.setVodId(ids.get(0));
vod.setVodName(idInfo[0]);
vod.setVodPic(idInfo[1]);
for (int i = 0; i < playList.size(); i++) playFrom.add("播放列表" + (i + 1));
video.setVodPlayFrom(TextUtils.join("$$$", playFrom));
video.setVodPlayUrl(TextUtils.join("$$$", playList));
return Result.string(video);
vod.setVodPlayFrom(TextUtils.join("$$$", playFrom));
vod.setVodPlayUrl(TextUtils.join("$$$", playList));
return Result.string(vod);
}
@Override
@ -174,21 +177,21 @@ public class Biubiu extends Spider {
String webUrlTmp = getRuleVal("url") + getRuleVal("sousuoqian") + key + getRuleVal("sousuohou");
String webUrl = webUrlTmp.split(";")[0];
String webContent = webUrlTmp.contains(";post") ? fetchPost(webUrl) : fetch(webUrl);
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
if (ssmoshiJson) {
JSONObject data = new JSONObject(webContent);
JSONArray vodArray = data.getJSONArray("list");
for (int j = 0; j < vodArray.length(); j++) {
JSONObject vod = vodArray.getJSONObject(j);
String name = vod.optString(getRuleVal("jsname")).trim();
String id = vod.optString(getRuleVal("jsid")).trim();
String pic = vod.optString(getRuleVal("jspic")).trim();
JSONObject object = vodArray.getJSONObject(j);
String name = object.optString(getRuleVal("jsname")).trim();
String id = object.optString(getRuleVal("jsid")).trim();
String pic = object.optString(getRuleVal("jspic")).trim();
pic = Misc.fixUrl(webUrl, pic);
Vod video = new Vod();
video.setVodId(name + "$$$" + pic + "$$$" + getRuleVal("sousuohouzhui") + id);
video.setVodName(name);
video.setVodPic(pic);
videos.add(video);
Vod vod = new Vod();
vod.setVodId(name + "$$$" + pic + "$$$" + getRuleVal("sousuohouzhui") + id);
vod.setVodName(name);
vod.setVodPic(pic);
list.add(vod);
}
} else {
String parseContent = webContent;
@ -208,18 +211,18 @@ public class Biubiu extends Spider {
String pic = subContent(jiequContent, getRuleVal("sstupianqian"), getRuleVal("sstupianhou")).get(0);
pic = Misc.fixUrl(webUrl, pic);
String link = subContent(jiequContent, getRuleVal("sslianjieqian"), getRuleVal("sslianjiehou")).get(0);
Vod video = new Vod();
video.setVodId(title + "$$$" + pic + "$$$" + link);
video.setVodName(title);
video.setVodPic(pic);
videos.add(video);
Vod vod = new Vod();
vod.setVodId(title + "$$$" + pic + "$$$" + link);
vod.setVodName(title);
vod.setVodPic(pic);
list.add(vod);
} catch (Exception e) {
e.printStackTrace();
break;
}
}
}
return Result.string(videos);
return Result.string(list);
}
private String fetch(String webUrl) {

18
app/src/main/java/com/github/catvod/spider/Dm84.java
View File

@ -51,7 +51,7 @@ public class Dm84 extends Spider {
@Override
public String homeContent(boolean filter) {
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
List<Class> classes = new ArrayList<>();
LinkedHashMap<String, List<Filter>> filters = new LinkedHashMap<>();
Document doc = Jsoup.parse(OkHttpUtil.string(siteUrl, getHeaders()));
@ -77,14 +77,14 @@ public class Dm84 extends Spider {
String name = element.select("a.title").text();
String remark = element.select("span.desc").text();
String id = url.split("/")[2];
videos.add(new Vod(id, name, img, remark));
list.add(new Vod(id, name, img, remark));
}
return Result.string(classes, videos, filters);
return Result.string(classes, list, filters);
}
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
if (extend.get("type") == null) extend.put("type", "");
if (extend.get("year") == null) extend.put("year", "");
if (extend.get("by") == null) extend.put("by", "time");
@ -99,9 +99,9 @@ public class Dm84 extends Spider {
String name = element.select("a.title").text();
String remark = element.select("span.desc").text();
String id = url.split("/")[2];
videos.add(new Vod(id, name, img, remark));
list.add(new Vod(id, name, img, remark));
}
return Result.string(videos);
return Result.string(list);
}
@Override
@ -154,7 +154,7 @@ public class Dm84 extends Spider {
@Override
public String searchContent(String key, boolean quick) {
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
String target = siteUrl.concat("/s----------.html?wd=").concat(key);
Document doc = Jsoup.parse(OkHttpUtil.string(target, getHeaders()));
for (Element element : doc.select("div.item")) {
@ -163,9 +163,9 @@ public class Dm84 extends Spider {
String name = element.select("a.title").text();
String remark = element.select("span.desc").text();
String id = url.split("/")[2];
videos.add(new Vod(id, name, img, remark));
list.add(new Vod(id, name, img, remark));
}
return Result.string(videos);
return Result.string(list);
}
@Override

20
app/src/main/java/com/github/catvod/spider/XPath.java
View File

@ -53,7 +53,7 @@ public class XPath extends Spider {
@Override
public String homeContent(boolean filter) throws JSONException {
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
List<Class> classes = new ArrayList<>();
if (rule.getCateManual().size() > 0) {
Set<String> keys = rule.getCateManual().keySet();
@ -90,10 +90,10 @@ public class XPath extends Spider {
SpiderDebug.log(e);
}
}
videos.add(new Vod(id, name, pic, mark));
list.add(new Vod(id, name, pic, mark));
}
}
return Result.string(classes, videos, rule.getFilter());
return Result.string(classes, list, rule.getFilter());
}
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
@ -103,7 +103,7 @@ public class XPath extends Spider {
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
String webUrl = categoryUrl(tid, pg, filter, extend);
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
JXDocument doc = JXDocument.create(fetch(webUrl));
List<JXNode> vodNodes = doc.selN(rule.getCateVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
@ -123,9 +123,9 @@ public class XPath extends Spider {
SpiderDebug.log(e);
}
}
videos.add(new Vod(id, name, pic, mark));
list.add(new Vod(id, name, pic, mark));
}
return Result.string(videos);
return Result.string(list);
}
@Override
@ -268,7 +268,7 @@ public class XPath extends Spider {
if (rule.getSearchUrl().isEmpty()) return "";
String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key));
String webContent = fetch(webUrl);
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
if (rule.getSearchVodNode().startsWith("json:")) {
String[] node = rule.getSearchVodNode().substring(5).split(">");
JSONObject data = new JSONObject(webContent);
@ -286,7 +286,7 @@ public class XPath extends Spider {
pic = Misc.fixUrl(webUrl, pic);
String mark = vod.optString(rule.getSearchVodMark()).trim();
mark = rule.getSearchVodMarkR(mark);
videos.add(new Vod(id, name, pic, mark));
list.add(new Vod(id, name, pic, mark));
}
} else {
data = data.getJSONObject(node[i]);
@ -312,10 +312,10 @@ public class XPath extends Spider {
SpiderDebug.log(e);
}
}
videos.add(new Vod(id, name, pic, mark));
list.add(new Vod(id, name, pic, mark));
}
}
return Result.string(videos);
return Result.string(list);
}
protected void loadRuleExt(String json) {

18
app/src/main/java/com/github/catvod/spider/wait/Anime1.java
View File

@ -120,14 +120,14 @@ public class Anime1 extends Spider {
classes.add(new Class(i, "更早"));
}
}
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
for (int i = 0; i < 10; i++) {
String id = srcArray.getJSONObject(i).getString("link");
String name = srcArray.getJSONObject(i).getString("name");
String mark = srcArray.getJSONObject(i).getString("hit");
videos.add(new Vod(id, name, vodPic, mark));
list.add(new Vod(id, name, vodPic, mark));
}
return Result.string(classes, videos);
return Result.string(classes, list);
} catch (Exception e) {
SpiderDebug.log(e);
return "";
@ -166,14 +166,14 @@ public class Anime1 extends Spider {
if (yearint < yy) array.put(subobj);
}
}
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
for (int i = 0; i < array.length(); i++) {
String id = array.getJSONObject(i).getString("link");
String name = array.getJSONObject(i).getString("name");
String mark = array.getJSONObject(i).getString("hit");
videos.add(new Vod(id, name, vodPic, mark));
list.add(new Vod(id, name, vodPic, mark));
}
return Result.string(videos);
return Result.string(list);
} catch (Exception e) {
SpiderDebug.log(e);
return "";
@ -355,17 +355,17 @@ public class Anime1 extends Spider {
String kw = srobj.getString("name");
if (kw.contains(key)) searchResult.put(srobj);
}
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
if (searchResult.length() > 0) {
int ch = Math.min(searchResult.length(), 10);
for (int i = 0; i < ch; i++) {
String id = searchResult.getJSONObject(i).getString("link");
String name = searchResult.getJSONObject(i).getString("name");
String mark = searchResult.getJSONObject(i).getString("hit");
videos.add(new Vod(id, name, vodPic, mark));
list.add(new Vod(id, name, vodPic, mark));
}
}
return Result.string(videos);
return Result.string(list);
} catch (Exception e) {
SpiderDebug.log(e);
return "";

36
app/src/main/java/com/github/catvod/spider/wait/Tangrenjie.java
View File

@ -77,19 +77,19 @@ public class Tangrenjie extends Spider {
}
}
Element homeList = doc.select("div.cbox1").get(1);
Elements list = homeList.select("ul.vodlist li.vodlist_item");
List<Vod> videos = new ArrayList<>();
for (int i = 0; i < list.size(); i++) {
Element vod = list.get(i);
elements = homeList.select("ul.vodlist li.vodlist_item");
List<Vod> list = new ArrayList<>();
for (int i = 0; i < elements.size(); i++) {
Element vod = elements.get(i);
String title = vod.selectFirst("a.vodlist_thumb").attr("title");
String cover = siteUrl + vod.selectFirst("a.vodlist_thumb").attr("data-original");
String remark = vod.select("span.pic_text b").text();
Matcher matcher = regexVid.matcher(vod.selectFirst("a.vodlist_thumb").attr("href"));
if (!matcher.find()) continue;
String id = "1=" + matcher.group(1);
videos.add(new Vod(id, title, cover, remark));
list.add(new Vod(id, title, cover, remark));
}
return Result.string(classes, videos, filterConfig);
return Result.string(classes, list, filterConfig);
}
@Override
@ -110,21 +110,21 @@ public class Tangrenjie extends Spider {
url.append("/page/").append(pg).append(".html");
String html = OkHttpUtil.string(url.toString(), getHeaders());
Document doc = Jsoup.parse(html);
List<Vod> videos = new ArrayList<>();
List<Vod> list = new ArrayList<>();
if (!html.contains("没有找到您想要的结果哦")) {
Elements list = doc.select("ul.vodlist li.vodlist_item");
for (int i = 0; i < list.size(); i++) {
Element vod = list.get(i);
Elements elements = doc.select("ul.vodlist li.vodlist_item");
for (int i = 0; i < elements.size(); i++) {
Element vod = elements.get(i);
String title = vod.selectFirst("a.vodlist_thumb").attr("title");
String cover = siteUrl + vod.selectFirst("a.vodlist_thumb").attr("data-original");
String remark = vod.select("span.pic_text b").text();
Matcher matcher = regexVid.matcher(vod.selectFirst("a.vodlist_thumb").attr("href"));
if (!matcher.find()) continue;
String id = tid + "=" + matcher.group(1);
videos.add(new Vod(id, title, cover, remark));
list.add(new Vod(id, title, cover, remark));
}
}
return Result.string(videos);
return Result.string(list);
}
@Override
@ -296,10 +296,10 @@ public class Tangrenjie extends Spider {
String url = siteUrl + "/vod/search.html?wd=" + URLEncoder.encode(key) + "&submit=";
String html = OkHttpUtil.string(url, getHeaders());
Document doc = Jsoup.parse(html);
List<Vod> videos = new ArrayList<>();
Elements list = doc.select("li.searchlist_item");
for (int i = 0; i < list.size(); i++) {
Element vod = list.get(i);
List<Vod> list = new ArrayList<>();
Elements elements = doc.select("li.searchlist_item");
for (int i = 0; i < elements.size(); i++) {
Element vod = elements.get(i);
String classes = vod.selectFirst("h4.vodlist_title > a >span").text();
String tid = "";
switch (classes) {
@ -322,8 +322,8 @@ public class Tangrenjie extends Spider {
Matcher matcher = regexVid.matcher(vod.selectFirst("a.vodlist_thumb").attr("href"));
if (!matcher.find()) continue;
String id = tid + "=" + matcher.group(1);
videos.add(new Vod(id, title, cover, remark));
list.add(new Vod(id, title, cover, remark));
}
return Result.string(videos);
return Result.string(list);
}
}

0
buildAndGenJar.bat → build.bat
View File

BIN
jar/custom_spider.jar
View File

Binary file not shown.

2
jar/custom_spider.jar.md5
View File

@ -1 +1 @@
f49a9466967b43602e3252dac1249d99
8099b863f81113740c24c05d7feb3851