blankie
/
hanimers
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update dependencies

Browse Source
master
blank X 9 months ago
parent c706dce677
commit f4ee85becb
Signed by: blankie
GPG Key ID: CC15FC822C7F61F5
10 changed files with 489 additions and 460 deletions
Show Stats Download Patch File Download Diff File

546
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

8
Cargo.toml
Unescape View File

@ -1,6 +1,6 @@
[package]
name = "hanimers"
version = "0.1.8"
version = "0.1.9"
authors = ["blank X <theblankx@protonmail.com>"]
edition = "2018"
@ -10,10 +10,10 @@ edition = "2018"
lto = true
[dependencies]
tokio = { version = "1.4", features = ["rt"] }
tokio = { version = "1.18", features = ["rt"] }
reqwest = "0.11"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
quick-xml = "0.22"
clap = { version = "2.33", default-features = false }
regex = "1.4"
clap = { version = "3.1", features = ["std"], default-features = false }
regex = "1.5"

79
src/commands/download.rs
Unescape View File

@ -1,16 +1,16 @@
use crate::utils;
use clap::ArgMatches;
use reqwest::redirect::Policy;
use reqwest::Client;
use std::collections::HashMap;
use std::fs::rename;
use std::path::Path;
use std::process::{exit, Command};
use std::collections::HashMap;
use clap::ArgMatches;
use reqwest::Client;
use reqwest::redirect::Policy;
const MAX_DOWNLOAD_ATTEMPTS: i32 = 5;
pub async fn download(arg_m: &ArgMatches<'_>) {
pub async fn download(arg_m: &ArgMatches) {
let print_only = arg_m.is_present("print");
let resolution = arg_m.value_of("resolution");
let ids = arg_m.values_of("id").unwrap().collect::<Vec<_>>();
@ -42,18 +42,20 @@ pub async fn download(arg_m: &ArgMatches<'_>) {
for server in hentai_info.state.data.video.videos_manifest.servers {
let mut to_hashmap = match server.is_permanent {
true => perm_urls.clone(),
false => temp_urls.clone()
false => temp_urls.clone(),
};
for stream in server.streams {
if stream.url.is_empty() {
continue;
}
if server.is_permanent && Some(stream.height.as_str()) == resolution {
if server.is_permanent && Some(stream.height.as_str()) == resolution
{
download_url = Some((stream.url, stream.filesize_mbs));
break;
}
if !to_hashmap.contains_key(&stream.height) {
to_hashmap.insert(stream.height, (stream.url, stream.filesize_mbs));
to_hashmap
.insert(stream.height, (stream.url, stream.filesize_mbs));
};
}
if download_url.is_some() {
@ -61,15 +63,18 @@ pub async fn download(arg_m: &ArgMatches<'_>) {
}
match server.is_permanent {
true => perm_urls.extend(to_hashmap),
false => temp_urls.extend(to_hashmap)
false => temp_urls.extend(to_hashmap),
};
}
if download_url.is_none() {
if resolution.is_some() {
download_url = temp_urls.get(resolution.unwrap()).map(|i| (i.0.to_string(), i.1));
download_url = temp_urls
.get(resolution.unwrap())
.map(|i| (i.0.to_string(), i.1));
}
if download_url.is_none() {
download_url = magic_thing(perm_urls).or_else(|| { magic_thing(temp_urls) });
download_url =
magic_thing(perm_urls).or_else(|| magic_thing(temp_urls));
if download_url.is_none() {
eprintln!("Failed to get {}: cannot get download url", id);
return_fail = true;
@ -84,8 +89,27 @@ pub async fn download(arg_m: &ArgMatches<'_>) {
let mut fail_dl = true;
let tmp_filename = format!("{}.tmp", &slug);
for i in 0..MAX_DOWNLOAD_ATTEMPTS {
eprintln!("Downloading {} ({}MB, attempt {})", &filename, filesize_mbs, i);
match Command::new("ffmpeg").args(&["-v", "warning", "-stats", "-nostdin", "-y", "-i", &download_url, "-c", "copy", "-f", "matroska", &tmp_filename]).spawn() {
eprintln!(
"Downloading {} ({}MB, attempt {})",
&filename, filesize_mbs, i
);
match Command::new("ffmpeg")
.args(&[
"-v",
"warning",
"-stats",
"-nostdin",
"-y",
"-i",
&download_url,
"-c",
"copy",
"-f",
"matroska",
&tmp_filename,
])
.spawn()
{
Ok(mut child) => {
match child.wait() {
Ok(exit_status) => {
@ -93,16 +117,27 @@ pub async fn download(arg_m: &ArgMatches<'_>) {
fail_dl = false;
match rename(&tmp_filename, &filename) {
Ok(_) => (),
Err(err) => eprintln!("Failed to rename {} to {} due to {}", &tmp_filename, &filename, err)
Err(err) => eprintln!(
"Failed to rename {} to {} due to {}",
&tmp_filename, &filename, err
),
};
break;
}
eprintln!("ffmpeg exited with {:?}", exit_status.code());
},
Err(err) => eprintln!("Failed to wait on ffmpeg process due to {}", err)
eprintln!(
"ffmpeg exited with {:?}",
exit_status.code()
);
}
Err(err) => eprintln!(
"Failed to wait on ffmpeg process due to {}",
err
),
};
},
Err(err) => eprintln!("Failed to spawn ffmpeg process due to {}", err)
}
Err(err) => {
eprintln!("Failed to spawn ffmpeg process due to {}", err)
}
};
}
if fail_dl {
@ -110,13 +145,13 @@ pub async fn download(arg_m: &ArgMatches<'_>) {
return_fail = true;
}
}
},
}
None => {
eprintln!("Failed to get {}: does not exist", id);
return_fail = true;
}
};
},
}
Err(err) => {
eprintln!("Failed to get {}: {}", id, err);
return_fail = true;
@ -147,7 +182,7 @@ fn magic_thing(map: HashMap<String, (String, i32)>) -> Option<(String, i32)> {
keys.sort();
match keys.pop() {
Some(key) => map.get(key.as_str()).map(|i| i.clone()),
None => None
None => None,
}
}
}

8
src/commands/mod.rs
Unescape View File

@ -1,6 +1,6 @@
mod view;
mod search;
mod download;
pub use view::view;
pub use search::{search, AVALIABLE_TAGS};
mod search;
mod view;
pub use download::download;
pub use search::{search, AVALIABLE_TAGS};
pub use view::view;

81
src/commands/search.rs
Unescape View File

@ -1,17 +1,88 @@
use crate::utils;
use std::process::exit;
use clap::ArgMatches;
use reqwest::Client;
use std::process::exit;
pub const AVALIABLE_TAGS: &[&str] = &["3D", "Ahegao", "Anal", "BDSM", "Big Boobs", "Blow Job", "Bondage", "Boob Job", "Censored", "Comedy", "Cosplay", "Creampie", "Dark Skin", "Facial", "Fantasy", "Filmed", "Foot Job", "Futanari", "Gangbang", "Glasses", "Hand Job", "Harem", "HD", "Horror", "Incest", "Inflation", "Lactation", "Loli", "Maid", "Masturbation", "Milf", "Mind Break", "Mind Control", "Monster", "Nekomimi", "NTR", "Nurse", "Orgy", "Plot", "POV", "Pregnant", "Public Sex", "Rape", "Reverse Rape", "Rimjob", "Scat", "School Girl", "Shota", "Softcore", "Swimsuit", "Teacher", "Tentacle", "Threesome", "Toys", "Trap", "Tsundere", "Ugly Bastard", "Uncensored", "Vanilla", "Virgin", "Watersports", "X-Ray", "Yaoi", "Yuri"];
pub const AVALIABLE_TAGS: &[&str] = &[
"3D",
"Ahegao",
"Anal",
"BDSM",
"Big Boobs",
"Blow Job",
"Bondage",
"Boob Job",
"Censored",
"Comedy",
"Cosplay",
"Creampie",
"Dark Skin",
"Facial",
"Fantasy",
"Filmed",
"Foot Job",
"Futanari",
"Gangbang",
"Glasses",
"Hand Job",
"Harem",
"HD",
"Horror",
"Incest",
"Inflation",
"Lactation",
"Loli",
"Maid",
"Masturbation",
"Milf",
"Mind Break",
"Mind Control",
"Monster",
"Nekomimi",
"NTR",
"Nurse",
"Orgy",
"Plot",
"POV",
"Pregnant",
"Public Sex",
"Rape",
"Reverse Rape",
"Rimjob",
"Scat",
"School Girl",
"Shota",
"Softcore",
"Swimsuit",
"Teacher",
"Tentacle",
"Threesome",
"Toys",
"Trap",
"Tsundere",
"Ugly Bastard",
"Uncensored",
"Vanilla",
"Virgin",
"Watersports",
"X-Ray",
"Yaoi",
"Yuri",
];
pub async fn search(arg_m: &ArgMatches<'_>) {
pub async fn search(arg_m: &ArgMatches) {
let broad_search = arg_m.is_present("broad");
let tags = arg_m.values_of("tags").unwrap_or_default().collect();
let query = arg_m.values_of("query").unwrap_or_default().collect::<Vec<_>>().join(" ");
let query = arg_m
.values_of("query")
.unwrap_or_default()
.collect::<Vec<_>>()
.join(" ");
let query = query.trim();
let results = utils::search(Client::new(), query, tags, broad_search).await.unwrap();
let results = utils::search(Client::new(), query, tags, broad_search)
.await
.unwrap();
if results.is_empty() {
eprintln!("No results found");
exit(1);

31
src/commands/view.rs
Unescape View File

@ -1,12 +1,12 @@
use crate::utils;
use std::process::exit;
use clap::ArgMatches;
use reqwest::Client;
use reqwest::redirect::Policy;
use reqwest::Client;
use std::process::exit;
extern crate tokio;
pub async fn view(arg_m: &ArgMatches<'_>) {
pub async fn view(arg_m: &ArgMatches) {
let policy = Policy::custom(|attempt| {
if attempt.previous().len() > 10 {
attempt.error("too many redirects")
@ -17,14 +17,19 @@ pub async fn view(arg_m: &ArgMatches<'_>) {
}
});
let client = Client::builder().redirect(policy).build().unwrap();
let handles = arg_m.values_of("id").unwrap().map(|id| {
let cloned_client = client.clone();
let id = id.to_string();
let cid = id.clone();
(tokio::spawn(async move {
utils::get_hentai(cloned_client, &cid).await
}), id)
}).collect::<Vec<_>>();
let handles = arg_m
.values_of("id")
.unwrap()
.map(|id| {
let cloned_client = client.clone();
let id = id.to_string();
let cid = id.clone();
(
tokio::spawn(async move { utils::get_hentai(cloned_client, &cid).await }),
id,
)
})
.collect::<Vec<_>>();
let mut fail = false;
let mut one_done = false;
for handle in handles {
@ -49,7 +54,7 @@ pub async fn view(arg_m: &ArgMatches<'_>) {
println!("");
}
println!("{}", &hentai);
},
}
None => {
if one_done {
eprintln!("");
@ -58,7 +63,7 @@ pub async fn view(arg_m: &ArgMatches<'_>) {
fail = true;
}
};
},
}
Err(err) => {
if one_done {
eprintln!("");

60
src/main.rs
Unescape View File

@ -1,65 +1,65 @@
mod commands;
mod unescape;
mod structs;
mod unescape;
mod utils;
use clap::{App, AppSettings, Arg, SubCommand};
use clap::{Arg, Command};
extern crate tokio;
fn main() {
let matches = App::new("hanimers")
let matches = Command::new("hanimers")
.about("hanime.tv downloader in rust")
.version(env!("CARGO_PKG_VERSION"))
.setting(AppSettings::SubcommandRequiredElseHelp)
.subcommand_required(true)
.subcommand(
SubCommand::with_name("search")
Command::new("search")
.arg(
Arg::with_name("tags")
Arg::new("tags")
.long("tags")
.short("t")
.short('t')
.takes_value(true)
.use_delimiter(true)
.case_insensitive(true)
.possible_values(&commands::AVALIABLE_TAGS)
.use_value_delimiter(true)
.ignore_case(true)
.possible_values(commands::AVALIABLE_TAGS)
).arg(
Arg::with_name("broad")
Arg::new("broad")
.long("broad")
.short("b")
.short('b')
.help("More results, but less accurate. Videos will match if they contain any selected tag rather than all selected tags.")
).arg(
Arg::with_name("query")
Arg::new("query")
.takes_value(true)
.multiple(true)
.multiple_values(true)
.help("Search query")
)
)
.subcommand(
SubCommand::with_name("view")
.aliases(&["info", "show"])
Command::new("view")
.visible_aliases(&["info", "show"])
.arg(
Arg::with_name("id")
Arg::new("id")
.takes_value(true)
.multiple(true)
.multiple_values(true)
.required(true)
)
)
.subcommand(
SubCommand::with_name("download")
.alias("dl")
Command::new("download")
.visible_alias("dl")
.arg(
Arg::with_name("print")
Arg::new("print")
.long("print")
.short("p")
.short('p')
.help("Print the URL to download only")
).arg(
Arg::with_name("resolution")
Arg::new("resolution")
.long("resolution")
.short("r")
.short('r')
.help("Set preferred resolution")
.takes_value(true)
).arg(
Arg::with_name("id")
Arg::new("id")
.takes_value(true)
.multiple(true)
.multiple_values(true)
.required(true)
)
)
@ -70,9 +70,9 @@ fn main() {
.build()
.unwrap();
match matches.subcommand() {
("search", Some(sub_m)) => runtime.block_on(commands::search(sub_m)),
("view", Some(sub_m)) => runtime.block_on(commands::view(sub_m)),
("download", Some(sub_m)) => runtime.block_on(commands::download(sub_m)),
_ => panic!("AppSettings::SubcommandRequiredElseHelp do your job please")
Some(("search", sub_m)) => runtime.block_on(commands::search(sub_m)),
Some(("view", sub_m)) => runtime.block_on(commands::view(sub_m)),
Some(("download", sub_m)) => runtime.block_on(commands::download(sub_m)),
_ => unreachable!("subcommand_required do your job please"),
};
}

90
src/structs.rs
Unescape View File

@ -1,20 +1,20 @@
use std::fmt;
use std::marker::PhantomData;
use std::collections::HashMap;
use crate::unescape::unescape;
use quick_xml::events::Event;
use quick_xml::Reader;
use serde::de::{self, Visitor};
use serde::{Deserialize, Deserializer};
use quick_xml::Reader;
use quick_xml::events::Event;
use crate::unescape::unescape;
extern crate reqwest;
use std::collections::HashMap;
use std::fmt;
use std::marker::PhantomData;
extern crate quick_xml;
extern crate reqwest;
extern crate serde_json;
// contains more info that i won't add for search results, maybe non_exhaustive? not sure
#[derive(Deserialize, Debug)]
pub struct SearchResult {
pub id: i32,
pub name: String
pub name: String,
}
impl fmt::Display for SearchResult {
@ -25,28 +25,28 @@ impl fmt::Display for SearchResult {
#[derive(Deserialize, Debug)]
pub struct SearchResultMetadata {
pub hits: String
pub hits: String,
}
#[derive(Deserialize, Debug)]
pub struct HentaiInfo {
pub state: HentaiState
pub state: HentaiState,
}
#[derive(Deserialize, Debug)]
pub struct HentaiState {
pub data: HentaiData
pub data: HentaiData,
}
#[derive(Deserialize, Debug)]
pub struct HentaiData {
pub video: HentaiDataVideo
pub video: HentaiDataVideo,
}
#[derive(Deserialize, Debug)]
pub struct HentaiDataVideo {
pub hentai_video: HentaiVideo,
pub videos_manifest: VideosManifest
pub videos_manifest: VideosManifest,
}
#[derive(Deserialize, Debug)]
@ -62,7 +62,7 @@ pub struct HentaiVideo {
pub is_censored: bool,
pub likes: usize,
pub dislikes: usize,
pub hentai_tags: Vec<HentaiTag>
pub hentai_tags: Vec<HentaiTag>,
}
impl fmt::Display for HentaiInfo {
@ -107,17 +107,24 @@ impl fmt::Display for HentaiInfo {
}
};
}
text.push_str(&format!("Resolution: {}\n",
text.push_str(&format!(
"Resolution: {}\n",
match int_servers.is_empty() {
true => {
let mut keys: Vec<&&String> = string_servers.keys().collect();
keys.sort();
keys.iter().map(|&i| format!("{} ({}MB)", i, string_servers.get(i).unwrap())).collect::<Vec<_>>().join(", ")
},
keys.iter()
.map(|&i| format!("{} ({}MB)", i, string_servers.get(i).unwrap()))
.collect::<Vec<_>>()
.join(", ")
}
false => {
let mut keys: Vec<&i32> = int_servers.keys().collect();
keys.sort();
keys.iter().map(|i| format!("{} ({}MB)", i, int_servers.get(i).unwrap())).collect::<Vec<_>>().join(", ")
keys.iter()
.map(|i| format!("{} ({}MB)", i, int_servers.get(i).unwrap()))
.collect::<Vec<_>>()
.join(", ")
}
}
));
@ -143,7 +150,15 @@ impl fmt::Display for HentaiInfo {
}
text.push_str(&format!("Duration: {}{:02}\n", &duration, seconds));
}
text.push_str(&format!("Tags: {}", video.hentai_tags.iter().map(|i| i.text.as_str()).collect::<Vec<_>>().join(", ")));
text.push_str(&format!(
"Tags: {}",
video
.hentai_tags
.iter()
.map(|i| i.text.as_str())
.collect::<Vec<_>>()
.join(", ")
));
if !video.description.is_empty() {
text.push_str(&format!("\nDescription: {}", &video.description));
}
@ -154,35 +169,34 @@ impl fmt::Display for HentaiInfo {
#[derive(Deserialize, Debug)]
pub struct HentaiTag {
pub id: i32,
pub text: String
pub text: String,
}
#[derive(Deserialize, Debug)]
pub struct VideosManifest {
pub servers: Vec<VideoServer>
pub servers: Vec<VideoServer>,
}
#[derive(Deserialize, Debug)]
pub struct VideoServer {
pub is_permanent: bool,
pub streams: Vec<VideoStream>
pub streams: Vec<VideoStream>,
}
#[derive(Deserialize, Debug)]
pub struct VideoStream {
pub height: String,
pub url: String,
pub filesize_mbs: i32
pub filesize_mbs: i32,
}
fn remove_html<'de, D>(deserializer: D) -> Result<String, D::Error>
where
D: Deserializer<'de>
D: Deserializer<'de>,
{
struct RemoveHTML<T>(PhantomData<fn() -> T>);
impl<'de> Visitor<'de> for RemoveHTML<String>
{
impl<'de> Visitor<'de> for RemoveHTML<String> {
type Value = String;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
@ -191,19 +205,23 @@ where
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
where
E: de::Error
E: de::Error,
{
// https://brokenco.de/2020/08/03/serde-deserialize-with-string.html
// https://brokenco.de/2020/08/03/serde-deserialize-with-string.html
let mut text = String::with_capacity(value.len());
let mut reader = Reader::from_str(value);
reader.check_end_names(false);
let mut buf = Vec::new();
loop {
match reader.read_event(&mut buf) {
Ok(Event::Text(e)) => text.push_str(&unescape(reader.decode(e.escaped()).map_err(serde::de::Error::custom)?)),
Ok(Event::Text(e)) => text.push_str(&unescape(
reader
.decode(e.escaped())
.map_err(serde::de::Error::custom)?,
)),
Ok(Event::Eof) => break,
Err(err) => panic!("Error at position {}: {:?}", reader.buffer_position(), err),
_ => ()
_ => (),
};
buf.clear();
}
@ -244,12 +262,10 @@ impl From<serde_json::Error> for Error {
impl fmt::Display for Error {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(
&match self {
Error::Reqwest(err) => format!("reqwest error: {}", err),
Error::QuickXML(err) => format!("quick-xml error: {}", err),
Error::SerdeJSON(err) => format!("serde_json error: {}", err),
}
)
formatter.write_str(&match self {
Error::Reqwest(err) => format!("reqwest error: {}", err),
Error::QuickXML(err) => format!("quick-xml error: {}", err),
Error::SerdeJSON(err) => format!("serde_json error: {}", err),
})
}
}

15
src/unescape.rs
Unescape View File

@ -1,12 +1,12 @@
use regex::{Captures, RegexBuilder};
use std::borrow::Cow;
use regex::{RegexBuilder, Captures};
fn hex_char_decode(c: u8) -> u8 {
match c {
b'a'..=b'f' => c - b'a' + 10,
b'A'..=b'F' => c - b'A' + 10,
b'0'..=b'9' => c - b'0',
_ => panic!("Invalid hex character passed: {}", c)
_ => panic!("Invalid hex character passed: {}", c),
}
}
@ -470,7 +470,11 @@ pub fn unescape<'t>(text: &'t str) -> Cow<'t, str> {
"puncsp" => " ",
"thinsp" | "ThinSpace" => " ",
"hairsp" | "VeryThinSpace" => " ",
"ZeroWidthSpace" | "NegativeVeryThinSpace" | "NegativeThinSpace" | "NegativeMediumSpace" | "NegativeThickSpace" => "​",
"ZeroWidthSpace"
| "NegativeVeryThinSpace"
| "NegativeThinSpace"
| "NegativeMediumSpace"
| "NegativeThickSpace" => "​",
"zwnj" => "‌",
"zwj" => "‍",
"lrm" => "‎",
@ -1497,8 +1501,9 @@ pub fn unescape<'t>(text: &'t str) -> Cow<'t, str> {
"xopf" => "𝕩",
"yopf" => "𝕪",
"zopf" => "𝕫",
_ => &caps[0]
}.to_string()
_ => &caps[0],
}
.to_string()
}
})
}

31
src/utils.rs
Unescape View File

@ -1,14 +1,19 @@
use crate::structs;
use quick_xml::Reader;
use quick_xml::events::Event;
use quick_xml::Reader;
extern crate reqwest;
extern crate serde_json;
pub async fn search(client: reqwest::Client, query: &str, tags: Vec<&str>, broad_search: bool) -> Result<Vec<structs::SearchResult>, structs::Error> {
pub async fn search(
client: reqwest::Client,
query: &str,
tags: Vec<&str>,
broad_search: bool,
) -> Result<Vec<structs::SearchResult>, structs::Error> {
let tags_mode = match broad_search {
true => "OR",
false => "AND"
false => "AND",
};
Ok(serde_json::from_str(
&serde_json::from_str::<structs::SearchResultMetadata>(
@ -25,8 +30,12 @@ pub async fn search(client: reqwest::Client, query: &str, tags: Vec<&str>, broad
)?)
}
pub async fn get_hentai(client: reqwest::Client, id: &str) -> Result<Option<structs::HentaiInfo>, structs::Error> {
let resp = client.get(&format!("https://hanime.tv/videos/hentai/{}", id))
pub async fn get_hentai(
client: reqwest::Client,
id: &str,
) -> Result<Option<structs::HentaiInfo>, structs::Error> {
let resp = client
.get(&format!("https://hanime.tv/videos/hentai/{}", id))
.send()
.await?;
if resp.status() != 200 {
@ -45,23 +54,25 @@ pub async fn get_hentai(client: reqwest::Client, id: &str) -> Result<Option<stru
Ok(Event::Text(e)) if is_inside_script => {
let text = match reader.decode(e.escaped()) {
Ok(text) => text,
Err(_) => continue
Err(_) => continue,
};
if !script.is_empty() || text.starts_with("window.__NUXT__={") {
script.push_str(&text);
}
},
}
Ok(Event::End(ref e)) if e.name() == b"script" && is_inside_script => {
if script.is_empty() {
is_inside_script = false;
} else {
to_return = Some(serde_json::from_str(script.splitn(2, "=").nth(1).unwrap().trim_end_matches(';'))?);
to_return = Some(serde_json::from_str(
script.splitn(2, "=").nth(1).unwrap().trim_end_matches(';'),
)?);
break;
}
},
}
Err(err) => panic!("Error at position {}: {:?}", reader.buffer_position(), err),
Ok(Event::Eof) => break,
_ => ()
_ => (),
};
buf.clear();
}

Write Preview
Loading…
Cancel
Save