some things idk

This commit is contained in:
blank X 2021-02-05 12:29:34 +07:00
parent e00fb4b652
commit 17ac499fb7
Signed by: blankie
GPG Key ID: CC15FC822C7F61F5
5 changed files with 47 additions and 21 deletions

2
Cargo.lock generated
View File

@ -330,7 +330,7 @@ dependencies = [
[[package]] [[package]]
name = "mangafetchi" name = "mangafetchi"
version = "0.1.7" version = "0.1.8"
dependencies = [ dependencies = [
"quick-xml", "quick-xml",
"reqwest", "reqwest",

View File

@ -1,6 +1,6 @@
[package] [package]
name = "mangafetchi" name = "mangafetchi"
version = "0.1.7" version = "0.1.8"
authors = ["blank X <theblankx@protonmail.com>"] authors = ["blank X <theblankx@protonmail.com>"]
edition = "2018" edition = "2018"

View File

@ -5,6 +5,7 @@ use std::env;
use std::sync::Arc; use std::sync::Arc;
use std::process::exit; use std::process::exit;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::collections::VecDeque;
use url::Url; use url::Url;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
@ -53,7 +54,7 @@ pub async fn run(mut args: env::Args) {
exit(1); exit(1);
} }
}; };
let mutex = Arc::new(Mutex::new(DownloadData { data: Vec::new(), is_done: false })); let mutex = Arc::new(Mutex::new(DownloadData { data: VecDeque::new(), is_done: false }));
let handles: Vec<JoinHandle<()>> = summon_handles(client.clone(), Arc::clone(&mutex)).await; let handles: Vec<JoinHandle<()>> = summon_handles(client.clone(), Arc::clone(&mutex)).await;
for chapter in chapters { for chapter in chapters {
loop { loop {
@ -99,21 +100,26 @@ async fn summon_handles(client: reqwest::Client, mutex: Arc<Mutex<DownloadData>>
let cloned_mutex = Arc::clone(&tcloned_mutex); let cloned_mutex = Arc::clone(&tcloned_mutex);
let cloned_client = tcloned_client.clone(); let cloned_client = tcloned_client.clone();
let mut download_data = cloned_mutex.lock().await; let mut download_data = cloned_mutex.lock().await;
if download_data.data.is_empty() { let data = download_data.data.pop_front();
if download_data.is_done { let is_done = download_data.is_done;
break;
}
drop(download_data);
sleep(Duration::from_millis(NO_ITEM_WAIT_TIME)).await;
continue;
}
let (url, file_name, referer) = download_data.data.remove(0);
drop(download_data); drop(download_data);
let (url, file_name, referer) = match data {
Some(data) => data,
None => {
if is_done {
break;
}
sleep(Duration::from_millis(NO_ITEM_WAIT_TIME)).await;
continue;
}
};
eprintln!("[DW{}] Downloading {} to {}", worker_id, &url, file_name.display()); eprintln!("[DW{}] Downloading {} to {}", worker_id, &url, file_name.display());
loop { loop {
if utils::download_file(cloned_client.clone(), &url, &file_name, &referer).await.unwrap() { match utils::download_file(cloned_client.clone(), &url, &file_name, &referer).await {
break; Ok(result) if result => break,
} Ok(_) => (),
Err(err) => eprintln!("[DW{}] Error while downloading {}: {}", worker_id, file_name.display(), err)
};
sleep(Duration::from_millis(NON_IMAGE_WAIT_TIME)).await; sleep(Duration::from_millis(NON_IMAGE_WAIT_TIME)).await;
} }
eprintln!("[DW{}] Downloaded {} to {}", worker_id, &url, file_name.display()); eprintln!("[DW{}] Downloaded {} to {}", worker_id, &url, file_name.display());
@ -125,6 +131,6 @@ async fn summon_handles(client: reqwest::Client, mutex: Arc<Mutex<DownloadData>>
} }
struct DownloadData { struct DownloadData {
pub data: Vec<(String, PathBuf, String)>, pub data: VecDeque<(String, PathBuf, String)>,
pub is_done: bool pub is_done: bool
} }

View File

@ -1,3 +1,4 @@
use std::io;
use std::fmt; use std::fmt;
use serde::Deserialize; use serde::Deserialize;
extern crate url; extern crate url;
@ -88,6 +89,7 @@ pub enum Error {
Reqwest(reqwest::Error), Reqwest(reqwest::Error),
URL(url::ParseError), URL(url::ParseError),
SerdeJSON(serde_json::Error), SerdeJSON(serde_json::Error),
IO(io::Error)
} }
impl From<reqwest::Error> for Error { impl From<reqwest::Error> for Error {
@ -110,3 +112,21 @@ impl From<serde_json::Error> for Error {
Error::SerdeJSON(error) Error::SerdeJSON(error)
} }
} }
impl From<io::Error> for Error {
#[inline]
fn from(error: io::Error) -> Error {
Error::IO(error)
}
}
impl fmt::Display for Error {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&match self {
Error::Reqwest(err) => format!("reqwest: {}", err),
Error::URL(err) => format!("url: {}", err),
Error::SerdeJSON(err) => format!("serde_json: {}", err),
Error::IO(err) => format!("io: {}", err)
})
}
}

View File

@ -722,20 +722,20 @@ fn parse_manganelo_pages(text: &str) -> Vec<String> {
pages pages
} }
pub async fn download_file(client: reqwest::Client, url: &str, file_name: &PathBuf, referer: &str) -> Result<bool, reqwest::Error> { pub async fn download_file(client: reqwest::Client, url: &str, file_name: &PathBuf, referer: &str) -> Result<bool, structs::Error> {
let resp = client.get(url) let resp = client.get(url)
.header("Referer", referer) .header("Referer", referer)
.send() .send()
.await?; .await?;
match resp.headers().get("Content-Type") { match resp.headers().get("Content-Type") {
Some(header_value) => { Some(header_value) => {
if header_value.to_str().unwrap().starts_with("image/") { if header_value.to_str().unwrap_or_default().starts_with("image/") {
let bytes = resp.bytes().await?; let bytes = resp.bytes().await?;
if !file_name.parent().unwrap().is_dir() { if !file_name.parent().unwrap().is_dir() {
create_dir(file_name.parent().unwrap()).unwrap(); create_dir(file_name.parent().unwrap())?;
} }
let mut file = File::create(&file_name).unwrap(); let mut file = File::create(&file_name)?;
file.write_all(&bytes).unwrap(); file.write_all(&bytes)?;
return Ok(true); return Ok(true);
} }
return Ok(false); return Ok(false);