Download workers only commit suicide when all chapters are retrieved

This commit is contained in:
blank X 2021-01-11 13:46:56 +07:00
parent 05cc8ed95b
commit e8470cb8b9
3 changed files with 27 additions and 20 deletions

2
Cargo.lock generated
View File

@ -330,7 +330,7 @@ dependencies = [
[[package]] [[package]]
name = "mangafetchi" name = "mangafetchi"
version = "0.1.4" version = "0.1.5"
dependencies = [ dependencies = [
"quick-xml", "quick-xml",
"reqwest", "reqwest",

View File

@ -1,6 +1,6 @@
[package] [package]
name = "mangafetchi" name = "mangafetchi"
version = "0.1.4" version = "0.1.5"
authors = ["blank X <theblankx@protonmail.com>"] authors = ["blank X <theblankx@protonmail.com>"]
edition = "2018" edition = "2018"

View File

@ -13,6 +13,7 @@ extern crate reqwest;
const DOWNLOAD_WORKERS: usize = 5; const DOWNLOAD_WORKERS: usize = 5;
const NON_IMAGE_WAIT_TIME: u64 = 5000; const NON_IMAGE_WAIT_TIME: u64 = 5000;
const NO_ITEM_WAIT_TIME: u64 = 1000;
pub async fn run(mut args: env::Args) { pub async fn run(mut args: env::Args) {
let manga_id = match args.next() { let manga_id = match args.next() {
@ -50,8 +51,8 @@ pub async fn run(mut args: env::Args) {
exit(1); exit(1);
} }
}; };
let mutex: Arc<Mutex<Vec<(String, PathBuf, String)>>> = Arc::new(Mutex::new(Vec::new())); let mutex = Arc::new(Mutex::new(DownloadData { data: Vec::new(), is_done: false }));
let mut handles: Option<Vec<JoinHandle<()>>> = None; let handles: Vec<JoinHandle<()>> = summon_handles(client.clone(), Arc::clone(&mutex)).await;
for chapter in chapters { for chapter in chapters {
let cloned_mutex = Arc::clone(&mutex); let cloned_mutex = Arc::clone(&mutex);
let chapter_pages = utils::get_pages(client.clone(), &chapter, &manga_id).await.unwrap(); let chapter_pages = utils::get_pages(client.clone(), &chapter, &manga_id).await.unwrap();
@ -64,22 +65,18 @@ pub async fn run(mut args: env::Args) {
} }
} }
if !to_extend.is_empty() { if !to_extend.is_empty() {
cloned_mutex.lock().await.extend(to_extend); cloned_mutex.lock().await.data.extend(to_extend);
if handles.is_none() {
handles = Some(summon_handles(client.clone(), cloned_mutex).await);
}
} }
} }
if handles.is_some() { {
for handle in handles.unwrap() { mutex.lock().await.is_done = true;
handle.await.unwrap(); }
} for handle in handles {
} else { handle.await.unwrap();
eprintln!("Everything has already been downloaded");
} }
} }
async fn summon_handles(client: reqwest::Client, mutex: Arc<Mutex<Vec<(String, PathBuf, String)>>>) -> Vec<JoinHandle<()>> { async fn summon_handles(client: reqwest::Client, mutex: Arc<Mutex<DownloadData>>) -> Vec<JoinHandle<()>> {
let mut handles = Vec::with_capacity(DOWNLOAD_WORKERS); let mut handles = Vec::with_capacity(DOWNLOAD_WORKERS);
for worker_id in 0..DOWNLOAD_WORKERS { for worker_id in 0..DOWNLOAD_WORKERS {
let tcloned_mutex = Arc::clone(&mutex); let tcloned_mutex = Arc::clone(&mutex);
@ -89,12 +86,17 @@ async fn summon_handles(client: reqwest::Client, mutex: Arc<Mutex<Vec<(String, P
loop { loop {
let cloned_mutex = Arc::clone(&tcloned_mutex); let cloned_mutex = Arc::clone(&tcloned_mutex);
let cloned_client = tcloned_client.clone(); let cloned_client = tcloned_client.clone();
let mut vec = cloned_mutex.lock().await; let mut download_data = cloned_mutex.lock().await;
if vec.is_empty() { if download_data.data.is_empty() {
break; if download_data.is_done {
break;
}
drop(download_data);
sleep(Duration::from_millis(NO_ITEM_WAIT_TIME)).await;
continue;
} }
let (url, file_name, referer) = vec.remove(0); let (url, file_name, referer) = download_data.data.remove(0);
drop(vec); drop(download_data);
eprintln!("[DW{}] Downloading {} to {}", worker_id, &url, file_name.display()); eprintln!("[DW{}] Downloading {} to {}", worker_id, &url, file_name.display());
loop { loop {
if utils::download_file(cloned_client.clone(), &url, &file_name, &referer).await.unwrap() { if utils::download_file(cloned_client.clone(), &url, &file_name, &referer).await.unwrap() {
@ -109,3 +111,8 @@ async fn summon_handles(client: reqwest::Client, mutex: Arc<Mutex<Vec<(String, P
} }
handles handles
} }
struct DownloadData {
pub data: Vec<(String, PathBuf, String)>,
pub is_done: bool
}