From 7c06d58c434130610a5d011400a7379cee580087 Mon Sep 17 00:00:00 2001 From: einisto <79069176+einisto@users.noreply.github.com> Date: Mon, 6 Jun 2022 14:22:20 +0300 Subject: [PATCH] Pretty printing & simplified logic --- Cargo.lock | 12 ++++++++++++ Cargo.toml | 1 + src/board.rs | 40 ++------------------------------------ src/downloader.rs | 47 +++++++++++++++++++++++++++++++++++++++++---- src/main.rs | 49 ++++++++++++++++++++++++++++++++++++----------- src/thread.rs | 33 ------------------------------- 6 files changed, 96 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df9c5ad..b4d4bcb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,17 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "colored" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" +dependencies = [ + "atty", + "lazy_static", + "winapi", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -685,6 +696,7 @@ name = "rusty-downloader" version = "0.1.0" dependencies = [ "clap", + "colored", "futures", "regex", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 68518de..6076a65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,3 +12,4 @@ reqwest = { version = "0.11", features = ["json"] } tokio = { version = "1.18.2", features = ["full"] } serde_json = "1.0" futures = "0.3" +colored = "2.0.0" diff --git a/src/board.rs b/src/board.rs index f0a2169..f197e07 100644 --- a/src/board.rs +++ b/src/board.rs @@ -1,5 +1,4 @@ use serde_json::Value; -use std::path::PathBuf; type Result = std::result::Result>; @@ -13,42 +12,7 @@ pub fn parse_url(url: &str) -> (String, String) { ) } -// TODO: use futures -pub async fn get_imagelist( - json_url: &str, - board_name: &str, - output_path: &PathBuf, -) -> Result<(Vec<(String, PathBuf)>, usize)> { - let mut img_data: Vec<(String, PathBuf)> = Vec::new(); - let board_data = get_threadlist(json_url, board_name).await?; - - for thread_url in &board_data { - let res_body = reqwest::get(thread_url).await?.text().await?; - let json_data: Value = serde_json::from_str(res_body.as_str())?; - - json_data["posts"] - .as_array() - .unwrap() - .iter() - .filter(|post| post["tim"].is_i64()) - .for_each(|post| { - let id = post["tim"].to_string(); - let ext = post["ext"].as_str().unwrap().to_string(); - let filepath = output_path.join(format!("{}{}", id, ext).as_str()); - - println!("{:#?}", filepath); - - img_data.push(( - format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext), - filepath, - )); - }); - } - - Ok((img_data, board_data.len())) -} - -async fn get_threadlist(json_url: &str, board_name: &str) -> Result> { +pub async fn get_threadlist(json_url: &str, board_name: &str) -> Result<(usize, Vec)> { let req_body = reqwest::get(json_url).await?.text().await?; let json_data: Value = serde_json::from_str(req_body.as_str())?; let board: Vec = json_data @@ -69,5 +33,5 @@ async fn get_threadlist(json_url: &str, board_name: &str) -> Result> }); }); - Ok(board_data) + Ok((board_data.len(), board_data)) } diff --git a/src/downloader.rs b/src/downloader.rs index 7a5ec03..cb31362 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -1,11 +1,41 @@ +use colored::Colorize; use futures::{stream, StreamExt}; use reqwest::Client; +use serde_json::Value; use std::path::PathBuf; use tokio::{fs::File, io::AsyncWriteExt}; type Result = std::result::Result>; -pub async fn download_images(img_data: &Vec<(String, PathBuf)>) -> Result { +pub async fn get_imagelist( + json_url: &str, + board_name: &str, + output_path: &PathBuf, +) -> Result> { + let req_body = reqwest::get(json_url).await?.text().await?; + let json_data: Value = serde_json::from_str(req_body.as_str())?; + + let mut img_data: Vec<(String, PathBuf)> = Vec::new(); + json_data["posts"] + .as_array() + .unwrap() + .iter() + .filter(|post| post["tim"].is_i64()) + .for_each(|post| { + let id = post["tim"].to_string(); + let ext = post["ext"].as_str().unwrap().to_string(); + let filepath = output_path.join(format!("{}{}", id, ext).as_str()); + + img_data.push(( + format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext), + filepath, + )) + }); + + Ok(img_data) +} + +pub async fn get_images(img_data: &Vec<(String, PathBuf)>) -> Result { let client = Client::builder().build()?; let futures = stream::iter(img_data.iter().map(|data| async { @@ -17,11 +47,20 @@ pub async fn download_images(img_data: &Vec<(String, PathBuf)>) -> Result Ok(bytes) => { let mut file = File::create(path).await.unwrap(); file.write_all(&bytes).await.unwrap(); - println!("{} bytes from {:?} to {:?}", bytes.len(), &url, &path); + + println!( + "{}", + format!("{} bytes: {:?} -> {:?}", bytes.len(), url, path) + .italic() + .purple() + ); } - Err(_) => eprintln!("Error reading bytes from {}", url), + Err(_) => eprintln!( + "{}", + format!("Error reading bytes from {}", url).bold().red() + ), }, - Err(_) => eprintln!("Error downloading {}", url), + Err(_) => eprintln!("{}", format!("Error downloading {}", url).bold().red()), } })) .buffer_unordered(100) diff --git a/src/main.rs b/src/main.rs index 97dc921..6ccb68c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ mod downloader; mod thread; use clap::{Arg, ArgGroup, Command}; +use colored::Colorize; use regex::Regex; use std::{path::PathBuf, process::exit}; use tokio; @@ -61,7 +62,7 @@ fn parse_cli_args() -> Result<(PathBuf, String, Mode)> { let target = match re.is_match(target_match) { true => target_match, false => { - eprintln!("Error: Invalid URL"); + eprintln!("{}", format!("Error: Invalid URL format").bold().red()); exit(0x0100); } }; @@ -77,25 +78,51 @@ fn parse_cli_args() -> Result<(PathBuf, String, Mode)> { async fn main() -> Result<()> { let (path, target, mode) = parse_cli_args()?; println!( - "\nDownload configuration:\n\tOUTPUT PATH: {:?}\n\tURL: {}\n\tDOWNLOAD MODE: {:?}\n", - path, target, mode + "{}", + format!( + "\nDownload configuration:\n\tOUTPUT PATH: {:?}\n\tURL: {}\n\tDOWNLOAD MODE: {:?}\n", + path, target, mode + ) + .bold() + .green() ); match mode { Mode::Thread => { let (json_url, board_name) = thread::parse_url(&target); - let img_data = thread::get_imagelist(&json_url, &board_name, &path).await?; - let total_amt = downloader::download_images(&img_data).await?; - println!("Total of {} files downloaded from 1 thread.\n", total_amt); + println!( + "{}", + format!("Parsing JSON from {}", json_url).bold().blue() + ); + let img_data = downloader::get_imagelist(&json_url, &board_name, &path).await?; + let filecount = downloader::get_images(&img_data).await?; + + println!( + "{}", + format!("Total of {} files downloaded from 1 thread.\n", filecount) + .bold() + .green() + ); } Mode::Board => { let (json_url, board_name) = board::parse_url(&target); - let (img_data, thread_amt) = - board::get_imagelist(&json_url, &board_name, &path).await?; - let total_amt = downloader::download_images(&img_data).await?; + let (thread_amt, thread_data) = board::get_threadlist(&json_url, &board_name).await?; + let mut filecount: usize = 0; + for url in &thread_data { + println!("{}", format!("Parsing JSON from {}", url).bold().blue()); + let img_data = downloader::get_imagelist(&url, &board_name, &path).await?; + let total_amt = downloader::get_images(&img_data).await?; + filecount += total_amt; + } + println!( - "Total of {} files downloaded from {} threads.\n", - total_amt, thread_amt + "{}", + format!( + "Total of {} files downloaded from {} threads.\n", + filecount, thread_amt + ) + .bold() + .green() ); } } diff --git a/src/thread.rs b/src/thread.rs index efb82e3..14d240a 100644 --- a/src/thread.rs +++ b/src/thread.rs @@ -1,8 +1,3 @@ -use serde_json::Value; -use std::path::PathBuf; - -type Result = std::result::Result>; - pub fn parse_url(url: &str) -> (String, String) { let url_split: Vec<&str> = url.split("/").collect(); let thread_id = url_split.get(url_split.len() - 1).unwrap(); @@ -16,31 +11,3 @@ pub fn parse_url(url: &str) -> (String, String) { board_name.to_string(), ) } - -pub async fn get_imagelist( - json_url: &str, - board_name: &str, - output_path: &PathBuf, -) -> Result> { - let req_body = reqwest::get(json_url).await?.text().await?; - let json_data: Value = serde_json::from_str(req_body.as_str())?; - - let mut img_data: Vec<(String, PathBuf)> = Vec::new(); - json_data["posts"] - .as_array() - .unwrap() - .iter() - .filter(|post| post["tim"].is_i64()) - .for_each(|post| { - let id = post["tim"].to_string(); - let ext = post["ext"].as_str().unwrap().to_string(); - let filepath = output_path.join(format!("{}{}", id, ext).as_str()); - - img_data.push(( - format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext), - filepath, - )) - }); - - Ok(img_data) -}