ae/dlrs
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Pretty printing & simplified logic

Browse Source
This commit is contained in:
einisto 2022-06-06 14:22:20 +03:00
parent 235591e1b0
commit 7c06d58c43
6 changed files with 96 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Cargo.lock generated
View File

@ -88,6 +88,17 @@ dependencies = [
"os_str_bytes",
]
[[package]]
name = "colored"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd"
dependencies = [
"atty",
"lazy_static",
"winapi",
]
[[package]]
name = "core-foundation"
version = "0.9.3"
@ -685,6 +696,7 @@ name = "rusty-downloader"
version = "0.1.0"
dependencies = [
"clap",
"colored",
"futures",
"regex",
"reqwest",

1
Cargo.toml
View File

@ -12,3 +12,4 @@ reqwest = { version = "0.11", features = ["json"] }
tokio = { version = "1.18.2", features = ["full"] }
serde_json = "1.0"
futures = "0.3"
colored = "2.0.0"

40
src/board.rs
View File

@ -1,5 +1,4 @@
use serde_json::Value;
use std::path::PathBuf;
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
@ -13,42 +12,7 @@ pub fn parse_url(url: &str) -> (String, String) {
)
}
// TODO: use futures
pub async fn get_imagelist(
json_url: &str,
board_name: &str,
output_path: &PathBuf,
) -> Result<(Vec<(String, PathBuf)>, usize)> {
let mut img_data: Vec<(String, PathBuf)> = Vec::new();
let board_data = get_threadlist(json_url, board_name).await?;
for thread_url in &board_data {
let res_body = reqwest::get(thread_url).await?.text().await?;
let json_data: Value = serde_json::from_str(res_body.as_str())?;
json_data["posts"]
.as_array()
.unwrap()
.iter()
.filter(|post| post["tim"].is_i64())
.for_each(|post| {
let id = post["tim"].to_string();
let ext = post["ext"].as_str().unwrap().to_string();
let filepath = output_path.join(format!("{}{}", id, ext).as_str());
println!("{:#?}", filepath);
img_data.push((
format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext),
filepath,
));
});
}
Ok((img_data, board_data.len()))
}
async fn get_threadlist(json_url: &str, board_name: &str) -> Result<Vec<String>> {
pub async fn get_threadlist(json_url: &str, board_name: &str) -> Result<(usize, Vec<String>)> {
let req_body = reqwest::get(json_url).await?.text().await?;
let json_data: Value = serde_json::from_str(req_body.as_str())?;
let board: Vec<Value> = json_data
@ -69,5 +33,5 @@ async fn get_threadlist(json_url: &str, board_name: &str) -> Result<Vec<String>>
});
});
Ok(board_data)
Ok((board_data.len(), board_data))
}

47
src/downloader.rs
View File

@ -1,11 +1,41 @@
use colored::Colorize;
use futures::{stream, StreamExt};
use reqwest::Client;
use serde_json::Value;
use std::path::PathBuf;
use tokio::{fs::File, io::AsyncWriteExt};
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
pub async fn download_images(img_data: &Vec<(String, PathBuf)>) -> Result<usize> {
pub async fn get_imagelist(
json_url: &str,
board_name: &str,
output_path: &PathBuf,
) -> Result<Vec<(String, PathBuf)>> {
let req_body = reqwest::get(json_url).await?.text().await?;
let json_data: Value = serde_json::from_str(req_body.as_str())?;
let mut img_data: Vec<(String, PathBuf)> = Vec::new();
json_data["posts"]
.as_array()
.unwrap()
.iter()
.filter(|post| post["tim"].is_i64())
.for_each(|post| {
let id = post["tim"].to_string();
let ext = post["ext"].as_str().unwrap().to_string();
let filepath = output_path.join(format!("{}{}", id, ext).as_str());
img_data.push((
format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext),
filepath,
))
});
Ok(img_data)
}
pub async fn get_images(img_data: &Vec<(String, PathBuf)>) -> Result<usize> {
let client = Client::builder().build()?;
let futures = stream::iter(img_data.iter().map(|data| async {
@ -17,11 +47,20 @@ pub async fn download_images(img_data: &Vec<(String, PathBuf)>) -> Result<usize>
Ok(bytes) => {
let mut file = File::create(path).await.unwrap();
file.write_all(&bytes).await.unwrap();
println!("{} bytes from {:?} to {:?}", bytes.len(), &url, &path);
println!(
"{}",
format!("{} bytes: {:?} -> {:?}", bytes.len(), url, path)
.italic()
.purple()
);
}
Err(_) => eprintln!("Error reading bytes from {}", url),
Err(_) => eprintln!(
"{}",
format!("Error reading bytes from {}", url).bold().red()
),
},
Err(_) => eprintln!("Error downloading {}", url),
Err(_) => eprintln!("{}", format!("Error downloading {}", url).bold().red()),
}
}))
.buffer_unordered(100)

43
src/main.rs
View File

@ -3,6 +3,7 @@ mod downloader;
mod thread;
use clap::{Arg, ArgGroup, Command};
use colored::Colorize;
use regex::Regex;
use std::{path::PathBuf, process::exit};
use tokio;
@ -61,7 +62,7 @@ fn parse_cli_args() -> Result<(PathBuf, String, Mode)> {
let target = match re.is_match(target_match) {
true => target_match,
false => {
eprintln!("Error: Invalid URL");
eprintln!("{}", format!("Error: Invalid URL format").bold().red());
exit(0x0100);
}
};
@ -77,25 +78,51 @@ fn parse_cli_args() -> Result<(PathBuf, String, Mode)> {
async fn main() -> Result<()> {
let (path, target, mode) = parse_cli_args()?;
println!(
"{}",
format!(
"\nDownload configuration:\n\tOUTPUT PATH: {:?}\n\tURL: {}\n\tDOWNLOAD MODE: {:?}\n",
path, target, mode
)
.bold()
.green()
);
match mode {
Mode::Thread => {
let (json_url, board_name) = thread::parse_url(&target);
let img_data = thread::get_imagelist(&json_url, &board_name, &path).await?;
let total_amt = downloader::download_images(&img_data).await?;
println!("Total of {} files downloaded from 1 thread.\n", total_amt);
println!(
"{}",
format!("Parsing JSON from {}", json_url).bold().blue()
);
let img_data = downloader::get_imagelist(&json_url, &board_name, &path).await?;
let filecount = downloader::get_images(&img_data).await?;
println!(
"{}",
format!("Total of {} files downloaded from 1 thread.\n", filecount)
.bold()
.green()
);
}
Mode::Board => {
let (json_url, board_name) = board::parse_url(&target);
let (img_data, thread_amt) =
board::get_imagelist(&json_url, &board_name, &path).await?;
let total_amt = downloader::download_images(&img_data).await?;
let (thread_amt, thread_data) = board::get_threadlist(&json_url, &board_name).await?;
let mut filecount: usize = 0;
for url in &thread_data {
println!("{}", format!("Parsing JSON from {}", url).bold().blue());
let img_data = downloader::get_imagelist(&url, &board_name, &path).await?;
let total_amt = downloader::get_images(&img_data).await?;
filecount += total_amt;
}
println!(
"{}",
format!(
"Total of {} files downloaded from {} threads.\n",
total_amt, thread_amt
filecount, thread_amt
)
.bold()
.green()
);
}
}

33
src/thread.rs
View File

@ -1,8 +1,3 @@
use serde_json::Value;
use std::path::PathBuf;
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
pub fn parse_url(url: &str) -> (String, String) {
let url_split: Vec<&str> = url.split("/").collect();
let thread_id = url_split.get(url_split.len() - 1).unwrap();
@ -16,31 +11,3 @@ pub fn parse_url(url: &str) -> (String, String) {
board_name.to_string(),
)
}
pub async fn get_imagelist(
json_url: &str,
board_name: &str,
output_path: &PathBuf,
) -> Result<Vec<(String, PathBuf)>> {
let req_body = reqwest::get(json_url).await?.text().await?;
let json_data: Value = serde_json::from_str(req_body.as_str())?;
let mut img_data: Vec<(String, PathBuf)> = Vec::new();
json_data["posts"]
.as_array()
.unwrap()
.iter()
.filter(|post| post["tim"].is_i64())
.for_each(|post| {
let id = post["tim"].to_string();
let ext = post["ext"].as_str().unwrap().to_string();
let filepath = output_path.join(format!("{}{}", id, ext).as_str());
img_data.push((
format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext),
filepath,
))
});
Ok(img_data)
}