From 351a534ce11a1460d2c7ba8364b5a9a56c8dc131 Mon Sep 17 00:00:00 2001 From: einisto <79069176+einisto@users.noreply.github.com> Date: Fri, 3 Jun 2022 01:14:54 +0300 Subject: [PATCH] Asynchronous file-dl --- Cargo.lock | 442 +++++++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + src/main.rs | 119 ++++++++++---- 3 files changed, 532 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37b5176..c8f14c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aho-corasick" version = "0.7.18" @@ -34,6 +46,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bit_field" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4" + [[package]] name = "bitflags" version = "1.3.2" @@ -46,6 +64,18 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +[[package]] +name = "bytemuck" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.1.0" @@ -88,6 +118,12 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "core-foundation" version = "0.9.3" @@ -104,6 +140,75 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if", + "lazy_static", +] + +[[package]] +name = "deflate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86f7e25f518f4b81808a2cf1c50996a61f5c2eb394b2393bd87f2a4780a432f" +dependencies = [ + "adler32", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -113,6 +218,22 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "exr" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14cc0e06fb5f67e5d6beadf3a382fec9baca1aa751c6d5368fdeee7e5932c215" +dependencies = [ + "bit_field", + "deflate", + "flume", + "half", + "inflate", + "lebe", + "smallvec", + "threadpool", +] + [[package]] name = "fastrand" version = "1.7.0" @@ -122,6 +243,29 @@ dependencies = [ "instant", ] +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "flume" +version = "0.10.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843c03199d0c0ca54bc1ea90ac0d507274c28abcc4f691ae8b4eaa375087c76a" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "pin-project", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -153,6 +297,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.21" @@ -160,6 +319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -168,6 +328,34 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" +[[package]] +name = "futures-executor" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-macro" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.21" @@ -186,10 +374,39 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "gif" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b" +dependencies = [ + "color_quant", + "weezl", ] [[package]] @@ -211,6 +428,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.11.2" @@ -308,6 +531,26 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28edd9d7bc256be2502e325ac0628bde30b7001b9b52e0abe31a1a9dc2701212" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "exr", + "gif", + "jpeg-decoder", + "num-iter", + "num-rational", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] + [[package]] name = "indexmap" version = "1.8.2" @@ -318,6 +561,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inflate" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff" +dependencies = [ + "adler32", +] + [[package]] name = "instant" version = "0.1.12" @@ -339,6 +591,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" +[[package]] +name = "jpeg-decoder" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9478aa10f73e7528198d75109c8be5cd7d15fb530238040148d5f9a22d4c5b3b" +dependencies = [ + "rayon", +] + [[package]] name = "js-sys" version = "0.3.57" @@ -354,6 +615,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lebe" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7efd1d698db0759e6ef11a7cd44407407399a910c774dd804c64c032da7826ff" + [[package]] name = "libc" version = "0.2.126" @@ -391,12 +658,30 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "miniz_oxide" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.3" @@ -405,10 +690,19 @@ checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom", +] + [[package]] name = "native-tls" version = "0.2.10" @@ -427,6 +721,47 @@ dependencies = [ "tempfile", ] +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.13.1" @@ -523,6 +858,26 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pin-project" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -541,6 +896,18 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +[[package]] +name = "png" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc38c0ad57efb786dd57b9864e5b18bae478c00c824dc55a38bbc9da95dde3ba" +dependencies = [ + "bitflags", + "crc32fast", + "deflate", + "miniz_oxide", +] + [[package]] name = "proc-macro2" version = "1.0.39" @@ -559,6 +926,30 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.13" @@ -635,6 +1026,8 @@ name = "rusty-wgdl" version = "0.1.0" dependencies = [ "clap", + "futures", + "image", "regex", "reqwest", "serde_json", @@ -657,6 +1050,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + [[package]] name = "scopeguard" version = "1.1.0" @@ -746,6 +1145,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c530c2b0d0bf8b69304b39fe2001993e267461948b890cd037d8ad4293fa1a0d" +dependencies = [ + "lock_api", +] + [[package]] name = "strsim" version = "0.10.0" @@ -792,6 +1200,26 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "tiff" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cfada0986f446a770eca461e8c6566cb879682f7d687c8348aa0c857bd52286" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -955,6 +1383,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1037,6 +1471,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c97e489d8f836838d497091de568cf16b117486d529ec5579233521065bd5e4" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 28efc64..4e0f67a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,3 +11,5 @@ regex = "1.5.6" reqwest = { version = "0.11", features = ["json"] } tokio = { version = "1.18.2", features = ["full"] } serde_json = "1.0" +futures = "0.3" +image = "0.24.2" diff --git a/src/main.rs b/src/main.rs index 46d2cb2..1e1ccff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,12 @@ use clap::{Arg, ArgGroup, Command}; +use futures::{stream, StreamExt}; +//use image::io::Reader; use regex::Regex; +use reqwest::Client; use serde_json::Value; +//use std::io::Cursor; use std::{path::PathBuf, process::exit}; +use tokio; // General error type to make error handling easier type Result = std::result::Result>; @@ -20,7 +25,7 @@ fn parse_cli_args() -> Result<(PathBuf, String, Mode)> { .arg( Arg::new("output") .short('o') - .long("path") + .long("output") .value_name("PATH") .help("Set an output directory") .takes_value(true) @@ -70,40 +75,91 @@ fn parse_cli_args() -> Result<(PathBuf, String, Mode)> { Ok((path, String::from(target), mode)) } -fn create_thread_url(target: String) -> String { +fn create_thread_url(target: String) -> (String, String) { let url_vec: Vec<&str> = target.split("/").collect(); let thread_id = url_vec.get(url_vec.len() - 1).unwrap(); let board = url_vec.get(url_vec.len() - 3).unwrap(); - format!("https://a.4cdn.org/{}/thread/{}.json", board, thread_id) + ( + format!("https://a.4cdn.org/{}/thread/{}.json", board, thread_id), + board.to_string(), + ) } -fn create_board_url(target: String) -> String { - let url_vec: Vec<&str> = target.split("/").collect(); - let board = url_vec.get(url_vec.len()).unwrap(); +//fn _create_board_url(target: String) -> String { +// let url_vec: Vec<&str> = target.split("/").collect(); +// let board = url_vec.get(url_vec.len()).unwrap(); +// +// format!("https://a.4cdn.org/{}/catalog.json", board) +//} - format!("https://a.4cdn.org/{}/catalog.json", board) -} - -async fn get_imagelist(json_url: &str) -> Result> { +async fn get_imagelist( + json_url: &str, + board_name: &str, + output_path: &PathBuf, +) -> Result> { let req_body = reqwest::get(json_url).await?.text().await?; let json_data: Value = serde_json::from_str(req_body.as_str())?; - let mut thread_img_data: Vec<(String, String)> = Vec::new(); - for post in json_data["posts"].as_array().unwrap() { - if post["tim"].is_i64() { + let mut thread_img_data: Vec<(String, PathBuf)> = Vec::new(); + json_data["posts"] + .as_array() + .unwrap() + .iter() + .filter(|post| post["tim"].is_i64()) + .for_each(|post| { + let id = post["tim"].to_string(); + let ext = post["ext"].as_str().unwrap().to_string(); + let filepath = output_path.join(format!("{}{}", id, ext).as_str()); + thread_img_data.push(( - post["tim"].to_string(), - post["ext"].as_str().unwrap().to_string(), - )); - } else { - continue; - } - } + format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext), + filepath, + )) + }); Ok(thread_img_data) } +async fn download_images(input_output_data: &Vec<(String, PathBuf)>) -> Result { + let client = Client::new(); + + let bodies = stream::iter(input_output_data.iter()) + .map(|data| { + let client = &client; + async move { + let res = client.get(&data.0).send().await?; + res.bytes().await + } + }) + .buffer_unordered(input_output_data.len()); + + bodies + .for_each(|body| async move { + println!("{:?}", body); + + match body { + Ok(bytes) => { + //let img = Reader::new(Cursor::new(bytes)) + // .with_guessed_format() + // .map(|guess| guess.decode()) + // .map(|decode| decode); + + // TODO: Write 'img' to file asynchronously + // Requires access to pathnames (stored alongside urls in tuples inside input_output_data-vec) + + println!("Got {} bytes", bytes.len()); + } + Err(e) => { + eprintln!("Error during request: {e}"); + } + } + }) + .await; + + Ok(0) +} + #[tokio::main] async fn main() -> Result<()> { // TODO: add possible config-file for default output path (similar to wgdl.py) @@ -116,17 +172,20 @@ async fn main() -> Result<()> { match mode { Mode::Thread => { - let json_url = create_thread_url(target); - let id_list = get_imagelist(&json_url.as_str()).await?; - println!("{:#?}", id_list); - // 3.) download img based on json - } - Mode::Board => { - let json_url = create_board_url(target); - let id_list = get_imagelist(&json_url.as_str()).await?; - println!("{:#?}", id_list); - // 3.) download img based on json + let (json_url, board_name) = create_thread_url(target); + let targets = get_imagelist(&json_url.as_str(), &board_name.as_str(), &path).await?; + println!("LIST of URLs:\n{:?}", targets); + let _total_file_amount = download_images(&targets).await?; + + //println!("Total files downloaded: {}", total_file_amount); } + //Mode::Board => { + // let json_url = create_board_url(target); + // let id_list = get_imagelist(&json_url.as_str()).await?; + // //println!("{:#?}", id_list); + // get_images(id_list).await?; + //} + _ => println!("Not supported"), }; Ok(())