ae/dlrs
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Improved project structure

Browse Source
This commit is contained in:
einisto 2022-06-06 11:56:52 +03:00
parent a101ec3344
commit 235591e1b0
6 changed files with 180 additions and 491 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

393
Cargo.lock generated
View File

@ -2,18 +2,6 @@
# It is not intended for manual editing. # It is not intended for manual editing.
version = 3 version = 3
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "adler32"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
version = "0.7.18" version = "0.7.18"
@ -46,12 +34,6 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bit_field"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.3.2" version = "1.3.2"
@ -64,18 +46,6 @@ version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
[[package]]
name = "bytemuck"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc"
[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.1.0" version = "1.1.0"
@ -118,12 +88,6 @@ dependencies = [
"os_str_bytes", "os_str_bytes",
] ]
[[package]]
name = "color_quant"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]] [[package]]
name = "core-foundation" name = "core-foundation"
version = "0.9.3" version = "0.9.3"
@ -140,75 +104,6 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "crc32fast"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
dependencies = [
"cfg-if",
"lazy_static",
]
[[package]]
name = "deflate"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c86f7e25f518f4b81808a2cf1c50996a61f5c2eb394b2393bd87f2a4780a432f"
dependencies = [
"adler32",
]
[[package]]
name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]] [[package]]
name = "encoding_rs" name = "encoding_rs"
version = "0.8.31" version = "0.8.31"
@ -218,22 +113,6 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "exr"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14cc0e06fb5f67e5d6beadf3a382fec9baca1aa751c6d5368fdeee7e5932c215"
dependencies = [
"bit_field",
"deflate",
"flume",
"half",
"inflate",
"lebe",
"smallvec",
"threadpool",
]
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "1.7.0" version = "1.7.0"
@ -243,29 +122,6 @@ dependencies = [
"instant", "instant",
] ]
[[package]]
name = "flate2"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]]
name = "flume"
version = "0.10.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843c03199d0c0ca54bc1ea90ac0d507274c28abcc4f691ae8b4eaa375087c76a"
dependencies = [
"futures-core",
"futures-sink",
"nanorand",
"pin-project",
"spin",
]
[[package]] [[package]]
name = "fnv" name = "fnv"
version = "1.0.7" version = "1.0.7"
@ -386,29 +242,6 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "getrandom"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"wasi 0.10.2+wasi-snapshot-preview1",
"wasm-bindgen",
]
[[package]]
name = "gif"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b"
dependencies = [
"color_quant",
"weezl",
]
[[package]] [[package]]
name = "h2" name = "h2"
version = "0.3.13" version = "0.3.13"
@ -428,12 +261,6 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "half"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.11.2" version = "0.11.2"
@ -531,26 +358,6 @@ dependencies = [
"unicode-normalization", "unicode-normalization",
] ]
[[package]]
name = "image"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28edd9d7bc256be2502e325ac0628bde30b7001b9b52e0abe31a1a9dc2701212"
dependencies = [
"bytemuck",
"byteorder",
"color_quant",
"exr",
"gif",
"jpeg-decoder",
"num-iter",
"num-rational",
"num-traits",
"png",
"scoped_threadpool",
"tiff",
]
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "1.8.2" version = "1.8.2"
@ -561,15 +368,6 @@ dependencies = [
"hashbrown", "hashbrown",
] ]
[[package]]
name = "inflate"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff"
dependencies = [
"adler32",
]
[[package]] [[package]]
name = "instant" name = "instant"
version = "0.1.12" version = "0.1.12"
@ -591,15 +389,6 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
[[package]]
name = "jpeg-decoder"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9478aa10f73e7528198d75109c8be5cd7d15fb530238040148d5f9a22d4c5b3b"
dependencies = [
"rayon",
]
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.57" version = "0.3.57"
@ -615,12 +404,6 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lebe"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7efd1d698db0759e6ef11a7cd44407407399a910c774dd804c64c032da7826ff"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.126" version = "0.2.126"
@ -658,30 +441,12 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memoffset"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
dependencies = [
"autocfg",
]
[[package]] [[package]]
name = "mime" name = "mime"
version = "0.3.16" version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "miniz_oxide"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc"
dependencies = [
"adler",
]
[[package]] [[package]]
name = "mio" name = "mio"
version = "0.8.3" version = "0.8.3"
@ -690,19 +455,10 @@ checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799"
dependencies = [ dependencies = [
"libc", "libc",
"log", "log",
"wasi 0.11.0+wasi-snapshot-preview1", "wasi",
"windows-sys", "windows-sys",
] ]
[[package]]
name = "nanorand"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
dependencies = [
"getrandom",
]
[[package]] [[package]]
name = "native-tls" name = "native-tls"
version = "0.2.10" version = "0.2.10"
@ -721,47 +477,6 @@ dependencies = [
"tempfile", "tempfile",
] ]
[[package]]
name = "num-integer"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg",
]
[[package]] [[package]]
name = "num_cpus" name = "num_cpus"
version = "1.13.1" version = "1.13.1"
@ -858,26 +573,6 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pin-project"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.9" version = "0.2.9"
@ -896,18 +591,6 @@ version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
[[package]]
name = "png"
version = "0.17.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc38c0ad57efb786dd57b9864e5b18bae478c00c824dc55a38bbc9da95dde3ba"
dependencies = [
"bitflags",
"crc32fast",
"deflate",
"miniz_oxide",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.39" version = "1.0.39"
@ -926,30 +609,6 @@ dependencies = [
"proc-macro2", "proc-macro2",
] ]
[[package]]
name = "rayon"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"num_cpus",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.2.13" version = "0.2.13"
@ -1022,12 +681,11 @@ dependencies = [
] ]
[[package]] [[package]]
name = "rusty-wgdl" name = "rusty-downloader"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"clap", "clap",
"futures", "futures",
"image",
"regex", "regex",
"reqwest", "reqwest",
"serde_json", "serde_json",
@ -1050,12 +708,6 @@ dependencies = [
"windows-sys", "windows-sys",
] ]
[[package]]
name = "scoped_threadpool"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8"
[[package]] [[package]]
name = "scopeguard" name = "scopeguard"
version = "1.1.0" version = "1.1.0"
@ -1145,15 +797,6 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "spin"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c530c2b0d0bf8b69304b39fe2001993e267461948b890cd037d8ad4293fa1a0d"
dependencies = [
"lock_api",
]
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.10.0" version = "0.10.0"
@ -1200,26 +843,6 @@ version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
[[package]]
name = "threadpool"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
dependencies = [
"num_cpus",
]
[[package]]
name = "tiff"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cfada0986f446a770eca461e8c6566cb879682f7d687c8348aa0c857bd52286"
dependencies = [
"flate2",
"jpeg-decoder",
"weezl",
]
[[package]] [[package]]
name = "tinyvec" name = "tinyvec"
version = "1.6.0" version = "1.6.0"
@ -1383,12 +1006,6 @@ dependencies = [
"try-lock", "try-lock",
] ]
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]] [[package]]
name = "wasi" name = "wasi"
version = "0.11.0+wasi-snapshot-preview1" version = "0.11.0+wasi-snapshot-preview1"
@ -1471,12 +1088,6 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "weezl"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c97e489d8f836838d497091de568cf16b117486d529ec5579233521065bd5e4"
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"

3
Cargo.toml
View File

@ -1,5 +1,5 @@
[package] [package]
name = "rusty-wgdl" name = "rusty-downloader"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2021"
@ -12,4 +12,3 @@ reqwest = { version = "0.11", features = ["json"] }
tokio = { version = "1.18.2", features = ["full"] } tokio = { version = "1.18.2", features = ["full"] }
serde_json = "1.0" serde_json = "1.0"
futures = "0.3" futures = "0.3"
image = "0.24.2"

73
src/board.rs Normal file
View File

@ -0,0 +1,73 @@
use serde_json::Value;
use std::path::PathBuf;
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
pub fn parse_url(url: &str) -> (String, String) {
let url_split: Vec<&str> = url.split("/").collect();
let board_name = url_split.get(url_split.len() - 2).unwrap();
(
format!("https://a.4cdn.org/{}/catalog.json", board_name),
board_name.to_string(),
)
}
// TODO: use futures
pub async fn get_imagelist(
json_url: &str,
board_name: &str,
output_path: &PathBuf,
) -> Result<(Vec<(String, PathBuf)>, usize)> {
let mut img_data: Vec<(String, PathBuf)> = Vec::new();
let board_data = get_threadlist(json_url, board_name).await?;
for thread_url in &board_data {
let res_body = reqwest::get(thread_url).await?.text().await?;
let json_data: Value = serde_json::from_str(res_body.as_str())?;
json_data["posts"]
.as_array()
.unwrap()
.iter()
.filter(|post| post["tim"].is_i64())
.for_each(|post| {
let id = post["tim"].to_string();
let ext = post["ext"].as_str().unwrap().to_string();
let filepath = output_path.join(format!("{}{}", id, ext).as_str());
println!("{:#?}", filepath);
img_data.push((
format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext),
filepath,
));
});
}
Ok((img_data, board_data.len()))
}
async fn get_threadlist(json_url: &str, board_name: &str) -> Result<Vec<String>> {
let req_body = reqwest::get(json_url).await?.text().await?;
let json_data: Value = serde_json::from_str(req_body.as_str())?;
let board: Vec<Value> = json_data
.as_array()
.unwrap()
.iter()
.map(|page| page["threads"].clone())
.collect();
let mut board_data: Vec<String> = Vec::new();
board.iter().for_each(|thread_arr| {
thread_arr.as_array().unwrap().iter().for_each(|thread| {
let url = format!(
"https://a.4cdn.org/{}/thread/{}.json",
board_name, thread["no"]
);
board_data.push(url);
});
});
Ok(board_data)
}

33
src/downloader.rs Normal file
View File

@ -0,0 +1,33 @@
use futures::{stream, StreamExt};
use reqwest::Client;
use std::path::PathBuf;
use tokio::{fs::File, io::AsyncWriteExt};
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
pub async fn download_images(img_data: &Vec<(String, PathBuf)>) -> Result<usize> {
let client = Client::builder().build()?;
let futures = stream::iter(img_data.iter().map(|data| async {
let (url, path) = data;
let send_fut = client.get(url).send();
match send_fut.await {
Ok(res) => match res.bytes().await {
Ok(bytes) => {
let mut file = File::create(path).await.unwrap();
file.write_all(&bytes).await.unwrap();
println!("{} bytes from {:?} to {:?}", bytes.len(), &url, &path);
}
Err(_) => eprintln!("Error reading bytes from {}", url),
},
Err(_) => eprintln!("Error downloading {}", url),
}
}))
.buffer_unordered(100)
.collect::<Vec<()>>();
futures.await;
Ok(img_data.len())
}

123
src/main.rs
View File

@ -1,15 +1,15 @@
use clap::{Arg, ArgGroup, Command}; mod board;
use futures::{stream, StreamExt}; mod downloader;
use regex::Regex; mod thread;
use reqwest::Client;
use serde_json::Value; use clap::{Arg, ArgGroup, Command};
use std::{path::PathBuf, process::exit}; use regex::Regex;
use tokio::{fs::File, io::AsyncWriteExt}; use std::{path::PathBuf, process::exit};
use tokio;
// General type to make error handling easier
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>; type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
#[derive(Debug, PartialEq, Eq)] #[derive(Debug)]
enum Mode { enum Mode {
Thread, Thread,
Board, Board,
@ -73,105 +73,32 @@ fn parse_cli_args() -> Result<(PathBuf, String, Mode)> {
Ok((path, String::from(target), mode)) Ok((path, String::from(target), mode))
} }
fn create_thread_url(target: String) -> (String, String) {
let url_vec: Vec<&str> = target.split("/").collect();
let thread_id = url_vec.get(url_vec.len() - 1).unwrap();
let board = url_vec.get(url_vec.len() - 3).unwrap();
(
format!("https://a.4cdn.org/{}/thread/{}.json", board, thread_id),
board.to_string(),
)
}
//fn _create_board_url(target: String) -> String {
// let url_vec: Vec<&str> = target.split("/").collect();
// let board = url_vec.get(url_vec.len()).unwrap();
//
// format!("https://a.4cdn.org/{}/catalog.json", board)
//}
async fn get_imagelist(
json_url: &str,
board_name: &str,
output_path: &PathBuf,
) -> Result<Vec<(String, PathBuf)>> {
let req_body = reqwest::get(json_url).await?.text().await?;
let json_data: Value = serde_json::from_str(req_body.as_str())?;
let mut thread_img_data: Vec<(String, PathBuf)> = Vec::new();
json_data["posts"]
.as_array()
.unwrap()
.iter()
.filter(|post| post["tim"].is_i64())
.for_each(|post| {
let id = post["tim"].to_string();
let ext = post["ext"].as_str().unwrap().to_string();
let filepath = output_path.join(format!("{}{}", id, ext).as_str());
thread_img_data.push((
format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext),
filepath,
))
});
Ok(thread_img_data)
}
async fn download_images(input_output_data: &Vec<(String, PathBuf)>) -> Result<u16> {
let client = Client::builder().build()?;
let futures = stream::iter(input_output_data.iter().map(|data| async {
let (url, path) = data;
let send_fut = client.get(url).send();
match send_fut.await {
Ok(res) => match res.bytes().await {
Ok(bytes) => {
let mut file = File::create(path).await.unwrap();
file.write_all(&bytes).await.unwrap();
println!("Got {} bytes from {:?} to {:?}", bytes.len(), &url, &path)
}
Err(_) => eprintln!("Error reading bytes from {}", url),
},
Err(_) => eprintln!("Error downloading {}", url),
}
}))
.buffer_unordered(100)
.collect::<Vec<()>>();
futures.await;
Ok(0)
}
#[tokio::main] #[tokio::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
let (path, target, mode) = parse_cli_args()?; let (path, target, mode) = parse_cli_args()?;
println!( println!(
"\nDownload configuration:\n\tOUTPUT: {:?}\n\tURL: {}\n\tDOWNLOAD-MODE: {:?}\n", "\nDownload configuration:\n\tOUTPUT PATH: {:?}\n\tURL: {}\n\tDOWNLOAD MODE: {:?}\n",
path, target, mode path, target, mode
); );
match mode { match mode {
Mode::Thread => { Mode::Thread => {
let (json_url, board_name) = create_thread_url(target); let (json_url, board_name) = thread::parse_url(&target);
let targets = get_imagelist(&json_url.as_str(), &board_name.as_str(), &path).await?; let img_data = thread::get_imagelist(&json_url, &board_name, &path).await?;
let _total_file_amount = download_images(&targets).await?; let total_amt = downloader::download_images(&img_data).await?;
println!("Total of {} files downloaded from 1 thread.\n", total_amt);
//println!("Total files downloaded: {}", total_file_amount);
} }
//Mode::Board => { Mode::Board => {
// let json_url = create_board_url(target); let (json_url, board_name) = board::parse_url(&target);
// let id_list = get_imagelist(&json_url.as_str()).await?; let (img_data, thread_amt) =
// //println!("{:#?}", id_list); board::get_imagelist(&json_url, &board_name, &path).await?;
// get_images(id_list).await?; let total_amt = downloader::download_images(&img_data).await?;
//} println!(
_ => println!("Not supported"), "Total of {} files downloaded from {} threads.\n",
}; total_amt, thread_amt
);
}
}
Ok(()) Ok(())
} }

46
src/thread.rs Normal file
View File

@ -0,0 +1,46 @@
use serde_json::Value;
use std::path::PathBuf;
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
pub fn parse_url(url: &str) -> (String, String) {
let url_split: Vec<&str> = url.split("/").collect();
let thread_id = url_split.get(url_split.len() - 1).unwrap();
let board_name = url_split.get(url_split.len() - 3).unwrap();
(
format!(
"https://a.4cdn.org/{}/thread/{}.json",
board_name, thread_id
),
board_name.to_string(),
)
}
pub async fn get_imagelist(
json_url: &str,
board_name: &str,
output_path: &PathBuf,
) -> Result<Vec<(String, PathBuf)>> {
let req_body = reqwest::get(json_url).await?.text().await?;
let json_data: Value = serde_json::from_str(req_body.as_str())?;
let mut img_data: Vec<(String, PathBuf)> = Vec::new();
json_data["posts"]
.as_array()
.unwrap()
.iter()
.filter(|post| post["tim"].is_i64())
.for_each(|post| {
let id = post["tim"].to_string();
let ext = post["ext"].as_str().unwrap().to_string();
let filepath = output_path.join(format!("{}{}", id, ext).as_str());
img_data.push((
format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext),
filepath,
))
});
Ok(img_data)
}