From 235591e1b09a39296f903fa769d984c7a1d12771 Mon Sep 17 00:00:00 2001
From: einisto <79069176+einisto@users.noreply.github.com>
Date: Mon, 6 Jun 2022 11:56:52 +0300
Subject: [PATCH] Improved project structure

---
 Cargo.lock        | 393 +---------------------------------------------
 Cargo.toml        |   3 +-
 src/board.rs      |  73 +++++++++
 src/downloader.rs |  33 ++++
 src/main.rs       | 123 +++------------
 src/thread.rs     |  46 ++++++
 6 files changed, 180 insertions(+), 491 deletions(-)
 create mode 100644 src/board.rs
 create mode 100644 src/downloader.rs
 create mode 100644 src/thread.rs

diff --git a/Cargo.lock b/Cargo.lock
index c8f14c7..df9c5ad 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,18 +2,6 @@
 # It is not intended for manual editing.
 version = 3
 
-[[package]]
-name = "adler"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
-
-[[package]]
-name = "adler32"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
-
 [[package]]
 name = "aho-corasick"
 version = "0.7.18"
@@ -46,12 +34,6 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
-[[package]]
-name = "bit_field"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4"
-
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -64,18 +46,6 @@ version = "3.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
 
-[[package]]
-name = "bytemuck"
-version = "1.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc"
-
-[[package]]
-name = "byteorder"
-version = "1.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
-
 [[package]]
 name = "bytes"
 version = "1.1.0"
@@ -118,12 +88,6 @@ dependencies = [
  "os_str_bytes",
 ]
 
-[[package]]
-name = "color_quant"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
-
 [[package]]
 name = "core-foundation"
 version = "0.9.3"
@@ -140,75 +104,6 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
 
-[[package]]
-name = "crc32fast"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "crossbeam-channel"
-version = "0.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
-dependencies = [
- "cfg-if",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
-dependencies = [
- "cfg-if",
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
-dependencies = [
- "autocfg",
- "cfg-if",
- "crossbeam-utils",
- "lazy_static",
- "memoffset",
- "scopeguard",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
-dependencies = [
- "cfg-if",
- "lazy_static",
-]
-
-[[package]]
-name = "deflate"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c86f7e25f518f4b81808a2cf1c50996a61f5c2eb394b2393bd87f2a4780a432f"
-dependencies = [
- "adler32",
-]
-
-[[package]]
-name = "either"
-version = "1.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
-
 [[package]]
 name = "encoding_rs"
 version = "0.8.31"
@@ -218,22 +113,6 @@ dependencies = [
  "cfg-if",
 ]
 
-[[package]]
-name = "exr"
-version = "1.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14cc0e06fb5f67e5d6beadf3a382fec9baca1aa751c6d5368fdeee7e5932c215"
-dependencies = [
- "bit_field",
- "deflate",
- "flume",
- "half",
- "inflate",
- "lebe",
- "smallvec",
- "threadpool",
-]
-
 [[package]]
 name = "fastrand"
 version = "1.7.0"
@@ -243,29 +122,6 @@ dependencies = [
  "instant",
 ]
 
-[[package]]
-name = "flate2"
-version = "1.0.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
-dependencies = [
- "crc32fast",
- "miniz_oxide",
-]
-
-[[package]]
-name = "flume"
-version = "0.10.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843c03199d0c0ca54bc1ea90ac0d507274c28abcc4f691ae8b4eaa375087c76a"
-dependencies = [
- "futures-core",
- "futures-sink",
- "nanorand",
- "pin-project",
- "spin",
-]
-
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -386,29 +242,6 @@ dependencies = [
  "slab",
 ]
 
-[[package]]
-name = "getrandom"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
-dependencies = [
- "cfg-if",
- "js-sys",
- "libc",
- "wasi 0.10.2+wasi-snapshot-preview1",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "gif"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b"
-dependencies = [
- "color_quant",
- "weezl",
-]
-
 [[package]]
 name = "h2"
 version = "0.3.13"
@@ -428,12 +261,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "half"
-version = "1.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
-
 [[package]]
 name = "hashbrown"
 version = "0.11.2"
@@ -531,26 +358,6 @@ dependencies = [
  "unicode-normalization",
 ]
 
-[[package]]
-name = "image"
-version = "0.24.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28edd9d7bc256be2502e325ac0628bde30b7001b9b52e0abe31a1a9dc2701212"
-dependencies = [
- "bytemuck",
- "byteorder",
- "color_quant",
- "exr",
- "gif",
- "jpeg-decoder",
- "num-iter",
- "num-rational",
- "num-traits",
- "png",
- "scoped_threadpool",
- "tiff",
-]
-
 [[package]]
 name = "indexmap"
 version = "1.8.2"
@@ -561,15 +368,6 @@ dependencies = [
  "hashbrown",
 ]
 
-[[package]]
-name = "inflate"
-version = "0.4.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff"
-dependencies = [
- "adler32",
-]
-
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -591,15 +389,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
 
-[[package]]
-name = "jpeg-decoder"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9478aa10f73e7528198d75109c8be5cd7d15fb530238040148d5f9a22d4c5b3b"
-dependencies = [
- "rayon",
-]
-
 [[package]]
 name = "js-sys"
 version = "0.3.57"
@@ -615,12 +404,6 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
-[[package]]
-name = "lebe"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7efd1d698db0759e6ef11a7cd44407407399a910c774dd804c64c032da7826ff"
-
 [[package]]
 name = "libc"
 version = "0.2.126"
@@ -658,30 +441,12 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
-[[package]]
-name = "memoffset"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "mime"
 version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
-[[package]]
-name = "miniz_oxide"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc"
-dependencies = [
- "adler",
-]
-
 [[package]]
 name = "mio"
 version = "0.8.3"
@@ -690,19 +455,10 @@ checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799"
 dependencies = [
  "libc",
  "log",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi",
  "windows-sys",
 ]
 
-[[package]]
-name = "nanorand"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
-dependencies = [
- "getrandom",
-]
-
 [[package]]
 name = "native-tls"
 version = "0.2.10"
@@ -721,47 +477,6 @@ dependencies = [
  "tempfile",
 ]
 
-[[package]]
-name = "num-integer"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
-dependencies = [
- "autocfg",
- "num-traits",
-]
-
-[[package]]
-name = "num-iter"
-version = "0.1.43"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-rational"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-traits"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "num_cpus"
 version = "1.13.1"
@@ -858,26 +573,6 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
-[[package]]
-name = "pin-project"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e"
-dependencies = [
- "pin-project-internal",
-]
-
-[[package]]
-name = "pin-project-internal"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "pin-project-lite"
 version = "0.2.9"
@@ -896,18 +591,6 @@ version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
 
-[[package]]
-name = "png"
-version = "0.17.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc38c0ad57efb786dd57b9864e5b18bae478c00c824dc55a38bbc9da95dde3ba"
-dependencies = [
- "bitflags",
- "crc32fast",
- "deflate",
- "miniz_oxide",
-]
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.39"
@@ -926,30 +609,6 @@ dependencies = [
  "proc-macro2",
 ]
 
-[[package]]
-name = "rayon"
-version = "1.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d"
-dependencies = [
- "autocfg",
- "crossbeam-deque",
- "either",
- "rayon-core",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f"
-dependencies = [
- "crossbeam-channel",
- "crossbeam-deque",
- "crossbeam-utils",
- "num_cpus",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.2.13"
@@ -1022,12 +681,11 @@ dependencies = [
 ]
 
 [[package]]
-name = "rusty-wgdl"
+name = "rusty-downloader"
 version = "0.1.0"
 dependencies = [
  "clap",
  "futures",
- "image",
  "regex",
  "reqwest",
  "serde_json",
@@ -1050,12 +708,6 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "scoped_threadpool"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8"
-
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -1145,15 +797,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "spin"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c530c2b0d0bf8b69304b39fe2001993e267461948b890cd037d8ad4293fa1a0d"
-dependencies = [
- "lock_api",
-]
-
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -1200,26 +843,6 @@ version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
 
-[[package]]
-name = "threadpool"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
-dependencies = [
- "num_cpus",
-]
-
-[[package]]
-name = "tiff"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cfada0986f446a770eca461e8c6566cb879682f7d687c8348aa0c857bd52286"
-dependencies = [
- "flate2",
- "jpeg-decoder",
- "weezl",
-]
-
 [[package]]
 name = "tinyvec"
 version = "1.6.0"
@@ -1383,12 +1006,6 @@ dependencies = [
  "try-lock",
 ]
 
-[[package]]
-name = "wasi"
-version = "0.10.2+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
-
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -1471,12 +1088,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "weezl"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c97e489d8f836838d497091de568cf16b117486d529ec5579233521065bd5e4"
-
 [[package]]
 name = "winapi"
 version = "0.3.9"
diff --git a/Cargo.toml b/Cargo.toml
index 4e0f67a..68518de 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "rusty-wgdl"
+name = "rusty-downloader"
 version = "0.1.0"
 edition = "2021"
 
@@ -12,4 +12,3 @@ reqwest = { version = "0.11", features = ["json"] }
 tokio = { version = "1.18.2", features = ["full"] }
 serde_json = "1.0"
 futures = "0.3"
-image = "0.24.2"
diff --git a/src/board.rs b/src/board.rs
new file mode 100644
index 0000000..f0a2169
--- /dev/null
+++ b/src/board.rs
@@ -0,0 +1,73 @@
+use serde_json::Value;
+use std::path::PathBuf;
+
+type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
+
+pub fn parse_url(url: &str) -> (String, String) {
+    let url_split: Vec<&str> = url.split("/").collect();
+    let board_name = url_split.get(url_split.len() - 2).unwrap();
+
+    (
+        format!("https://a.4cdn.org/{}/catalog.json", board_name),
+        board_name.to_string(),
+    )
+}
+
+// TODO: use futures
+pub async fn get_imagelist(
+    json_url: &str,
+    board_name: &str,
+    output_path: &PathBuf,
+) -> Result<(Vec<(String, PathBuf)>, usize)> {
+    let mut img_data: Vec<(String, PathBuf)> = Vec::new();
+    let board_data = get_threadlist(json_url, board_name).await?;
+
+    for thread_url in &board_data {
+        let res_body = reqwest::get(thread_url).await?.text().await?;
+        let json_data: Value = serde_json::from_str(res_body.as_str())?;
+
+        json_data["posts"]
+            .as_array()
+            .unwrap()
+            .iter()
+            .filter(|post| post["tim"].is_i64())
+            .for_each(|post| {
+                let id = post["tim"].to_string();
+                let ext = post["ext"].as_str().unwrap().to_string();
+                let filepath = output_path.join(format!("{}{}", id, ext).as_str());
+
+                println!("{:#?}", filepath);
+
+                img_data.push((
+                    format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext),
+                    filepath,
+                ));
+            });
+    }
+
+    Ok((img_data, board_data.len()))
+}
+
+async fn get_threadlist(json_url: &str, board_name: &str) -> Result<Vec<String>> {
+    let req_body = reqwest::get(json_url).await?.text().await?;
+    let json_data: Value = serde_json::from_str(req_body.as_str())?;
+    let board: Vec<Value> = json_data
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|page| page["threads"].clone())
+        .collect();
+
+    let mut board_data: Vec<String> = Vec::new();
+    board.iter().for_each(|thread_arr| {
+        thread_arr.as_array().unwrap().iter().for_each(|thread| {
+            let url = format!(
+                "https://a.4cdn.org/{}/thread/{}.json",
+                board_name, thread["no"]
+            );
+            board_data.push(url);
+        });
+    });
+
+    Ok(board_data)
+}
diff --git a/src/downloader.rs b/src/downloader.rs
new file mode 100644
index 0000000..7a5ec03
--- /dev/null
+++ b/src/downloader.rs
@@ -0,0 +1,33 @@
+use futures::{stream, StreamExt};
+use reqwest::Client;
+use std::path::PathBuf;
+use tokio::{fs::File, io::AsyncWriteExt};
+
+type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
+
+pub async fn download_images(img_data: &Vec<(String, PathBuf)>) -> Result<usize> {
+    let client = Client::builder().build()?;
+
+    let futures = stream::iter(img_data.iter().map(|data| async {
+        let (url, path) = data;
+        let send_fut = client.get(url).send();
+
+        match send_fut.await {
+            Ok(res) => match res.bytes().await {
+                Ok(bytes) => {
+                    let mut file = File::create(path).await.unwrap();
+                    file.write_all(&bytes).await.unwrap();
+                    println!("{} bytes from {:?} to {:?}", bytes.len(), &url, &path);
+                }
+                Err(_) => eprintln!("Error reading bytes from {}", url),
+            },
+            Err(_) => eprintln!("Error downloading {}", url),
+        }
+    }))
+    .buffer_unordered(100)
+    .collect::<Vec<()>>();
+
+    futures.await;
+
+    Ok(img_data.len())
+}
diff --git a/src/main.rs b/src/main.rs
index 0453820..97dc921 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,15 +1,15 @@
-use clap::{Arg, ArgGroup, Command};
-use futures::{stream, StreamExt};
-use regex::Regex;
-use reqwest::Client;
-use serde_json::Value;
-use std::{path::PathBuf, process::exit};
-use tokio::{fs::File, io::AsyncWriteExt};
+mod board;
+mod downloader;
+mod thread;
+
+use clap::{Arg, ArgGroup, Command};
+use regex::Regex;
+use std::{path::PathBuf, process::exit};
+use tokio;
 
-// General type to make error handling easier
 type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug)]
 enum Mode {
     Thread,
     Board,
@@ -73,105 +73,32 @@ fn parse_cli_args() -> Result<(PathBuf, String, Mode)> {
     Ok((path, String::from(target), mode))
 }
 
-fn create_thread_url(target: String) -> (String, String) {
-    let url_vec: Vec<&str> = target.split("/").collect();
-    let thread_id = url_vec.get(url_vec.len() - 1).unwrap();
-    let board = url_vec.get(url_vec.len() - 3).unwrap();
-
-    (
-        format!("https://a.4cdn.org/{}/thread/{}.json", board, thread_id),
-        board.to_string(),
-    )
-}
-
-//fn _create_board_url(target: String) -> String {
-//    let url_vec: Vec<&str> = target.split("/").collect();
-//    let board = url_vec.get(url_vec.len()).unwrap();
-//
-//    format!("https://a.4cdn.org/{}/catalog.json", board)
-//}
-
-async fn get_imagelist(
-    json_url: &str,
-    board_name: &str,
-    output_path: &PathBuf,
-) -> Result<Vec<(String, PathBuf)>> {
-    let req_body = reqwest::get(json_url).await?.text().await?;
-    let json_data: Value = serde_json::from_str(req_body.as_str())?;
-
-    let mut thread_img_data: Vec<(String, PathBuf)> = Vec::new();
-    json_data["posts"]
-        .as_array()
-        .unwrap()
-        .iter()
-        .filter(|post| post["tim"].is_i64())
-        .for_each(|post| {
-            let id = post["tim"].to_string();
-            let ext = post["ext"].as_str().unwrap().to_string();
-            let filepath = output_path.join(format!("{}{}", id, ext).as_str());
-
-            thread_img_data.push((
-                format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext),
-                filepath,
-            ))
-        });
-
-    Ok(thread_img_data)
-}
-
-async fn download_images(input_output_data: &Vec<(String, PathBuf)>) -> Result<u16> {
-    let client = Client::builder().build()?;
-
-    let futures = stream::iter(input_output_data.iter().map(|data| async {
-        let (url, path) = data;
-        let send_fut = client.get(url).send();
-
-        match send_fut.await {
-            Ok(res) => match res.bytes().await {
-                Ok(bytes) => {
-                    let mut file = File::create(path).await.unwrap();
-                    file.write_all(&bytes).await.unwrap();
-
-                    println!("Got {} bytes from {:?} to {:?}", bytes.len(), &url, &path)
-                }
-                Err(_) => eprintln!("Error reading bytes from {}", url),
-            },
-            Err(_) => eprintln!("Error downloading {}", url),
-        }
-    }))
-    .buffer_unordered(100)
-    .collect::<Vec<()>>();
-
-    futures.await;
-
-    Ok(0)
-}
-
 #[tokio::main]
 async fn main() -> Result<()> {
     let (path, target, mode) = parse_cli_args()?;
-
     println!(
-        "\nDownload configuration:\n\tOUTPUT: {:?}\n\tURL: {}\n\tDOWNLOAD-MODE: {:?}\n",
+        "\nDownload configuration:\n\tOUTPUT PATH: {:?}\n\tURL: {}\n\tDOWNLOAD MODE: {:?}\n",
         path, target, mode
     );
 
     match mode {
         Mode::Thread => {
-            let (json_url, board_name) = create_thread_url(target);
-            let targets = get_imagelist(&json_url.as_str(), &board_name.as_str(), &path).await?;
-            let _total_file_amount = download_images(&targets).await?;
-
-            //println!("Total files downloaded: {}", total_file_amount);
+            let (json_url, board_name) = thread::parse_url(&target);
+            let img_data = thread::get_imagelist(&json_url, &board_name, &path).await?;
+            let total_amt = downloader::download_images(&img_data).await?;
+            println!("Total of {} files downloaded from 1 thread.\n", total_amt);
         }
-        //Mode::Board => {
-        //    let json_url = create_board_url(target);
-        //    let id_list = get_imagelist(&json_url.as_str()).await?;
-        //    //println!("{:#?}", id_list);
-        //    get_images(id_list).await?;
-        //}
-        _ => println!("Not supported"),
-    };
+        Mode::Board => {
+            let (json_url, board_name) = board::parse_url(&target);
+            let (img_data, thread_amt) =
+                board::get_imagelist(&json_url, &board_name, &path).await?;
+            let total_amt = downloader::download_images(&img_data).await?;
+            println!(
+                "Total of {} files downloaded from {} threads.\n",
+                total_amt, thread_amt
+            );
+        }
+    }
 
     Ok(())
 }
diff --git a/src/thread.rs b/src/thread.rs
new file mode 100644
index 0000000..efb82e3
--- /dev/null
+++ b/src/thread.rs
@@ -0,0 +1,46 @@
+use serde_json::Value;
+use std::path::PathBuf;
+
+type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
+
+pub fn parse_url(url: &str) -> (String, String) {
+    let url_split: Vec<&str> = url.split("/").collect();
+    let thread_id = url_split.get(url_split.len() - 1).unwrap();
+    let board_name = url_split.get(url_split.len() - 3).unwrap();
+
+    (
+        format!(
+            "https://a.4cdn.org/{}/thread/{}.json",
+            board_name, thread_id
+        ),
+        board_name.to_string(),
+    )
+}
+
+pub async fn get_imagelist(
+    json_url: &str,
+    board_name: &str,
+    output_path: &PathBuf,
+) -> Result<Vec<(String, PathBuf)>> {
+    let req_body = reqwest::get(json_url).await?.text().await?;
+    let json_data: Value = serde_json::from_str(req_body.as_str())?;
+
+    let mut img_data: Vec<(String, PathBuf)> = Vec::new();
+    json_data["posts"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter(|post| post["tim"].is_i64())
+        .for_each(|post| {
+            let id = post["tim"].to_string();
+            let ext = post["ext"].as_str().unwrap().to_string();
+            let filepath = output_path.join(format!("{}{}", id, ext).as_str());
+
+            img_data.push((
+                format!("https://i.4cdn.org/{}/{}{}", board_name, id, ext),
+                filepath,
+            ))
+        });
+
+    Ok(img_data)
+}