scraping/skallwar.suckit
1
0
Fork 0
mirror of https://github.com/Skallwar/suckit.git synced 2026-09-10 07:16:19 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

coding-style: Run cargo fmt

Browse source
This commit is contained in:
cohenarthur 2020-04-29 16:29:48 +02:00
parent af4eb66c50
commit 53888bb582
5 changed files with 121 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

105
Cargo.lock generated
View file

@ -75,7 +75,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2"
dependencies = [
"num-integer",
"num-traits",
"num-traits 0.2.11",
"time",
]
@ -200,7 +200,7 @@ checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
dependencies = [
"cssparser-macros",
"dtoa-short",
"itoa",
"itoa 0.4.5",
"matches",
"phf",
"proc-macro2",
@ -229,6 +229,16 @@ dependencies = [
"syn",
]
[[package]]
name = "ctrlc"
version = "3.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a4ba686dff9fa4c1c9636ce1010b0cf98ceb421361b0bb3d6faeec43bd217a7"
dependencies = [
"nix",
"winapi 0.3.8",
]
[[package]]
name = "derive_more"
version = "0.99.5"
@ -246,6 +256,12 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
[[package]]
name = "dtoa"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dd841b58510c9618291ffa448da2e4e0f699d984d436122372f446dae62263d"
[[package]]
name = "dtoa"
version = "0.4.5"
@ -258,7 +274,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59020b8513b76630c49d918c33db9f4c91638e7d3404a28084083b87e33f76f2"
dependencies = [
"dtoa",
"dtoa 0.4.5",
]
[[package]]
@ -443,7 +459,7 @@ checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9"
dependencies = [
"bytes",
"fnv",
"itoa",
"itoa 0.4.5",
]
[[package]]
@ -476,7 +492,7 @@ dependencies = [
"http",
"http-body",
"httparse",
"itoa",
"itoa 0.4.5",
"log",
"net2",
"pin-project",
@ -528,6 +544,12 @@ dependencies = [
"libc",
]
[[package]]
name = "itoa"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae3088ea4baeceb0284ee9eea42f591226e6beaecf65373e41b38d95a1b8e7a1"
[[package]]
name = "itoa"
version = "0.4.5"
@ -601,9 +623,9 @@ dependencies = [
"log",
"phf",
"phf_codegen",
"serde",
"serde 1.0.106",
"serde_derive",
"serde_json",
"serde_json 1.0.51",
"string_cache",
"string_cache_codegen",
"tendril",
@ -718,6 +740,19 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
[[package]]
name = "nix"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363"
dependencies = [
"bitflags",
"cc",
"cfg-if",
"libc",
"void",
]
[[package]]
name = "nodrop"
version = "0.1.14"
@ -731,7 +766,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba"
dependencies = [
"autocfg",
"num-traits",
"num-traits 0.2.11",
]
[[package]]
name = "num-traits"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
dependencies = [
"num-traits 0.2.11",
]
[[package]]
@ -1056,7 +1100,7 @@ dependencies = [
"native-tls",
"percent-encoding",
"pin-project-lite",
"serde",
"serde 1.0.106",
"serde_urlencoded",
"time",
"tokio",
@ -1133,6 +1177,12 @@ dependencies = [
"thin-slice",
]
[[package]]
name = "serde"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8"
[[package]]
name = "serde"
version = "1.0.106"
@ -1150,15 +1200,27 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_json"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67f7d2e9edc3523a9c8ec8cd6ec481b3a27810aafee3e625d311febd3e656b4c"
dependencies = [
"dtoa 0.2.2",
"itoa 0.1.1",
"num-traits 0.1.43",
"serde 0.8.23",
]
[[package]]
name = "serde_json"
version = "1.0.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9"
dependencies = [
"itoa",
"itoa 0.4.5",
"ryu",
"serde",
"serde 1.0.106",
]
[[package]]
@ -1167,9 +1229,9 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97"
dependencies = [
"dtoa",
"itoa",
"serde",
"dtoa 0.4.5",
"itoa 0.4.5",
"serde 1.0.106",
"url",
]
@ -1217,7 +1279,7 @@ dependencies = [
"new_debug_unreachable",
"phf_shared",
"precomputed-hash",
"serde",
"serde 1.0.106",
]
[[package]]
@ -1269,9 +1331,12 @@ dependencies = [
"chrono",
"colored",
"crossbeam",
"ctrlc",
"kuchiki",
"pretty_assertions",
"reqwest",
"serde 1.0.106",
"serde_json 0.8.6",
"structopt",
]
@ -1482,6 +1547,12 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
[[package]]
name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
[[package]]
name = "want"
version = "0.3.0"
@ -1505,8 +1576,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cc57ce05287f8376e998cbddfb4c8cb43b84a7ec55cf4551d7c00eef317a47f"
dependencies = [
"cfg-if",
"serde",
"serde_json",
"serde 1.0.106",
"serde_json 1.0.51",
"wasm-bindgen-macro",
]

4
Cargo.toml
View file

@ -14,3 +14,7 @@ pretty_assertions = "0.6"
kuchiki = "0.8"
colored = "1.9"
chrono = "0.4"
ctrlc = "3.1"
serde = "1.0"
serde_json = "0.8"

12
src/logger.rs
View file

@ -1,10 +1,9 @@
use colored::*;
use chrono::Local;
use colored::*;
/// Write a message using the following format
/// <time>: [<header>] <message>
pub struct Logger {
}
pub struct Logger {}
// Allow the dead code in case we don't use all the macros. Using it here means
// it doesn't affect the other modules in the crate
@ -34,7 +33,12 @@ impl Logger {
/// Display an ERROR message
pub fn error(message: String) {
eprintln!("{}: [{}] {}", Logger::get_timestamp(), "ERROR".red(), message);
eprintln!(
"{}: [{}] {}",
Logger::get_timestamp(),
"ERROR".red(),
message
);
}
}

3
src/main.rs
View file

@ -2,8 +2,8 @@ mod args;
mod disk;
mod dom;
mod downloader;
mod scraper;
mod logger;
mod scraper;
use scraper::Scraper;
@ -13,5 +13,4 @@ fn main() {
let mut scraper = Scraper::new(args);
scraper.run();
}

21
src/scraper.rs
View file

@ -1,9 +1,11 @@
use crossbeam::channel::{Receiver, Sender, TryRecvError};
use crossbeam::thread;
use reqwest::Url;
use serde::{Deserialize, Serialize, Serializer};
use std::collections::HashMap;
use std::collections::HashSet;
use std::vec::Vec;
use std::sync::Mutex;
use std::time;
@ -41,7 +43,7 @@ impl Scraper {
Scraper {
downloader: downloader::Downloader::new(args.tries),
args: args,
args,
transmitter: tx,
receiver: rx,
visited_urls: Mutex::new(HashSet::new()),
@ -168,6 +170,23 @@ impl Scraper {
}
}
impl Serialize for Scraper {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let path_map = self.path_map.lock().unwrap();
let visited_urls: Vec<&String> = self.visited_urls
.lock()
.unwrap()
.iter()
.collect();
serializer.collect_map(path_map.iter())
}
}
#[cfg(test)]
mod tests {
use super::*;