mirror of
https://github.com/Skallwar/suckit.git
synced 2026-09-10 07:16:19 -04:00
Merge branch 'master' into add-logging
This commit is contained in:
cohenarthur
commit
debe1eca64
129
Cargo.lock
generated
129
Cargo.lock
generated
|
|
@ -121,6 +121,77 @@ version = "0.7.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-queue",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
"maybe-uninit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"maybe-uninit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"lazy_static",
|
||||
"maybe-uninit",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cssparser"
|
||||
version = "0.27.2"
|
||||
|
|
@ -343,9 +414,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.10"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e"
|
||||
checksum = "8a0d737e0f947a1864e93d33fdef4af8445a00d1ed8dc0c8ddb73139ea6abf15"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
|
@ -393,9 +464,9 @@ checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
|
|||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.13.4"
|
||||
version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed6081100e960d9d74734659ffc9cc91daf1c0fc7aceb8eaa94ee1a3f5046f2e"
|
||||
checksum = "96816e1d921eca64d208a85aab4f7798455a8e34229ee5a88c935bdee1b78b14"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
|
|
@ -502,9 +573,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
|||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.68"
|
||||
version = "0.2.69"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0"
|
||||
checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
|
|
@ -544,12 +615,27 @@ version = "0.1.8"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
|
||||
|
||||
[[package]]
|
||||
name = "maybe-uninit"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.16"
|
||||
|
|
@ -659,9 +745,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.12.0"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6"
|
||||
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
|
|
@ -771,18 +857,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "pin-project"
|
||||
version = "0.4.8"
|
||||
version = "0.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7804a463a8d9572f13453c516a5faea534a2403d7ced2f0c7e100eeff072772c"
|
||||
checksum = "6f6a7f5eee6292c559c793430c55c00aea9d3b3d1905e855806ca4d7253426a2"
|
||||
dependencies = [
|
||||
"pin-project-internal",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-internal"
|
||||
version = "0.4.8"
|
||||
version = "0.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f"
|
||||
checksum = "8988430ce790d8682672117bc06dda364c0be32d3abd738234f19f3240bad99a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -833,9 +919,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "proc-macro-error"
|
||||
version = "1.0.1"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8931031034aa65c73f3f1a05c3ec0fa51287fcd06557ecf4e88b2768bdca375e"
|
||||
checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678"
|
||||
dependencies = [
|
||||
"proc-macro-error-attr",
|
||||
"proc-macro2",
|
||||
|
|
@ -846,9 +932,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "proc-macro-error-attr"
|
||||
version = "1.0.1"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2147536f412ee7ae5529364ed50172ca0220fd64591e236296f45f36b38b2f98"
|
||||
checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -998,6 +1084,12 @@ dependencies = [
|
|||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "0.4.2"
|
||||
|
|
@ -1176,6 +1268,7 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"chrono",
|
||||
"colored",
|
||||
"crossbeam",
|
||||
"kuchiki",
|
||||
"pretty_assertions",
|
||||
"reqwest",
|
||||
|
|
@ -1257,9 +1350,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "0.2.17"
|
||||
version = "0.2.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39fb9142eb6e9cc37f4f29144e62618440b149a138eee01a7bbe9b9226aaf17c"
|
||||
checksum = "34ef16d072d2b6dc8b4a56c70f5c5ced1a37752116f8e7c1e80c659aa7cb6713"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ edition = "2018"
|
|||
|
||||
[dependencies]
|
||||
structopt = "0.3"
|
||||
crossbeam = "0.7"
|
||||
reqwest = { version = "0.10", features = ["blocking"] }
|
||||
pretty_assertions = "0.6"
|
||||
kuchiki = "0.8"
|
||||
|
|
|
|||
|
|
@ -8,6 +8,6 @@ your disk.
|
|||
# Features
|
||||
|
||||
* [ ] Vacuum the entirety of a website recursively
|
||||
* [ ] Use multithreading
|
||||
* [x] Use multithreading
|
||||
* [x] Write the website's content to your disk
|
||||
* [x] Offline navigation
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ pub struct Args {
|
|||
#[structopt(short, long, parse(from_os_str))]
|
||||
pub output: Option<PathBuf>,
|
||||
|
||||
#[structopt(short, long, default_value = "1")]
|
||||
pub jobs: usize,
|
||||
|
||||
#[structopt(short, long, default_value = "5")]
|
||||
pub depth: usize,
|
||||
|
||||
|
|
|
|||
288
src/scraper.rs
288
src/scraper.rs
|
|
@ -1,7 +1,11 @@
|
|||
use crossbeam::channel::{Receiver, Sender, TryRecvError};
|
||||
use crossbeam::thread;
|
||||
use reqwest::Url;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::VecDeque;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Mutex;
|
||||
use std::time;
|
||||
|
||||
#[cfg(not(test))] //For the "mock" at the end of file
|
||||
use super::downloader;
|
||||
|
|
@ -10,76 +14,140 @@ use super::args;
|
|||
use super::disk;
|
||||
use super::dom;
|
||||
|
||||
static DEFAULT_CAPACITY: usize = 128;
|
||||
/// Maximum number of empty recv() from the channel
|
||||
static MAX_EMPTY_RECEIVES: usize = 10;
|
||||
|
||||
/// Sleep duration on empty recv()
|
||||
static SLEEP_MILLIS: u64 = 100;
|
||||
static SLEEP_DURATION: time::Duration = time::Duration::from_millis(SLEEP_MILLIS);
|
||||
|
||||
/// Producer and Consumer data structure. Handles the incoming requests and
|
||||
/// adds more as new URLs are found
|
||||
pub struct Scraper {
|
||||
args: args::Args,
|
||||
queue: VecDeque<Option<Url>>,
|
||||
visited_urls: HashMap<String, String>,
|
||||
transmitter: Sender<(Url, usize)>,
|
||||
receiver: Receiver<(Url, usize)>,
|
||||
downloader: downloader::Downloader,
|
||||
depth_level: usize,
|
||||
visited_urls: Mutex<HashSet<String>>,
|
||||
path_map: Mutex<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
impl Scraper {
|
||||
/// Create a new scraper with command line options
|
||||
pub fn new(args: args::Args) -> Scraper {
|
||||
let mut scraper = Scraper {
|
||||
queue: VecDeque::with_capacity(DEFAULT_CAPACITY),
|
||||
visited_urls: HashMap::new(),
|
||||
let (tx, rx) = crossbeam::channel::unbounded();
|
||||
|
||||
Scraper {
|
||||
downloader: downloader::Downloader::new(args.tries),
|
||||
args: args,
|
||||
depth_level: 0,
|
||||
};
|
||||
|
||||
scraper.queue_init(scraper.args.origin.clone());
|
||||
|
||||
scraper
|
||||
transmitter: tx,
|
||||
receiver: rx,
|
||||
visited_urls: Mutex::new(HashSet::new()),
|
||||
path_map: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/* Use wrappers functions for consistency */
|
||||
/// Add an URL to the path_map HashMap
|
||||
fn map_url(&self, url: &Url, path: String) -> bool {
|
||||
let mut path_map = self.path_map.lock().unwrap();
|
||||
|
||||
fn push_depth_delimiter(&mut self) {
|
||||
self.queue.push_back(None);
|
||||
}
|
||||
|
||||
fn queue_init(&mut self, url: Url) {
|
||||
//Entry point + depth delimiter
|
||||
self.push(url);
|
||||
self.push_depth_delimiter();
|
||||
}
|
||||
|
||||
fn push(&mut self, url: Url) {
|
||||
match self.visited_urls.contains_key(url.as_str()) {
|
||||
match path_map.contains_key(url.as_str()) {
|
||||
false => {
|
||||
self.visited_urls
|
||||
.insert(url.to_string(), disk::url_to_path(&url));
|
||||
self.queue.push_back(Some(url));
|
||||
path_map.insert(url.to_string(), path);
|
||||
true
|
||||
}
|
||||
true => (),
|
||||
true => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn pop(&mut self) -> Option<Url> {
|
||||
//Only a depth delimiter remaining
|
||||
if self.queue.len() == 1 {
|
||||
return None;
|
||||
}
|
||||
|
||||
match self.queue.pop_front() {
|
||||
Some(url) => match url {
|
||||
Some(url) => Some(url),
|
||||
None => {
|
||||
self.depth_level += 1;
|
||||
self.push_depth_delimiter();
|
||||
self.pop()
|
||||
}
|
||||
},
|
||||
None => None,
|
||||
}
|
||||
/// Push a new URL into the channel
|
||||
fn push(transmitter: &Sender<(Url, usize)>, url: Url, depth: usize) {
|
||||
match transmitter.send((url, depth)) {
|
||||
Ok(_) => (),
|
||||
Err(e) => panic!("{}", e),
|
||||
};
|
||||
}
|
||||
|
||||
/// Fix the URLs contained in the DOM-tree so they point to each other
|
||||
fn fix_domtree(&self, old_url_str: &mut String, new_url: &Url) {
|
||||
let path_map = self.path_map.lock().unwrap();
|
||||
|
||||
old_url_str.clear();
|
||||
old_url_str.push_str(path_map.get(new_url.as_str()).unwrap());
|
||||
}
|
||||
|
||||
/// Process a single URL
|
||||
fn handle_url(scraper: &Scraper, transmitter: &Sender<(Url, usize)>, url: Url, depth: usize) {
|
||||
let page = scraper.downloader.get(&url).unwrap();
|
||||
let dom = dom::Dom::new(&page);
|
||||
|
||||
dom.find_urls_as_strings()
|
||||
.into_iter()
|
||||
.filter(|candidate| Scraper::should_visit(candidate, &url))
|
||||
.for_each(|next_url| {
|
||||
let next_full_url = url.join(&next_url).unwrap();
|
||||
match scraper.map_url(&next_full_url, disk::url_to_path(&next_full_url)) {
|
||||
true => {
|
||||
if depth < scraper.args.depth {
|
||||
Scraper::push(transmitter, next_full_url.clone(), depth + 1);
|
||||
}
|
||||
}
|
||||
false => (),
|
||||
};
|
||||
|
||||
scraper.fix_domtree(next_url, &next_full_url);
|
||||
});
|
||||
|
||||
let path_map = scraper.path_map.lock().unwrap();
|
||||
|
||||
disk::save_file(
|
||||
path_map.get(url.as_str()).unwrap(),
|
||||
&dom.serialize(),
|
||||
&scraper.args.output,
|
||||
);
|
||||
|
||||
scraper.visited_urls.lock().unwrap().insert(url.to_string());
|
||||
|
||||
println!("{} has been downloaded", url);
|
||||
}
|
||||
|
||||
/// Run through the channel and complete it
|
||||
pub fn run(&mut self) {
|
||||
/* Push the origin URL and depth (0) through the channel */
|
||||
self.map_url(&self.args.origin, disk::url_to_path(&self.args.origin));
|
||||
Scraper::push(&self.transmitter, self.args.origin.clone(), 0);
|
||||
|
||||
thread::scope(|thread_scope| {
|
||||
for _ in 0..self.args.jobs {
|
||||
let tx = self.transmitter.clone();
|
||||
let rx = self.receiver.clone();
|
||||
let self_clone = &self;
|
||||
|
||||
thread_scope.spawn(move |_| {
|
||||
let mut counter = 0;
|
||||
|
||||
while counter < MAX_EMPTY_RECEIVES {
|
||||
match rx.try_recv() {
|
||||
Err(e) => match e {
|
||||
TryRecvError::Empty => {
|
||||
counter += 1;
|
||||
std::thread::sleep(SLEEP_DURATION);
|
||||
}
|
||||
TryRecvError::Disconnected => panic!("{}", e),
|
||||
},
|
||||
Ok((url, depth)) => {
|
||||
counter = 0;
|
||||
Scraper::handle_url(&self_clone, &tx, url, depth);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// If a URL should be visited, or does it belong to another domain
|
||||
fn should_visit(url: &str, base: &Url) -> bool {
|
||||
match Url::parse(url) {
|
||||
/* The given candidate is a valid URL, and not a relative path to
|
||||
|
|
@ -94,44 +162,6 @@ impl Scraper {
|
|||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Run through the queue and complete it
|
||||
pub fn run(&mut self) {
|
||||
// TODO: Add multithreading handling
|
||||
loop {
|
||||
match self.pop() {
|
||||
None => break,
|
||||
Some(url) => {
|
||||
let page = self.downloader.get(&url).unwrap();
|
||||
let dom = dom::Dom::new(&page);
|
||||
|
||||
if self.depth_level < self.args.depth {
|
||||
let new_urls = dom.find_urls_as_strings();
|
||||
let new_urls = new_urls
|
||||
.into_iter()
|
||||
.filter(|candidate| Scraper::should_visit(candidate, &url));
|
||||
|
||||
for new_url_string in new_urls {
|
||||
let new_full_url = url.join(&new_url_string).unwrap();
|
||||
|
||||
self.push(new_full_url.clone());
|
||||
new_url_string.clear();
|
||||
new_url_string
|
||||
.push_str(self.visited_urls.get(new_full_url.as_str()).unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
disk::save_file(
|
||||
self.visited_urls.get(url.as_str()).unwrap(),
|
||||
&dom.serialize(),
|
||||
&self.args.output,
|
||||
);
|
||||
|
||||
println!("{} has been downloaded", url);
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -144,16 +174,12 @@ mod tests {
|
|||
let args = args::Args {
|
||||
origin: Url::parse("https://example.com/").unwrap(),
|
||||
output: Some(PathBuf::from("/tmp")),
|
||||
jobs: 1,
|
||||
tries: 1,
|
||||
depth: 5,
|
||||
};
|
||||
let mut s = Scraper::new(args);
|
||||
|
||||
assert_eq!(s.queue.len(), 2); //Base url + depth delimiter
|
||||
assert_eq!(
|
||||
s.queue.pop_front().unwrap().unwrap().to_string(),
|
||||
"https://example.com/"
|
||||
);
|
||||
let _ = Scraper::new(args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -161,19 +187,22 @@ mod tests {
|
|||
let args = args::Args {
|
||||
origin: Url::parse("https://fake_start.net/").unwrap(),
|
||||
output: Some(PathBuf::from("/tmp")),
|
||||
jobs: 1,
|
||||
tries: 1,
|
||||
depth: 5,
|
||||
};
|
||||
|
||||
let mut s = Scraper::new(args);
|
||||
|
||||
s.run();
|
||||
|
||||
assert!(!s.visited_urls.contains_key("https://example.net"));
|
||||
assert!(!s.visited_urls.contains_key("https://no-no-no.com"));
|
||||
assert!(s.visited_urls.contains_key("https://fake_start.net/a_file"));
|
||||
assert!(s
|
||||
.visited_urls
|
||||
.contains_key("https://fake_start.net/dir/nested/file"));
|
||||
let visited_urls = s.visited_urls.lock().unwrap();
|
||||
|
||||
assert!(!visited_urls.contains("https://example.net"));
|
||||
assert!(!visited_urls.contains("https://no-no-no.com"));
|
||||
assert!(visited_urls.contains("https://fake_start.net/a_file"));
|
||||
assert!(visited_urls.contains("https://fake_start.net/dir/nested/file"));
|
||||
assert!(visited_urls.contains("https://fake_start.net/an_answer_file"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -181,6 +210,7 @@ mod tests {
|
|||
let args = args::Args {
|
||||
origin: Url::parse("https://fake_start.net/").unwrap(),
|
||||
output: Some(PathBuf::from("/tmp")),
|
||||
jobs: 1,
|
||||
tries: 1,
|
||||
depth: 0,
|
||||
};
|
||||
|
|
@ -188,23 +218,52 @@ mod tests {
|
|||
|
||||
s.run();
|
||||
|
||||
assert!(!s.visited_urls.contains_key("https://example.net"));
|
||||
assert!(!s.visited_urls.contains_key("https://no-no-no.com"));
|
||||
assert!(!s.visited_urls.contains_key("https://fake_start.net/a_file"));
|
||||
assert!(!s
|
||||
.visited_urls
|
||||
.contains_key("https://fake_start.net/dir/nested/file"));
|
||||
let visited_urls = s.visited_urls.lock().unwrap();
|
||||
|
||||
assert!(!visited_urls.contains("https://example.net"));
|
||||
assert!(!visited_urls.contains("https://no-no-no.com"));
|
||||
assert!(!visited_urls.contains("https://fake_start.net/a_file"));
|
||||
assert!(!visited_urls.contains("https://fake_start.net/an_answer_file"));
|
||||
assert!(!visited_urls.contains("https://fake_start.net/dir/nested/file"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn depth_tricky() {
|
||||
let args = args::Args {
|
||||
origin: Url::parse("https://fake_start.net/").unwrap(),
|
||||
output: Some(PathBuf::from("/tmp")),
|
||||
jobs: 1,
|
||||
tries: 1,
|
||||
depth: 1,
|
||||
};
|
||||
let mut s = Scraper::new(args);
|
||||
|
||||
s.run();
|
||||
|
||||
let visited_urls = s.visited_urls.lock().unwrap();
|
||||
|
||||
assert!(visited_urls.contains("https://fake_start.net/a_file"));
|
||||
assert!(visited_urls.contains("https://fake_start.net/dir/nested/file"));
|
||||
assert!(!visited_urls.contains("https://fake_start.net/an_answer_file"));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod downloader {
|
||||
static SIMPLE_BODY: &str = "<!DOCTYPE html>
|
||||
static TEST_BEG: &str = "<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<p>Absolute <a href=\"https://no-no-no.com\"></a></p>
|
||||
<p>Relative <a href=\"a_file\"></a></p>
|
||||
<p>Relative nested <a href=\"dir/nested/file\"></a></p>
|
||||
<p>Absolute<a href=\"https://no-no-no.com\"></a></p>
|
||||
<p>Relative<a href=\"a_file\"></a></p>
|
||||
<p>Relative nested<a href=\"dir/nested/file\"></a></p>
|
||||
</body>
|
||||
</html>
|
||||
";
|
||||
|
||||
static TEST_ANS: &str = "<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<p>Relative<a href=\"an_answer_file\"></a></p>
|
||||
</body>
|
||||
</html>
|
||||
";
|
||||
|
|
@ -217,10 +276,19 @@ mod downloader {
|
|||
}
|
||||
|
||||
pub fn get(&self, url: &reqwest::Url) -> Result<String, reqwest::Error> {
|
||||
let mut res = String::from("");
|
||||
|
||||
match url.as_str() == "https://fake_start.net/" {
|
||||
true => Ok(String::from(SIMPLE_BODY)),
|
||||
false => Ok(String::from("")),
|
||||
true => res = String::from(TEST_BEG),
|
||||
false => {}
|
||||
}
|
||||
|
||||
match url.as_str() == "https://fake_start.net/a_file" {
|
||||
true => res = String::from(TEST_ANS),
|
||||
false => {}
|
||||
}
|
||||
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
8
tests/speed.csv
Normal file
8
tests/speed.csv
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
1, 1, 1
|
||||
92.01026630401611, 45.85903334617615, 24.938209056854248
|
||||
92.68794660568237, 45.51665663719177, 36.877896785736084
|
||||
94.26381759643554, 48.01623530387879, 39.034280157089235
|
||||
94.51657667160035, 47.43879041671753, 25.296046495437622
|
||||
94.91624474525452, 47.318924236297605, 38.720614719390866
|
||||
94.61654992103577, 57.59067335128784, 24.995196104049683
|
||||
95.28456358909607, 46.86330976486206, 24.708560609817503
|
||||
|
123
tests/speed_regression.py
Executable file
123
tests/speed_regression.py
Executable file
|
|
@ -0,0 +1,123 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
# Number of tests for each bench-set
|
||||
TEST_RETRIES = 5
|
||||
|
||||
# File to store the results
|
||||
FILENAME = "speed.csv"
|
||||
|
||||
# Path to the suckit binary
|
||||
SUCKIT = "suckit"
|
||||
|
||||
# URL to download
|
||||
URL = "http://books.toscrape.com"
|
||||
|
||||
# Path to store the downloaded data
|
||||
PATH = "/tmp/suckit_speed"
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from termcolor import colored
|
||||
|
||||
def parse_args():
|
||||
global FILENAME
|
||||
global SUCKIT
|
||||
|
||||
parser = argparse.ArgumentParser(description = "SuckIT benchmark")
|
||||
|
||||
parser.add_argument("-f", "--filename", action = "store", type = str, help = f"result file (default_value = '{FILENAME}')")
|
||||
parser.add_argument("-s", "--suckit", action = "store", type = str, help = f"path to the suckit binary (default_value = '{SUCKIT}')")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.filename:
|
||||
FILENAME = args.filename
|
||||
|
||||
if args.suckit:
|
||||
SUCKIT = args.suckit
|
||||
|
||||
def load_best_result(filename):
|
||||
try:
|
||||
with open(filename, "r") as results:
|
||||
res = [float('inf'), float('inf'), float('inf')]
|
||||
# There has to be at least a line of '1's in the file
|
||||
line = results.readlines()[-1]
|
||||
|
||||
reader = csv.reader([line], delimiter = ",")
|
||||
for row in reader:
|
||||
for i in range(len(row)):
|
||||
candidate = float(row[i])
|
||||
if candidate < res[i]:
|
||||
res[i] = candidate
|
||||
|
||||
except IOError:
|
||||
print(f"Could not read file {filename}")
|
||||
exit(1)
|
||||
|
||||
return res
|
||||
|
||||
def write_new_result(filename, result):
|
||||
try:
|
||||
with open(filename, "a") as csv:
|
||||
csv.write("{}, {}, {}\n".format(result[0], result[1], result[2]))
|
||||
except IOError:
|
||||
print(f"Could not write to file {filename}")
|
||||
exit(1)
|
||||
|
||||
def compute_new_result():
|
||||
thread_counts = ["1", "2", "4"]
|
||||
res = []
|
||||
|
||||
for count in thread_counts:
|
||||
time_total = 0
|
||||
for i in range(TEST_RETRIES):
|
||||
start_time = time.time()
|
||||
|
||||
suckit_pid = subprocess.Popen([SUCKIT, "-j", count, "-o", PATH, URL],
|
||||
stdout = open("/dev/null", "w"), shell = False)
|
||||
suckit_pid.wait()
|
||||
|
||||
end_time = time.time()
|
||||
|
||||
time_total += end_time - start_time
|
||||
print(f"Completed {i + 1} iteration for job with {count} thread(s)", end = "\r")
|
||||
|
||||
res.append(time_total / TEST_RETRIES)
|
||||
print("")
|
||||
|
||||
return res
|
||||
|
||||
def main():
|
||||
parse_args()
|
||||
|
||||
# If the directory already exists, just ignore it
|
||||
try:
|
||||
os.mkdir(PATH)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
test_names = ["Single thread", "Two threads", "Four threads"]
|
||||
old_result = load_best_result(FILENAME)
|
||||
|
||||
new_result = compute_new_result()
|
||||
|
||||
for i in range(0, len(test_names)):
|
||||
speed_up = new_result[i] * 100 / old_result[i] - 100;
|
||||
|
||||
str_speed_up = f"{colored(speed_up, 'green')}"
|
||||
|
||||
if speed_up > 0:
|
||||
str_speed_up = f"{colored(speed_up, 'red')}"
|
||||
|
||||
print(f"{test_names[i]} was {str_speed_up} slower")
|
||||
|
||||
write_new_result(FILENAME, new_result)
|
||||
|
||||
shutil.rmtree(PATH)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in a new issue