diff --git a/Cargo.lock b/Cargo.lock index 786a6c6..cbe7c93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,6 +121,77 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" +[[package]] +name = "crossbeam" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" +dependencies = [ + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static", + "maybe-uninit", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if", + "lazy_static", +] + [[package]] name = "cssparser" version = "0.27.2" @@ -343,9 +414,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e" +checksum = "8a0d737e0f947a1864e93d33fdef4af8445a00d1ed8dc0c8ddb73139ea6abf15" dependencies = [ "libc", ] @@ -393,9 +464,9 @@ checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" [[package]] name = "hyper" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6081100e960d9d74734659ffc9cc91daf1c0fc7aceb8eaa94ee1a3f5046f2e" +checksum = "96816e1d921eca64d208a85aab4f7798455a8e34229ee5a88c935bdee1b78b14" dependencies = [ "bytes", "futures-channel", @@ -502,9 +573,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0" +checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" [[package]] name = "log" @@ -544,12 +615,27 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "memchr" version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +[[package]] +name = "memoffset" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -659,9 +745,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" dependencies = [ "hermit-abi", "libc", @@ -771,18 +857,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7804a463a8d9572f13453c516a5faea534a2403d7ced2f0c7e100eeff072772c" +checksum = "6f6a7f5eee6292c559c793430c55c00aea9d3b3d1905e855806ca4d7253426a2" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f" +checksum = "8988430ce790d8682672117bc06dda364c0be32d3abd738234f19f3240bad99a" dependencies = [ "proc-macro2", "quote", @@ -833,9 +919,9 @@ dependencies = [ [[package]] name = "proc-macro-error" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8931031034aa65c73f3f1a05c3ec0fa51287fcd06557ecf4e88b2768bdca375e" +checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" dependencies = [ "proc-macro-error-attr", "proc-macro2", @@ -846,9 +932,9 @@ dependencies = [ [[package]] name = "proc-macro-error-attr" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2147536f412ee7ae5529364ed50172ca0220fd64591e236296f45f36b38b2f98" +checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" dependencies = [ "proc-macro2", "quote", @@ -998,6 +1084,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "security-framework" version = "0.4.2" @@ -1176,6 +1268,7 @@ version = "0.1.0" dependencies = [ "chrono", "colored", + "crossbeam", "kuchiki", "pretty_assertions", "reqwest", @@ -1257,9 +1350,9 @@ dependencies = [ [[package]] name = "tokio" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39fb9142eb6e9cc37f4f29144e62618440b149a138eee01a7bbe9b9226aaf17c" +checksum = "34ef16d072d2b6dc8b4a56c70f5c5ced1a37752116f8e7c1e80c659aa7cb6713" dependencies = [ "bytes", "fnv", diff --git a/Cargo.toml b/Cargo.toml index 907c38f..575bb25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" [dependencies] structopt = "0.3" +crossbeam = "0.7" reqwest = { version = "0.10", features = ["blocking"] } pretty_assertions = "0.6" kuchiki = "0.8" diff --git a/README.md b/README.md index e06a7ab..66127fc 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,6 @@ your disk. # Features * [ ] Vacuum the entirety of a website recursively -* [ ] Use multithreading +* [x] Use multithreading * [x] Write the website's content to your disk * [x] Offline navigation diff --git a/src/args.rs b/src/args.rs index 5afb1b3..c129627 100644 --- a/src/args.rs +++ b/src/args.rs @@ -11,6 +11,9 @@ pub struct Args { #[structopt(short, long, parse(from_os_str))] pub output: Option, + #[structopt(short, long, default_value = "1")] + pub jobs: usize, + #[structopt(short, long, default_value = "5")] pub depth: usize, diff --git a/src/scraper.rs b/src/scraper.rs index 6359b36..695e635 100644 --- a/src/scraper.rs +++ b/src/scraper.rs @@ -1,7 +1,11 @@ +use crossbeam::channel::{Receiver, Sender, TryRecvError}; +use crossbeam::thread; use reqwest::Url; use std::collections::HashMap; -use std::collections::VecDeque; +use std::collections::HashSet; +use std::sync::Mutex; +use std::time; #[cfg(not(test))] //For the "mock" at the end of file use super::downloader; @@ -10,76 +14,140 @@ use super::args; use super::disk; use super::dom; -static DEFAULT_CAPACITY: usize = 128; +/// Maximum number of empty recv() from the channel +static MAX_EMPTY_RECEIVES: usize = 10; + +/// Sleep duration on empty recv() +static SLEEP_MILLIS: u64 = 100; +static SLEEP_DURATION: time::Duration = time::Duration::from_millis(SLEEP_MILLIS); /// Producer and Consumer data structure. Handles the incoming requests and /// adds more as new URLs are found pub struct Scraper { args: args::Args, - queue: VecDeque>, - visited_urls: HashMap, + transmitter: Sender<(Url, usize)>, + receiver: Receiver<(Url, usize)>, downloader: downloader::Downloader, - depth_level: usize, + visited_urls: Mutex>, + path_map: Mutex>, } impl Scraper { /// Create a new scraper with command line options pub fn new(args: args::Args) -> Scraper { - let mut scraper = Scraper { - queue: VecDeque::with_capacity(DEFAULT_CAPACITY), - visited_urls: HashMap::new(), + let (tx, rx) = crossbeam::channel::unbounded(); + + Scraper { downloader: downloader::Downloader::new(args.tries), args: args, - depth_level: 0, - }; - - scraper.queue_init(scraper.args.origin.clone()); - - scraper + transmitter: tx, + receiver: rx, + visited_urls: Mutex::new(HashSet::new()), + path_map: Mutex::new(HashMap::new()), + } } - /* Use wrappers functions for consistency */ + /// Add an URL to the path_map HashMap + fn map_url(&self, url: &Url, path: String) -> bool { + let mut path_map = self.path_map.lock().unwrap(); - fn push_depth_delimiter(&mut self) { - self.queue.push_back(None); - } - - fn queue_init(&mut self, url: Url) { - //Entry point + depth delimiter - self.push(url); - self.push_depth_delimiter(); - } - - fn push(&mut self, url: Url) { - match self.visited_urls.contains_key(url.as_str()) { + match path_map.contains_key(url.as_str()) { false => { - self.visited_urls - .insert(url.to_string(), disk::url_to_path(&url)); - self.queue.push_back(Some(url)); + path_map.insert(url.to_string(), path); + true } - true => (), + true => false, } } - fn pop(&mut self) -> Option { - //Only a depth delimiter remaining - if self.queue.len() == 1 { - return None; - } - - match self.queue.pop_front() { - Some(url) => match url { - Some(url) => Some(url), - None => { - self.depth_level += 1; - self.push_depth_delimiter(); - self.pop() - } - }, - None => None, - } + /// Push a new URL into the channel + fn push(transmitter: &Sender<(Url, usize)>, url: Url, depth: usize) { + match transmitter.send((url, depth)) { + Ok(_) => (), + Err(e) => panic!("{}", e), + }; } + /// Fix the URLs contained in the DOM-tree so they point to each other + fn fix_domtree(&self, old_url_str: &mut String, new_url: &Url) { + let path_map = self.path_map.lock().unwrap(); + + old_url_str.clear(); + old_url_str.push_str(path_map.get(new_url.as_str()).unwrap()); + } + + /// Process a single URL + fn handle_url(scraper: &Scraper, transmitter: &Sender<(Url, usize)>, url: Url, depth: usize) { + let page = scraper.downloader.get(&url).unwrap(); + let dom = dom::Dom::new(&page); + + dom.find_urls_as_strings() + .into_iter() + .filter(|candidate| Scraper::should_visit(candidate, &url)) + .for_each(|next_url| { + let next_full_url = url.join(&next_url).unwrap(); + match scraper.map_url(&next_full_url, disk::url_to_path(&next_full_url)) { + true => { + if depth < scraper.args.depth { + Scraper::push(transmitter, next_full_url.clone(), depth + 1); + } + } + false => (), + }; + + scraper.fix_domtree(next_url, &next_full_url); + }); + + let path_map = scraper.path_map.lock().unwrap(); + + disk::save_file( + path_map.get(url.as_str()).unwrap(), + &dom.serialize(), + &scraper.args.output, + ); + + scraper.visited_urls.lock().unwrap().insert(url.to_string()); + + println!("{} has been downloaded", url); + } + + /// Run through the channel and complete it + pub fn run(&mut self) { + /* Push the origin URL and depth (0) through the channel */ + self.map_url(&self.args.origin, disk::url_to_path(&self.args.origin)); + Scraper::push(&self.transmitter, self.args.origin.clone(), 0); + + thread::scope(|thread_scope| { + for _ in 0..self.args.jobs { + let tx = self.transmitter.clone(); + let rx = self.receiver.clone(); + let self_clone = &self; + + thread_scope.spawn(move |_| { + let mut counter = 0; + + while counter < MAX_EMPTY_RECEIVES { + match rx.try_recv() { + Err(e) => match e { + TryRecvError::Empty => { + counter += 1; + std::thread::sleep(SLEEP_DURATION); + } + TryRecvError::Disconnected => panic!("{}", e), + }, + Ok((url, depth)) => { + counter = 0; + Scraper::handle_url(&self_clone, &tx, url, depth); + } + } + } + }); + } + }) + .unwrap(); + } + + /// If a URL should be visited, or does it belong to another domain fn should_visit(url: &str, base: &Url) -> bool { match Url::parse(url) { /* The given candidate is a valid URL, and not a relative path to @@ -94,44 +162,6 @@ impl Scraper { Err(_) => true, } } - - /// Run through the queue and complete it - pub fn run(&mut self) { - // TODO: Add multithreading handling - loop { - match self.pop() { - None => break, - Some(url) => { - let page = self.downloader.get(&url).unwrap(); - let dom = dom::Dom::new(&page); - - if self.depth_level < self.args.depth { - let new_urls = dom.find_urls_as_strings(); - let new_urls = new_urls - .into_iter() - .filter(|candidate| Scraper::should_visit(candidate, &url)); - - for new_url_string in new_urls { - let new_full_url = url.join(&new_url_string).unwrap(); - - self.push(new_full_url.clone()); - new_url_string.clear(); - new_url_string - .push_str(self.visited_urls.get(new_full_url.as_str()).unwrap()); - } - } - - disk::save_file( - self.visited_urls.get(url.as_str()).unwrap(), - &dom.serialize(), - &self.args.output, - ); - - println!("{} has been downloaded", url); - } - }; - } - } } #[cfg(test)] @@ -144,16 +174,12 @@ mod tests { let args = args::Args { origin: Url::parse("https://example.com/").unwrap(), output: Some(PathBuf::from("/tmp")), + jobs: 1, tries: 1, depth: 5, }; - let mut s = Scraper::new(args); - assert_eq!(s.queue.len(), 2); //Base url + depth delimiter - assert_eq!( - s.queue.pop_front().unwrap().unwrap().to_string(), - "https://example.com/" - ); + let _ = Scraper::new(args); } #[test] @@ -161,19 +187,22 @@ mod tests { let args = args::Args { origin: Url::parse("https://fake_start.net/").unwrap(), output: Some(PathBuf::from("/tmp")), + jobs: 1, tries: 1, depth: 5, }; + let mut s = Scraper::new(args); s.run(); - assert!(!s.visited_urls.contains_key("https://example.net")); - assert!(!s.visited_urls.contains_key("https://no-no-no.com")); - assert!(s.visited_urls.contains_key("https://fake_start.net/a_file")); - assert!(s - .visited_urls - .contains_key("https://fake_start.net/dir/nested/file")); + let visited_urls = s.visited_urls.lock().unwrap(); + + assert!(!visited_urls.contains("https://example.net")); + assert!(!visited_urls.contains("https://no-no-no.com")); + assert!(visited_urls.contains("https://fake_start.net/a_file")); + assert!(visited_urls.contains("https://fake_start.net/dir/nested/file")); + assert!(visited_urls.contains("https://fake_start.net/an_answer_file")); } #[test] @@ -181,6 +210,7 @@ mod tests { let args = args::Args { origin: Url::parse("https://fake_start.net/").unwrap(), output: Some(PathBuf::from("/tmp")), + jobs: 1, tries: 1, depth: 0, }; @@ -188,23 +218,52 @@ mod tests { s.run(); - assert!(!s.visited_urls.contains_key("https://example.net")); - assert!(!s.visited_urls.contains_key("https://no-no-no.com")); - assert!(!s.visited_urls.contains_key("https://fake_start.net/a_file")); - assert!(!s - .visited_urls - .contains_key("https://fake_start.net/dir/nested/file")); + let visited_urls = s.visited_urls.lock().unwrap(); + + assert!(!visited_urls.contains("https://example.net")); + assert!(!visited_urls.contains("https://no-no-no.com")); + assert!(!visited_urls.contains("https://fake_start.net/a_file")); + assert!(!visited_urls.contains("https://fake_start.net/an_answer_file")); + assert!(!visited_urls.contains("https://fake_start.net/dir/nested/file")); + } + + #[test] + fn depth_tricky() { + let args = args::Args { + origin: Url::parse("https://fake_start.net/").unwrap(), + output: Some(PathBuf::from("/tmp")), + jobs: 1, + tries: 1, + depth: 1, + }; + let mut s = Scraper::new(args); + + s.run(); + + let visited_urls = s.visited_urls.lock().unwrap(); + + assert!(visited_urls.contains("https://fake_start.net/a_file")); + assert!(visited_urls.contains("https://fake_start.net/dir/nested/file")); + assert!(!visited_urls.contains("https://fake_start.net/an_answer_file")); } } #[cfg(test)] mod downloader { - static SIMPLE_BODY: &str = " + static TEST_BEG: &str = " -

Absolute

-

Relative

-

Relative nested

+

Absolute

+

Relative

+

Relative nested

+ + +"; + + static TEST_ANS: &str = " + + +

Relative

"; @@ -217,10 +276,19 @@ mod downloader { } pub fn get(&self, url: &reqwest::Url) -> Result { + let mut res = String::from(""); + match url.as_str() == "https://fake_start.net/" { - true => Ok(String::from(SIMPLE_BODY)), - false => Ok(String::from("")), + true => res = String::from(TEST_BEG), + false => {} } + + match url.as_str() == "https://fake_start.net/a_file" { + true => res = String::from(TEST_ANS), + false => {} + } + + Ok(res) } } } diff --git a/tests/speed.csv b/tests/speed.csv new file mode 100644 index 0000000..13f42e6 --- /dev/null +++ b/tests/speed.csv @@ -0,0 +1,8 @@ +1, 1, 1 +92.01026630401611, 45.85903334617615, 24.938209056854248 +92.68794660568237, 45.51665663719177, 36.877896785736084 +94.26381759643554, 48.01623530387879, 39.034280157089235 +94.51657667160035, 47.43879041671753, 25.296046495437622 +94.91624474525452, 47.318924236297605, 38.720614719390866 +94.61654992103577, 57.59067335128784, 24.995196104049683 +95.28456358909607, 46.86330976486206, 24.708560609817503 diff --git a/tests/speed_regression.py b/tests/speed_regression.py new file mode 100755 index 0000000..ef5685a --- /dev/null +++ b/tests/speed_regression.py @@ -0,0 +1,123 @@ +#!/usr/bin/python3 + +# Number of tests for each bench-set +TEST_RETRIES = 5 + +# File to store the results +FILENAME = "speed.csv" + +# Path to the suckit binary +SUCKIT = "suckit" + +# URL to download +URL = "http://books.toscrape.com" + +# Path to store the downloaded data +PATH = "/tmp/suckit_speed" + +import argparse +import csv +import os +import shutil +import subprocess +import time +from termcolor import colored + +def parse_args(): + global FILENAME + global SUCKIT + + parser = argparse.ArgumentParser(description = "SuckIT benchmark") + + parser.add_argument("-f", "--filename", action = "store", type = str, help = f"result file (default_value = '{FILENAME}')") + parser.add_argument("-s", "--suckit", action = "store", type = str, help = f"path to the suckit binary (default_value = '{SUCKIT}')") + + args = parser.parse_args() + + if args.filename: + FILENAME = args.filename + + if args.suckit: + SUCKIT = args.suckit + +def load_best_result(filename): + try: + with open(filename, "r") as results: + res = [float('inf'), float('inf'), float('inf')] + # There has to be at least a line of '1's in the file + line = results.readlines()[-1] + + reader = csv.reader([line], delimiter = ",") + for row in reader: + for i in range(len(row)): + candidate = float(row[i]) + if candidate < res[i]: + res[i] = candidate + + except IOError: + print(f"Could not read file {filename}") + exit(1) + + return res + +def write_new_result(filename, result): + try: + with open(filename, "a") as csv: + csv.write("{}, {}, {}\n".format(result[0], result[1], result[2])) + except IOError: + print(f"Could not write to file {filename}") + exit(1) + +def compute_new_result(): + thread_counts = ["1", "2", "4"] + res = [] + + for count in thread_counts: + time_total = 0 + for i in range(TEST_RETRIES): + start_time = time.time() + + suckit_pid = subprocess.Popen([SUCKIT, "-j", count, "-o", PATH, URL], + stdout = open("/dev/null", "w"), shell = False) + suckit_pid.wait() + + end_time = time.time() + + time_total += end_time - start_time + print(f"Completed {i + 1} iteration for job with {count} thread(s)", end = "\r") + + res.append(time_total / TEST_RETRIES) + print("") + + return res + +def main(): + parse_args() + + # If the directory already exists, just ignore it + try: + os.mkdir(PATH) + except OSError: + pass + + test_names = ["Single thread", "Two threads", "Four threads"] + old_result = load_best_result(FILENAME) + + new_result = compute_new_result() + + for i in range(0, len(test_names)): + speed_up = new_result[i] * 100 / old_result[i] - 100; + + str_speed_up = f"{colored(speed_up, 'green')}" + + if speed_up > 0: + str_speed_up = f"{colored(speed_up, 'red')}" + + print(f"{test_names[i]} was {str_speed_up} slower") + + write_new_result(FILENAME, new_result) + + shutil.rmtree(PATH) + +if __name__ == "__main__": + main()