scraping/skallwar.suckit
1
0
Fork 0
mirror of https://github.com/Skallwar/suckit.git synced 2026-09-10 07:16:19 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

multithreading: Add framework for multithreading

Browse source 
Add scraping with 2 threads instead of one !
This commit is contained in:
cohenarthur 2020-04-16 01:15:33 +02:00
parent 7a97fd7564
commit 1e80a86e74
3 changed files with 121 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
Cargo.lock generated
View file

@ -99,6 +99,27 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
[[package]]
name = "crossbeam-channel"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061"
dependencies = [
"crossbeam-utils",
"maybe-uninit",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg",
"cfg-if",
"lazy_static",
]
[[package]]
name = "cssparser"
version = "0.27.2"
@ -522,6 +543,12 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
[[package]]
name = "maybe-uninit"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]]
name = "memchr"
version = "2.3.3"
@ -1133,7 +1160,9 @@ dependencies = [
name = "suckit"
version = "0.1.0"
dependencies = [
"crossbeam-channel",
"kuchiki",
"lazy_static",
"pretty_assertions",
"reqwest",
"structopt",

2
Cargo.toml
View file

@ -8,6 +8,8 @@ edition = "2018"
[dependencies]
structopt = "0.3"
crossbeam-channel = "0.4"
lazy_static = "1.4"
reqwest = { version = "0.10", features = ["blocking"] }
pretty_assertions = "0.6"
kuchiki = "0.8"

153
src/scraper.rs
View file

@ -1,8 +1,8 @@
use reqwest::Url;
use lazy_static::lazy_static;
use crossbeam_channel::{Receiver, Sender};
use std::collections::HashMap;
use std::sync::mpsc;
use std::sync::mpsc::{Receiver, Sender};
use std::thread;
#[cfg(not(test))] //For the "mock" at the end of file
@ -14,6 +14,11 @@ use super::dom;
static DEFAULT_CAPACITY: usize = 128;
lazy_static! {
// FIXME: Get number of tries given to scraper
static ref DOWNLOADER: downloader::Downloader = downloader::Downloader::new(5);
}
/// Producer and Consumer data structure. Handles the incoming requests and
/// adds more as new URLs are found
pub struct Scraper {
@ -21,20 +26,18 @@ pub struct Scraper {
transmitter: Sender<Url>,
receiver: Receiver<Url>,
visited_urls: HashMap<String, String>,
downloader: downloader::Downloader,
depth_level: usize,
}
impl Scraper {
/// Create a new scraper with command line options
pub fn new(args: args::Args) -> Scraper {
let (tx, rx) = mpsc::channel();
let (tx, rx) = crossbeam_channel::unbounded();
let mut scraper = Scraper {
visited_urls: HashMap::new(),
transmitter: tx,
receiver: rx,
downloader: downloader::Downloader::new(args.tries),
args: args,
depth_level: 0,
};
@ -42,7 +45,7 @@ impl Scraper {
scraper
}
fn handle_url(tx: Sender<Url>, url: Url, downloader: Downloader) {
fn handle_url(transmitter: &Sender<Url>, url: Url, downloader: &downloader::Downloader) {
let page = downloader.get(&url).unwrap();
let dom = dom::Dom::new(&page);
@ -54,17 +57,24 @@ impl Scraper {
for new_url_string in new_urls {
let new_full_url = url.join(&new_url_string).unwrap();
self.transmitter.send(new_full_url.clone());
match transmitter.send(new_full_url.clone()) {
Ok(_) => {},
Err(e) => panic!("{}", e),
};
/*
new_url_string.clear();
new_url_string
.push_str(self.visited_urls.get(new_full_url.as_str()).unwrap());
*/
}
/*
disk::save_file(
self.visited_urls.get(url.as_str()).unwrap(),
&dom.serialize(),
&self.args.output,
);
*/
println!("{} has been downloaded", url);
}
@ -72,7 +82,10 @@ impl Scraper {
/// Run through the channel and complete it
pub fn run(&mut self) {
/* Push the origin URL through the channel */
self.transmitter.send(self.args.origin.clone());
match self.transmitter.send(self.args.origin.clone()) {
Ok(_) => {},
Err(e) => panic!("{}", e),
};
let tx0 = self.transmitter.clone();
let tx1 = self.transmitter.clone();
@ -80,78 +93,92 @@ impl Scraper {
let rx0 = self.receiver.clone();
let rx1 = self.receiver.clone();
thread::spawn(move || {
let t0 = thread::spawn(move || {
loop {
match rx0.recv().unwrap() {
Err() => continue, // FIXME: Sleep
match rx0.recv() {
Err(_) => continue, // FIXME: Sleep
Ok(url) => {
handle_url(tx0, url);
Scraper::handle_url(&tx0, url, &DOWNLOADER);
}
};
}
});
let t1 = thread::spawn(move || {
loop {
match rx1.recv() {
Err(_) => continue, // FIXME: Sleep
Ok(url) => {
Scraper::handle_url(&tx1, url, &DOWNLOADER);
}
};
}
});
t0.join();
t1.join();
}
/* Use wrappers functions for consistency */
/*
fn push_depth_delimiter(&mut self) {
self.queue.push_back(None);
fn push_depth_delimiter(&mut self) {
self.queue.push_back(None);
}
fn queue_init(&mut self, url: Url) {
//Entry point + depth delimiter
self.push(url);
self.push_depth_delimiter();
}
fn push(&mut self, url: Url) {
match self.visited_urls.contains_key(url.as_str()) {
false => {
self.visited_urls
.insert(url.to_string(), disk::url_to_path(&url));
self.queue.push_back(Some(url));
}
true => (),
}
}
fn pop(&mut self) -> Option<Url> {
//Only a depth delimiter remaining
if self.queue.len() == 1 {
return None;
}
fn queue_init(&mut self, url: Url) {
//Entry point + depth delimiter
self.push(url);
self.push_depth_delimiter();
}
fn push(&mut self, url: Url) {
match self.visited_urls.contains_key(url.as_str()) {
false => {
self.visited_urls
.insert(url.to_string(), disk::url_to_path(&url));
self.queue.push_back(Some(url));
match self.queue.pop_front() {
Some(url) => match url {
Some(url) => Some(url),
None => {
self.depth_level += 1;
self.push_depth_delimiter();
self.pop()
}
true => (),
}
}
fn pop(&mut self) -> Option<Url> {
//Only a depth delimiter remaining
if self.queue.len() == 1 {
return None;
}
match self.queue.pop_front() {
Some(url) => match url {
Some(url) => Some(url),
None => {
self.depth_level += 1;
self.push_depth_delimiter();
self.pop()
}
},
None => None,
}
}
fn should_visit(url: &str, base: &Url) -> bool {
match Url::parse(url) {
/* The given candidate is a valid URL, and not a relative path to
* the next one. Therefore, we have to check if this URL belongs
* to the same domain as our current URL. If the candidate has the
* same domain as our base, then we should visit it */
Ok(not_ok) => not_ok.domain() == base.domain(),
/* Since we couldn't parse this "URL", then it must be a relative
* path or a malformed URL. If the URL is malformed, then it will
* be handled during the join() call in run() */
Err(_) => true,
}
},
None => None,
}
}
*/
/*
fn should_visit(url: &str, base: &Url) -> bool {
match Url::parse(url) {
/* The given candidate is a valid URL, and not a relative path to
* the next one. Therefore, we have to check if this URL belongs
* to the same domain as our current URL. If the candidate has the
* same domain as our base, then we should visit it */
Ok(not_ok) => not_ok.domain() == base.domain(),
/* Since we couldn't parse this "URL", then it must be a relative
* path or a malformed URL. If the URL is malformed, then it will
* be handled during the join() call in run() */
Err(_) => true,
}
}
/*
/// Handle an URL
fn handle_url(&self, url: Url) {
let page = self.downloader.get(&url).unwrap();