scraping/skallwar.suckit
1
0
Fork 0
mirror of https://github.com/Skallwar/suckit.git synced 2026-09-10 07:16:19 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add error handling for persistent errors (#88)

Browse source 
* Add error handling for persistent errors

* lint file and add init for new parameter

* Update src/args.rs

Co-authored-by: CohenArthur <arthur.cohen@epita.fr>

* misc: Run cargo fmt

Co-authored-by: CohenArthur <arthur.cohen@epita.fr>
Co-authored-by: Esteban Blanc <estblcsk@gmail.com>
This commit is contained in:
Jakub Szymański 2020-07-02 15:24:42 +02:00 committed by GitHub
parent 397315b5f3
commit 8e6a5ebbb1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 38 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/args.rs
View file

@ -98,6 +98,10 @@ pub struct Args {
help = "Regex filter to exclude saving pages that match this expression"
)]
pub exclude: Regex,
/// Decides if we should bail out on download error (like, too many redirects)
#[structopt(short, long, help = "Flag to enable or disable exit on error")]
pub continue_on_error: bool,
}
impl Args {

57
src/scraper.rs
View file

@ -4,6 +4,7 @@ use url::Url;
use std::collections::HashMap;
use std::collections::HashSet;
use std::process;
use std::sync::Mutex;
use std::time;
@ -116,31 +117,39 @@ impl Scraper {
/// Process a single URL
fn handle_url(scraper: &Scraper, transmitter: &Sender<(Url, i32)>, url: Url, depth: i32) {
let response = scraper.downloader.get(&url).unwrap();
match scraper.downloader.get(&url) {
Ok(response) => {
let data = match response.data {
response::ResponseData::Html(data) => {
Scraper::handle_html(scraper, transmitter, &url, depth, &data)
}
response::ResponseData::Other(data) => data,
};
let data = match response.data {
response::ResponseData::Html(data) => {
Scraper::handle_html(scraper, transmitter, &url, depth, &data)
// Create a scope to unlock path_map automagicly
{
let path_map = scraper.path_map.lock().unwrap();
let path = path_map.get(url.as_str()).unwrap();
if !scraper.args.exclude.is_match(url.as_str())
&& scraper.args.include.is_match(url.as_str())
{
match response.filename {
Some(filename) => {
disk::save_file(&filename, &data, &scraper.args.output);
disk::symlink(path, &filename, &scraper.args.output);
}
None => {
disk::save_file(path, &data, &scraper.args.output);
}
}
}
}
}
response::ResponseData::Other(data) => data,
};
// Create a scope to unlock path_map automagicly
{
let path_map = scraper.path_map.lock().unwrap();
let path = path_map.get(url.as_str()).unwrap();
if !scraper.args.exclude.is_match(url.as_str())
&& scraper.args.include.is_match(url.as_str())
{
match response.filename {
Some(filename) => {
disk::save_file(&filename, &data, &scraper.args.output);
disk::symlink(path, &filename, &scraper.args.output);
}
None => {
disk::save_file(path, &data, &scraper.args.output);
}
Err(e) => {
println!("Couldn't download a page, {:?}", e);
if !scraper.args.continue_on_error {
process::exit(1);
}
}
}
@ -243,6 +252,7 @@ mod tests {
verbose: true,
include: Regex::new("jpg").unwrap(),
exclude: Regex::new("png").unwrap(),
continue_on_error: true,
};
let _ = Scraper::new(args);
@ -262,6 +272,7 @@ mod tests {
verbose: true,
include: Regex::new("jpg").unwrap(),
exclude: Regex::new("png").unwrap(),
continue_on_error: true,
};
let _ = Scraper::new(args);