diff --git a/src/args.rs b/src/args.rs index 636dc25..6b13cb6 100644 --- a/src/args.rs +++ b/src/args.rs @@ -37,6 +37,10 @@ pub struct Args { /// The max seconds of delay between downloads #[structopt(long, default_value = "0", help="Generate an extra random delay between downloads, from 0 to this number. This is added to the base delay seconds")] pub random_range: u64, + + /// User agent to be used to send requests + #[structopt(short, long, default_value = "suckit", help="User agent to be used for sending requests")] + pub user_agent: String, } impl Args { diff --git a/src/downloader.rs b/src/downloader.rs index d151646..7dc2180 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -6,18 +6,19 @@ use crate::warn; ///A Downloader to download web content pub struct Downloader { client: reqwest::blocking::Client, - tries: usize, + tries: usize } impl Downloader { /// Create a new Downloader - pub fn new(tries: usize) -> Downloader { + pub fn new(tries: usize, user_agent: &str) -> Downloader { Downloader { client: reqwest::blocking::ClientBuilder::new() .cookie_store(true) + .user_agent(user_agent) .build() .unwrap(), - tries, + tries } } @@ -92,7 +93,7 @@ mod tests { #[test] fn test_download_url() { let url: Url = Url::parse("https://lwn.net").unwrap(); - match Downloader::new(1).get(&url) { + match Downloader::new(1, "suckit").get(&url) { Err(e) => assert!(false, "Fail to download lwn.net: {:?}", e), _ => {} } diff --git a/src/main.rs b/src/main.rs index 2c2fdcb..fd985cd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,9 +3,9 @@ mod disk; mod dom; mod downloader; mod logger; +mod response; mod scraper; mod url_helper; -mod response; use scraper::Scraper; diff --git a/src/scraper.rs b/src/scraper.rs index fd20327..5bcaff1 100644 --- a/src/scraper.rs +++ b/src/scraper.rs @@ -46,7 +46,7 @@ impl Scraper { let (tx, rx) = crossbeam::channel::unbounded(); Scraper { - downloader: downloader::Downloader::new(args.tries), + downloader: downloader::Downloader::new(args.tries, &args.user_agent), args, transmitter: tx, receiver: rx, @@ -190,7 +190,9 @@ impl Scraper { let base_delay = self.args.delay; let random_range = self.args.random_range; - if base_delay == 0 && random_range == 0 { return; } + if base_delay == 0 && random_range == 0 { + return; + } // delay_range+1 because gen_range is exclusive on the upper limit let rand_delay_secs = rng.gen_range(0, random_range + 1); @@ -225,11 +227,12 @@ mod tests { let args = args::Args { origin: Url::parse("https://example.com/").unwrap(), output: Some(PathBuf::from("/tmp")), - jobs: 1, + jobs: 1, tries: 1, depth: 5, - delay: 0, - random_range: 0, + delay: 0, + user_agent: "suckit".to_string(), + random_range: 0, verbose: true, }; @@ -241,11 +244,12 @@ mod tests { let args = args::Args { origin: Url::parse("https://example.com/").unwrap(), output: Some(PathBuf::from("/tmp")), - jobs: 1, + jobs: 1, tries: 1, depth: 5, - delay: 2, - random_range: 5, + delay: 2, + user_agent: "suckit".to_string(), + random_range: 5, verbose: true, };