Feature: Custom user agent as a CLI argument (#76)

* Allow passing user agent via cli args

* user-agent: Short and long CLI arguments

Co-authored-by: creativcoder <rahul.sharma@atherenergy.com>
Co-authored-by: Esteban Blanc <estblcsk@gmail.com>

src/args.rs

@@ -37,6 +37,10 @@ pub struct Args {
     /// The max seconds of delay between downloads
     #[structopt(long, default_value = "0", help="Generate an extra random delay between downloads, from 0 to this number. This is added to the base delay seconds")]
     pub random_range: u64,
+
+    /// User agent to be used to send requests
+    #[structopt(short, long, default_value = "suckit", help="User agent to be used for sending requests")]
+    pub user_agent: String,
 }
 
 impl Args {

src/downloader.rs

@@ -6,18 +6,19 @@ use crate::warn;
 ///A Downloader to download web content
 pub struct Downloader {
     client: reqwest::blocking::Client,
-    tries: usize,
+    tries: usize
 }
 
 impl Downloader {
     /// Create a new Downloader
-    pub fn new(tries: usize) -> Downloader {
+    pub fn new(tries: usize, user_agent: &str) -> Downloader {
         Downloader {
             client: reqwest::blocking::ClientBuilder::new()
                 .cookie_store(true)
+                .user_agent(user_agent)
                 .build()
                 .unwrap(),
-            tries,
+            tries
         }
     }
 
@@ -92,7 +93,7 @@ mod tests {
     #[test]
     fn test_download_url() {
         let url: Url = Url::parse("https://lwn.net").unwrap();
-        match Downloader::new(1).get(&url) {
+        match Downloader::new(1, "suckit").get(&url) {
             Err(e) => assert!(false, "Fail to download lwn.net: {:?}", e),
             _ => {}
         }

src/main.rs

@@ -3,9 +3,9 @@ mod disk;
 mod dom;
 mod downloader;
 mod logger;
+mod response;
 mod scraper;
 mod url_helper;
-mod response;
 
 use scraper::Scraper;
 

src/scraper.rs

@@ -46,7 +46,7 @@ impl Scraper {
         let (tx, rx) = crossbeam::channel::unbounded();
 
         Scraper {
-            downloader: downloader::Downloader::new(args.tries),
+            downloader: downloader::Downloader::new(args.tries, &args.user_agent),
             args,
             transmitter: tx,
             receiver: rx,
@@ -190,7 +190,9 @@ impl Scraper {
         let base_delay = self.args.delay;
         let random_range = self.args.random_range;
 
-        if base_delay == 0 && random_range == 0 { return; }
+        if base_delay == 0 && random_range == 0 {
+            return;
+        }
 
         // delay_range+1 because gen_range is exclusive on the upper limit
         let rand_delay_secs = rng.gen_range(0, random_range + 1);
@@ -225,11 +227,12 @@ mod tests {
         let args = args::Args {
             origin: Url::parse("https://example.com/").unwrap(),
             output: Some(PathBuf::from("/tmp")),
-            jobs:  1,
+            jobs: 1,
             tries: 1,
             depth: 5,
-            delay:   0,
-            random_range:  0,
+            delay: 0,
+            user_agent: "suckit".to_string(),
+            random_range: 0,
             verbose: true,
         };
 
@@ -241,11 +244,12 @@ mod tests {
         let args = args::Args {
             origin: Url::parse("https://example.com/").unwrap(),
             output: Some(PathBuf::from("/tmp")),
-            jobs:  1,
+            jobs: 1,
             tries: 1,
             depth: 5,
-            delay:   2,
-            random_range:  5,
+            delay: 2,
+            user_agent: "suckit".to_string(),
+            random_range: 5,
             verbose: true,
         };