doc: Add documentation (#54)

* doc: Add documentation part 1

* doc: More doc

* doc: src/args.rs

Co-authored-by: CohenArthur <arthur.cohen@epita.fr>

* doc: src/downloader.rs

Co-authored-by: CohenArthur <arthur.cohen@epita.fr>

* doc: src/url_helper.rs

Co-authored-by: CohenArthur <arthur.cohen@epita.fr>

* doc: src/url_helper.rs

Co-authored-by: CohenArthur <arthur.cohen@epita.fr>

Co-authored-by: CohenArthur <arthur.cohen@epita.fr>

src/args.rs

@@ -3,28 +3,36 @@ use std::path::PathBuf;
 use structopt::StructOpt;
 use url::Url;
 
+///CLI arguments
 #[derive(Debug, StructOpt)]
 pub struct Args {
+    ///Entry point of scraping
     #[structopt(name = "url", parse(try_from_str))]
     pub origin: Url,
 
+    ///Output directory
     #[structopt(short, long, parse(from_os_str))]
     pub output: Option<PathBuf>,
 
+    ///Number of threads/workers
     #[structopt(short, long, default_value = "1")]
     pub jobs: usize,
 
+    ///Max depth of scraping recursion
     #[structopt(short, long, default_value = "5")]
     pub depth: usize,
 
+    ///Number of retries when downloading a page fails
     #[structopt(short, long, default_value = "20")]
     pub tries: usize,
 
+    ///Show logs
     #[structopt(short, long)]
     pub verbose: bool,
 }
 
 impl Args {
+    ///Collect args
     pub fn collect() -> Args {
         Args::from_args()
     }

src/disk.rs

@@ -4,6 +4,7 @@ use std::path::PathBuf;
 
 use crate::{error, warn};
 
+///Save content in a file
 pub fn save_file(file_name: &str, content: &[u8], path: &Option<PathBuf>) {
     let path = match path {
         Some(path) => path.join(file_name),
@@ -29,6 +30,7 @@ pub fn save_file(file_name: &str, content: &[u8], path: &Option<PathBuf>) {
     }
 }
 
+///Create a symlink
 pub fn symlink(source: &str, destination: &str, path: &Option<PathBuf>) {
     let destination = match path {
         Some(path) => path.join(destination),

src/dom.rs

@@ -13,12 +13,14 @@ pub struct Dom {
 }
 
 impl Dom {
+    ///Create a new dom tree
     pub fn new(str: &str) -> Dom {
         Dom {
             tree: kuchiki::parse_html().one(str),
         }
     }
 
+    ///Serialize the dom tree
     pub fn serialize(&self) -> String {
         let mut vec: Vec<u8> = Vec::new();
 
@@ -29,6 +31,7 @@ impl Dom {
         String::from_utf8(vec).unwrap()
     }
 
+    ///Returns all urls in the dom tree
     pub fn find_urls_as_strings(&self) -> Vec<&mut String> {
         let mut vec: Vec<&mut String> = Vec::new();
 

src/downloader.rs

@@ -3,7 +3,7 @@ use url::Url;
 
 use crate::warn;
 
-/// Wrapper around a reqwest client, used to get the content of web pages
+///A Downloader to download web content
 pub struct Downloader {
     client: reqwest::blocking::Client,
     tries: usize,
@@ -21,6 +21,7 @@ impl Downloader {
         }
     }
 
+    ///Check if the type in the 'content-type' head field is html
     fn is_html(content_type: &str) -> bool {
         content_type.contains("text/html")
     }
@@ -37,6 +38,7 @@ impl Downloader {
         }
     }
 
+    ///Download the content at this url
     fn make_request(&self, url: &Url) -> Result<Response, reqwest::Error> {
         match self.client.get(url.clone()).send() {
             Ok(mut data) => {
@@ -69,7 +71,7 @@ impl Downloader {
         }
     }
 
-    /// Download the content located at a given URL
+    ///Download the content of an url and retries at most 'tries' times on failure
     pub fn get(&self, url: &Url) -> Result<Response, reqwest::Error> {
         let mut error: Option<reqwest::Error> = None;
         for _ in 0..self.tries {

src/response.rs

@@ -11,6 +11,7 @@ pub struct Response {
 }
 
 impl Response {
+    ///Create a new Response
     pub fn new(data: ResponseData, filename: Option<String>) -> Response {
         Response { data, filename }
     }

src/scraper.rs

@@ -80,6 +80,7 @@ impl Scraper {
         old_url_str.push_str(&new_url_str);
     }
 
+    ///Proces an html file: add new url to the chanel and prepare for offline navigation
     fn handle_html(
         scraper: &Scraper,
         transmitter: &Sender<(Url, usize)>,

src/url_helper.rs

@@ -1,13 +1,17 @@
 use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
 use url::Url;
 
+///Max file name size supported by the file system
 const FILE_NAME_MAX_LENGTH: usize = 255;
+///Characters that need to be replaced by encode()
 const FRAGMENT: &AsciiSet = &CONTROLS.add(b'?');
 
+///Encode special character with '%' representation
 pub fn encode(path: &str) -> String {
     utf8_percent_encode(path, FRAGMENT).to_string()
 }
 
+///Convert an Url to the corresponding path
 pub fn to_path(url: &Url) -> String {
     let url = url.as_str().split("://").collect::<Vec<&str>>()[1];