Patches for #110 (#111)

* scrapper: Log an error if an url is not in the path map

* url_helper: Fix domain/1/ vs domain/1 file or dir conflict

* url_helper: Fix to_path()

Cargo.lock

@@ -691,6 +691,12 @@ version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
 
+[[package]]
+name = "md5"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
+
 [[package]]
 name = "memchr"
 version = "2.3.4"
@@ -1508,7 +1514,7 @@ dependencies = [
 
 [[package]]
 name = "suckit"
-version = "0.1.0"
+version = "0.1.1"
 dependencies = [
  "chrono",
  "colored",
@@ -1516,6 +1522,7 @@ dependencies = [
  "encoding_rs",
  "kuchiki",
  "lazy_static",
+ "md5",
  "pathdiff",
  "rand 0.8.1",
  "regex",

Cargo.toml

@@ -34,6 +34,7 @@ regex = "^1.4"
 encoding_rs = "^0.8"
 lazy_static = "1.4.0"
 pathdiff = "^0.2"
+md5 = "^0.7"
 
 [dev-dependencies]
 tiny_http = "^0.7"

src/scraper.rs

@@ -167,13 +167,10 @@ impl Scraper {
         };
 
         let dom = dom::Dom::new(&String::from_utf8_lossy(&utf8_data).into_owned());
-        let source_path = scraper
-            .path_map
-            .lock()
-            .unwrap()
-            .get(url.as_str())
-            .unwrap()
-            .clone();
+        let source_path = match scraper.path_map.lock().unwrap().get(url.as_str()) {
+            Some(path) => path.clone(),
+            None => error!("Url {} was not found in the path map", url.as_str()),
+        };
 
         dom.find_urls_as_strings()
             .into_iter()

src/url_helper.rs

@@ -1,22 +1,64 @@
+use std::path::Path;
+
+use md5;
 use url::Url;
 
+///Max file name size supported by the file system
+const FILE_NAME_MAX_LENGTH: usize = 255;
+
 /// Convert an Url to the corresponding path
 pub fn to_path(url: &Url) -> String {
-    let domain = url.host_str().unwrap();
-    let path = url.path();
+    let url_domain = url.host_str().unwrap();
+    let url_path = url.path();
+    let url_query = url.query();
 
-    let mut path = format!("{}{}", domain, path);
-    if path.ends_with("/") {
-        path = format!("{}index.html", path);
+    let path = Path::new(url_path);
+    let mut filename = path.file_name().map_or(String::from(""), |filename| {
+        filename.to_str().unwrap().to_string()
+    });
+    let mut parent = path
+        .parent()
+        .map_or("", |filename| filename.to_str().unwrap());
+
+    if url_path.ends_with("/") {
+        filename = "index.html".to_string();
+        parent = url_path.trim_end_matches("/");
+    } else if path.extension().is_none() {
+        parent = url_path;
+        filename = "index_no_slash.html".to_string();
     }
 
-    path
+    if url_query.is_some() {
+        filename.push('?');
+        filename.push_str(url_query.unwrap());
+    }
+
+    if filename.len() > FILE_NAME_MAX_LENGTH {
+        let digest = md5::compute(filename);
+        filename = format!("{:x}.html", digest);
+    }
+
+    format!("{}{}/{}", url_domain, parent, filename)
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    fn url_to_path_domain_only() {
+        let str = super::to_path(&Url::parse("https://lwn.net/").unwrap());
+
+        assert_eq!(str, "lwn.net/index.html");
+    }
+
+    #[test]
+    fn url_to_path_domain_only_no_slash() {
+        let str = super::to_path(&Url::parse("https://lwn.net").unwrap());
+
+        assert_eq!(str, "lwn.net/index.html");
+    }
+
     #[test]
     fn url_to_path() {
         let str = super::to_path(&Url::parse("https://lwn.net/Kernel/index.html").unwrap());
@@ -31,10 +73,24 @@ mod tests {
         assert_eq!(str, "lwn.net/Kernel/index.html");
     }
 
+    #[test]
+    fn url_to_path_index_no_slash() {
+        let str = super::to_path(&Url::parse("https://lwn.net/Kernel").unwrap());
+
+        assert_eq!(str, "lwn.net/Kernel/index_no_slash.html");
+    }
+
     #[test]
     fn url_to_path_fragment() {
         let str = super::to_path(&Url::parse("https://lwn.net/Kernel/#fragment").unwrap());
 
         assert_eq!(str, "lwn.net/Kernel/index.html");
     }
+
+    #[test]
+    fn url_to_path_to_long_md5() {
+        let str = super::to_path(&Url::parse("https://lwn.net/Kernel/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.html").unwrap());
+
+        assert_eq!(str, "lwn.net/Kernel/5ca82767de71fe8930587e82bb994903.html");
+    }
 }