diff --git a/Cargo.lock b/Cargo.lock index 1bb8fd1..8015d01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -691,6 +691,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.3.4" @@ -1508,7 +1514,7 @@ dependencies = [ [[package]] name = "suckit" -version = "0.1.0" +version = "0.1.1" dependencies = [ "chrono", "colored", @@ -1516,6 +1522,7 @@ dependencies = [ "encoding_rs", "kuchiki", "lazy_static", + "md5", "pathdiff", "rand 0.8.1", "regex", diff --git a/Cargo.toml b/Cargo.toml index 426c51a..9f0d7a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ regex = "^1.4" encoding_rs = "^0.8" lazy_static = "1.4.0" pathdiff = "^0.2" +md5 = "^0.7" [dev-dependencies] tiny_http = "^0.7" diff --git a/src/scraper.rs b/src/scraper.rs index 0088ff9..85f8da6 100644 --- a/src/scraper.rs +++ b/src/scraper.rs @@ -167,13 +167,10 @@ impl Scraper { }; let dom = dom::Dom::new(&String::from_utf8_lossy(&utf8_data).into_owned()); - let source_path = scraper - .path_map - .lock() - .unwrap() - .get(url.as_str()) - .unwrap() - .clone(); + let source_path = match scraper.path_map.lock().unwrap().get(url.as_str()) { + Some(path) => path.clone(), + None => error!("Url {} was not found in the path map", url.as_str()), + }; dom.find_urls_as_strings() .into_iter() diff --git a/src/url_helper.rs b/src/url_helper.rs index 9ad07a3..ca2a35b 100644 --- a/src/url_helper.rs +++ b/src/url_helper.rs @@ -1,22 +1,64 @@ +use std::path::Path; + +use md5; use url::Url; +///Max file name size supported by the file system +const FILE_NAME_MAX_LENGTH: usize = 255; + /// Convert an Url to the corresponding path pub fn to_path(url: &Url) -> String { - let domain = url.host_str().unwrap(); - let path = url.path(); + let url_domain = url.host_str().unwrap(); + let url_path = url.path(); + let url_query = url.query(); - let mut path = format!("{}{}", domain, path); - if path.ends_with("/") { - path = format!("{}index.html", path); + let path = Path::new(url_path); + let mut filename = path.file_name().map_or(String::from(""), |filename| { + filename.to_str().unwrap().to_string() + }); + let mut parent = path + .parent() + .map_or("", |filename| filename.to_str().unwrap()); + + if url_path.ends_with("/") { + filename = "index.html".to_string(); + parent = url_path.trim_end_matches("/"); + } else if path.extension().is_none() { + parent = url_path; + filename = "index_no_slash.html".to_string(); } - path + if url_query.is_some() { + filename.push('?'); + filename.push_str(url_query.unwrap()); + } + + if filename.len() > FILE_NAME_MAX_LENGTH { + let digest = md5::compute(filename); + filename = format!("{:x}.html", digest); + } + + format!("{}{}/{}", url_domain, parent, filename) } #[cfg(test)] mod tests { use super::*; + #[test] + fn url_to_path_domain_only() { + let str = super::to_path(&Url::parse("https://lwn.net/").unwrap()); + + assert_eq!(str, "lwn.net/index.html"); + } + + #[test] + fn url_to_path_domain_only_no_slash() { + let str = super::to_path(&Url::parse("https://lwn.net").unwrap()); + + assert_eq!(str, "lwn.net/index.html"); + } + #[test] fn url_to_path() { let str = super::to_path(&Url::parse("https://lwn.net/Kernel/index.html").unwrap()); @@ -31,10 +73,24 @@ mod tests { assert_eq!(str, "lwn.net/Kernel/index.html"); } + #[test] + fn url_to_path_index_no_slash() { + let str = super::to_path(&Url::parse("https://lwn.net/Kernel").unwrap()); + + assert_eq!(str, "lwn.net/Kernel/index_no_slash.html"); + } + #[test] fn url_to_path_fragment() { let str = super::to_path(&Url::parse("https://lwn.net/Kernel/#fragment").unwrap()); assert_eq!(str, "lwn.net/Kernel/index.html"); } + + #[test] + fn url_to_path_to_long_md5() { + let str = super::to_path(&Url::parse("https://lwn.net/Kernel/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.html").unwrap()); + + assert_eq!(str, "lwn.net/Kernel/5ca82767de71fe8930587e82bb994903.html"); + } }