From b67d34583b984d5dc24bbd4ce702e0b324a1c243 Mon Sep 17 00:00:00 2001 From: Daniel Gallups Date: Wed, 8 Mar 2023 22:45:48 -0500 Subject: [PATCH] added code to handle css --- src/downloader.rs | 28 ++++++++------ src/filetype.rs | 5 +++ src/lib.rs | 1 + src/response.rs | 1 + src/scraper.rs | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 src/filetype.rs diff --git a/src/downloader.rs b/src/downloader.rs index 4d0c30b..1c63b1d 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -4,7 +4,7 @@ use lazy_static::lazy_static; use regex::Regex; use url::Url; -use crate::warn; +use crate::{filetype::FileType, warn}; use super::response::{Response, ResponseData}; @@ -81,8 +81,14 @@ impl Downloader { } ///Check if the type in the 'content-type' head field is html - fn is_html(content_type: &str) -> bool { - content_type.contains("text/html") + fn get_filetype(content_type: &str) -> FileType { + if content_type.contains("text/html") { + FileType::Html + } else if content_type.contains("text/css") { + FileType::Css + } else { + FileType::Other + } } ///Return the filename based on the HTML header of the response @@ -140,18 +146,18 @@ impl Downloader { None => (String::from("text/html"), None), }; - let filename = if !Downloader::is_html(&data_type) { - Downloader::get_filename(data.headers()) - } else { - None + let filename = match Downloader::get_filetype(&data_type) { + FileType::Other => Downloader::get_filename(data.headers()), + _ => None, }; let mut raw_data: Vec = Vec::new(); data.copy_to(&mut raw_data).unwrap(); - let response_data = if Downloader::is_html(&data_type) { - ResponseData::Html(raw_data) - } else { - ResponseData::Other(raw_data) + + let response_data = match Downloader::get_filetype(&data_type) { + FileType::Html => ResponseData::Html(raw_data), + FileType::Css => ResponseData::Css(raw_data), + FileType::Other => ResponseData::Other(raw_data), }; Ok(Response::new(response_data, filename, charset)) diff --git a/src/filetype.rs b/src/filetype.rs new file mode 100644 index 0000000..8994c47 --- /dev/null +++ b/src/filetype.rs @@ -0,0 +1,5 @@ +pub enum FileType { + Html, + Css, + Other, +} diff --git a/src/lib.rs b/src/lib.rs index 165596b..3047880 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ pub mod args; pub mod disk; pub mod dom; pub mod downloader; +pub mod filetype; pub mod logger; pub mod response; pub mod scraper; diff --git a/src/response.rs b/src/response.rs index 3f622b8..8100d2b 100644 --- a/src/response.rs +++ b/src/response.rs @@ -1,6 +1,7 @@ /// Separates HTML responses and other content (PDFs, images...) pub enum ResponseData { Html(Vec), + Css(Vec), Other(Vec), } diff --git a/src/scraper.rs b/src/scraper.rs index ad44db3..0ef59a8 100644 --- a/src/scraper.rs +++ b/src/scraper.rs @@ -226,6 +226,91 @@ impl Scraper { } } + fn handle_css( + scraper: &Scraper, + transmitter: &Sender<(Url, i32, i32)>, + url: &Url, + depth: i32, + ext_depth: i32, + data: &[u8], + http_charset: Option, + ) -> Vec { + let charset_source_str = match Self::find_charset(data, http_charset) { + Some(s) => s, + None => { + warn!("Charset not found for {}, defaulting to UTF-8", url); + String::from("utf-8") + } + }; + + let need_charset_conversion = Self::needs_charset_conversion(&charset_source_str); + + let charset_source = match encoding_rs::Encoding::for_label(charset_source_str.as_bytes()) { + Some(encoder) => encoder, + None => { + warn!( + "Charset {} not supported for {}, defaulting to UTF-8", + charset_source_str, url + ); + encoding_rs::UTF_8 + } + }; + let charset_utf8 = encoding_rs::UTF_8; + let utf8_data = if need_charset_conversion { + Self::charset_convert(data, charset_source, charset_utf8) + } else { + Vec::from(data) + }; + + let dom = dom::Dom::new(&String::from_utf8_lossy(&utf8_data)); + let source_path = match scraper.path_map.lock().unwrap().get(url.as_str()) { + Some(path) => path.clone(), + None => error!("Url {} was not found in the path map", url.as_str()), + }; + + dom.find_urls_as_strings() + .into_iter() + .filter(|candidate| Scraper::should_visit(scraper, candidate)) + .for_each(|next_url| { + let url_to_parse = Scraper::normalize_url(next_url.clone()); + + let next_full_url = match url.join(url_to_parse.as_str()) { + Ok(url) => url, + Err(e) => panic!("Failed to parse url: {} | Error: {}", next_url, e), + }; + + let path = url_helper::to_path(&next_full_url, true); + let path_no_fragments = url_helper::to_path(&next_full_url, false); + + // We only add urls without fragments to avoid duplication + if scraper.map_url_path(&next_full_url, path_no_fragments.clone()) { + if !Scraper::is_on_another_domain(next_url, url) { + // If we are determining for a local domain + if scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth { + Scraper::push(transmitter, next_full_url, depth + 1, ext_depth); + } + } else { + // If we are determining for an external domain + if scraper.args.ext_depth == INFINITE_DEPTH + || ext_depth < scraper.args.ext_depth + { + Scraper::push(transmitter, next_full_url, depth, ext_depth + 1); + } + } + } + + scraper.fix_domtree(next_url, &source_path, &path); + }); + + let utf8_data = dom.serialize().into_bytes(); + + if need_charset_conversion { + Self::charset_convert(&utf8_data, charset_utf8, charset_source) + } else { + utf8_data + } + } + /// Process a single URL fn handle_url( scraper: &Scraper, @@ -250,6 +335,15 @@ impl Scraper { &data, response.charset, ), + response::ResponseData::Css(data) => Scraper::handle_css( + scraper, + transmitter, + &url, + depth, + ext_depth, + &data, + response.charset, + ), response::ResponseData::Other(data) => data, };