scraping/skallwar.suckit
1
0
Fork 0
mirror of https://github.com/Skallwar/suckit.git synced 2026-09-10 07:16:19 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

scraper: Split handle_url and parse_html

Browse source
This commit is contained in:
Esteban Blanc 2020-04-27 19:29:25 +02:00
parent fd0b7d4254
commit 1f41bf709d
1 changed files with 15 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/scraper.rs
View file

@ -78,10 +78,14 @@ impl Scraper {
old_url_str.push_str(path_map.get(new_url.as_str()).unwrap());
}
/// Process a single URL
fn handle_url(scraper: &Scraper, transmitter: &Sender<(Url, usize)>, url: Url, depth: usize) {
let page = scraper.downloader.get(&url).unwrap();
let dom = dom::Dom::new(&page);
fn handle_html(
scraper: &Scraper,
transmitter: &Sender<(Url, usize)>,
url: &Url,
depth: usize,
data: &String,
) {
let dom = dom::Dom::new(data);
dom.find_urls_as_strings()
.into_iter()
@ -101,12 +105,18 @@ impl Scraper {
});
let path_map = scraper.path_map.lock().unwrap();
disk::save_file(
path_map.get(url.as_str()).unwrap(),
&dom.serialize().as_bytes(),
&scraper.args.output,
);
}
/// Process a single URL
fn handle_url(scraper: &Scraper, transmitter: &Sender<(Url, usize)>, url: Url, depth: usize) {
let page = scraper.downloader.get(&url).unwrap();
Scraper::handle_html(scraper, transmitter, &url, depth, &page);
scraper.visited_urls.lock().unwrap().insert(url.to_string());