scraping/skallwar.suckit
1
0
Fork 0
mirror of https://github.com/Skallwar/suckit.git synced 2026-09-10 07:16:19 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix scraping on URLs that have fragments (#120)

Browse source
This commit is contained in:
Markus Unterwaditzer 2021-05-25 21:47:55 +02:00 committed by GitHub
parent dcb5080fca
commit 9a194d784e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/scraper.rs
View file

@ -184,15 +184,14 @@ impl Scraper {
.into_iter()
.filter(|candidate| Scraper::should_visit(candidate, &url))
.for_each(|next_url| {
let next_full_url = url.join(&next_url).unwrap();
let mut next_full_url = url.join(&next_url).unwrap();
next_full_url.set_fragment(None);
let path = url_helper::to_path(&next_full_url);
if scraper.map_url_path(&next_full_url, path.clone())
&& (scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth)
{
let mut next_full_download_url = next_full_url.clone();
next_full_download_url.set_fragment(None);
Scraper::push(transmitter, next_full_download_url, depth + 1);
Scraper::push(transmitter, next_full_url, depth + 1);
}
scraper.fix_domtree(next_url, &source_path, &path);