scraping/skallwar.suckit
1
0
Fork 0
mirror of https://github.com/Skallwar/suckit.git synced 2026-09-10 07:16:19 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Filter links before downloading / adding to the queue

Browse source 
This commit speeds up scraping for scenarios where pages have a high branch factor, that
is many links and a majority of these links is excluded by the --exclude / --include
rules.
This also improves memory usage in these scenarios since the links are not stored.
Also the network traffic is reduced by not downloading these links in the first place.
This commit is contained in:
raphTec 2022-03-10 13:22:36 +01:00 committed by Esteban Blanc
parent 84276b9631
commit 3a3f4784d2
1 changed files with 13 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/scraper.rs
View file

@ -178,7 +178,7 @@ impl Scraper {
dom.find_urls_as_strings()
.into_iter()
.filter(|candidate| Scraper::should_visit(candidate))
.filter(|candidate| Scraper::should_visit(scraper, candidate))
.for_each(|next_url| {
let url_to_parse = Scraper::normalize_url(next_url.clone());
@ -246,10 +246,14 @@ impl Scraper {
let path_map = scraper.path_map.lock().unwrap();
let path = path_map.get(url.as_str()).unwrap();
if !scraper.args.dry_run
&& !scraper.args.exclude.is_match(url.as_str())
&& scraper.args.include.is_match(url.as_str())
{
// for the origin URL, we need to check the in/exclude rules since
// it is pushed into the channel unconditionally.
// we want to process its links, but maybe not download it.
// all other links are filtered before they are added to the channel.
let filter_rules_match = (depth > 0
|| (!scraper.args.exclude.is_match(url.as_str())
&& scraper.args.include.is_match(url.as_str())));
if !scraper.args.dry_run && filter_rules_match {
match response.filename {
Some(filename) => {
disk::save_file(&filename, &data, &scraper.args.output);
@ -333,7 +337,10 @@ impl Scraper {
}
/// If a URL should be visited (ignores `mail:`, `javascript:` and other pseudo-links)
fn should_visit(url: &str) -> bool {
fn should_visit(scraper: &Scraper, url: &str) -> bool {
if scraper.args.exclude.is_match(url) || !scraper.args.include.is_match(url) {
return false;
}
match Url::parse(url) {
/* The given candidate is a valid URL, and not a relative path to
* the next one. Therefore, we have to check if this URL is valid.