scraping/skallwar.suckit
1
0
Fork 0
mirror of https://github.com/Skallwar/suckit.git synced 2026-09-10 07:16:19 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Better help for CLI arguments (#56)

Browse source 
* args: Add 'help' section

* benchmark: Add more data

* Update src/args.rs

Co-authored-by: Esteban Blanc <estblcsk@gmail.com>

* Update src/args.rs

Co-authored-by: Esteban Blanc <estblcsk@gmail.com>

Co-authored-by: Esteban Blanc <estblcsk@gmail.com>
This commit is contained in:
CohenArthur 2020-05-01 13:48:39 +02:00 committed by GitHub
parent 5233aecbc1
commit 6e93171185
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 10 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/args.rs
View file

@ -7,27 +7,27 @@ use url::Url;
#[derive(Debug, StructOpt)]
pub struct Args {
///Entry point of scraping
#[structopt(name = "url", parse(try_from_str))]
#[structopt(name = "url", parse(try_from_str), help = "Entry point of the scraping")]
pub origin: Url,
///Output directory
#[structopt(short, long, parse(from_os_str))]
#[structopt(short, long, parse(from_os_str), help = "Output directory")]
pub output: Option<PathBuf>,
///Number of threads/workers
#[structopt(short, long, default_value = "1")]
#[structopt(short, long, default_value = "1", help = "Maximum number of threads to use concurrently")]
pub jobs: usize,
///Max depth of scraping recursion
#[structopt(short, long, default_value = "5")]
#[structopt(short, long, default_value = "5", help = "Maximum recursion depth to reach when visiting")]
pub depth: usize,
///Number of retries when downloading a page fails
#[structopt(short, long, default_value = "20")]
#[structopt(short, long, default_value = "20", help = "Maximum amount of retries on download failure")]
pub tries: usize,
///Show logs
#[structopt(short, long)]
///Show all logs
#[structopt(short, long, help = "Enable more information regarding the scraping process")]
pub verbose: bool,
}

3
tests/speed.csv
View file

@ -7,3 +7,6 @@
94.61654992103577, 57.59067335128784, 24.995196104049683
95.28456358909607, 46.86330976486206, 24.708560609817503
77.68792352676391, 39.08626494407654, 19.765260219573975
67.09948434829712, 34.11661868095398, 17.574066400527954
68.2605634689331, 33.728125953674315, 17.433270835876463
70.19220662117004, 41.964057207107544, 17.660782480239867

1 1 1 1
7 94.61654992103577 57.59067335128784 24.995196104049683
8 95.28456358909607 46.86330976486206 24.708560609817503
9 77.68792352676391 39.08626494407654 19.765260219573975
10 67.09948434829712 34.11661868095398 17.574066400527954
11 68.2605634689331 33.728125953674315 17.433270835876463
12 70.19220662117004 41.964057207107544 17.660782480239867