mirror of
https://github.com/Skallwar/suckit.git
synced 2026-09-10 07:16:19 -04:00
Add external site downloading support
Introduces --edepth flag for external domain depth Transforms urls that start with `//` into full https links for more accurate detection of external links Fixes panic when url starts with `///` Readme: Added explanation of the `--edepth` flag --edepth -> --ext-depth Refactors code to avoid repetition (normalizing urls) Improved code documentation (url normalization) README.md upd: --ext-depth Minor docs changes (--depth) rustfmt coding style Updated `--ext_depth` command docs Small code documentation improvements Co-authored-by: CohenArthur <cohenarthur.dev@gmail.com> Co-authored-by: Esteban Blanc <estblcsk@gmail.com>
This commit is contained in:
marchellodev
Esteban Blanc
parent
1be1f856e9
commit
ef79cc2318
262
Cargo.lock
generated
262
Cargo.lock
generated
|
|
@ -1,5 +1,7 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.18"
|
||||
|
|
@ -55,15 +57,15 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
|||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.2.1"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.7.0"
|
||||
version = "3.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
|
||||
checksum = "d9df67f7bf9ef8498769f994239c45613ef0c5899415fb58e9add412d2c1a538"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
|
|
@ -73,15 +75,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
|||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.0.1"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040"
|
||||
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.68"
|
||||
version = "1.0.70"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787"
|
||||
checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
|
|
@ -215,9 +217,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
|
||||
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-epoch",
|
||||
|
|
@ -286,13 +288,14 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "0.99.14"
|
||||
version = "0.99.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5cc7b9cef1e351660e5443924e4f43ab25fbbed3e9a5f052df3677deb4d6b320"
|
||||
checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustc_version 0.3.3",
|
||||
"syn",
|
||||
]
|
||||
|
||||
|
|
@ -369,42 +372,42 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2"
|
||||
checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1"
|
||||
checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d"
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1"
|
||||
checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377"
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282"
|
||||
checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae"
|
||||
checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967"
|
||||
checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"futures-core",
|
||||
|
|
@ -449,9 +452,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.3.3"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726"
|
||||
checksum = "d7f3675cfef6a30c8031cf9e6493ebdc3bb3272a3fea3923c4210d1830e6a472"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
|
|
@ -506,9 +509,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.4"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11"
|
||||
checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
|
|
@ -517,9 +520,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "0.4.2"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9"
|
||||
checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http",
|
||||
|
|
@ -528,9 +531,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "httparse"
|
||||
version = "1.4.1"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68"
|
||||
checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503"
|
||||
|
||||
[[package]]
|
||||
name = "httpdate"
|
||||
|
|
@ -540,9 +543,9 @@ checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440"
|
|||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.14.9"
|
||||
version = "0.14.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07d6baa1b441335f3ce5098ac421fb6547c46dda735ca1bc6d0153c838f9dd83"
|
||||
checksum = "15d1cfb9e4f68655fa04c01f59edb405b6074a0f7118ea881e5026e4a1cd8593"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
|
|
@ -604,15 +607,15 @@ checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9"
|
|||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "0.4.7"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
|
||||
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.51"
|
||||
version = "0.3.55"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062"
|
||||
checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84"
|
||||
dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
|
@ -637,9 +640,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
|||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.97"
|
||||
version = "0.2.102"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6"
|
||||
checksum = "a2a5ac8f984bfcf3a823267e5fde638acc3325f6496633a5da6bb6eb2171e103"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
|
|
@ -672,9 +675,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "matches"
|
||||
version = "0.1.8"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
|
||||
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
|
||||
|
||||
[[package]]
|
||||
name = "md5"
|
||||
|
|
@ -684,9 +687,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
|
|||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.4.0"
|
||||
version = "2.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
|
||||
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
|
|
@ -727,9 +730,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "native-tls"
|
||||
version = "0.2.7"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4"
|
||||
checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"libc",
|
||||
|
|
@ -801,9 +804,9 @@ checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
|
|||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.35"
|
||||
version = "0.10.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885"
|
||||
checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
|
|
@ -821,9 +824,9 @@ checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"
|
|||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.65"
|
||||
version = "0.9.67"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d"
|
||||
checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cc",
|
||||
|
|
@ -844,6 +847,15 @@ version = "2.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||
|
||||
[[package]]
|
||||
name = "pest"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
|
||||
dependencies = [
|
||||
"ucd-trie",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.8.0"
|
||||
|
|
@ -960,9 +972,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
|
|||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.27"
|
||||
version = "1.0.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
|
||||
checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
|
@ -1079,9 +1091,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.9"
|
||||
version = "0.2.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
|
||||
checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
|
@ -1155,7 +1167,16 @@ version = "0.2.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
|
||||
dependencies = [
|
||||
"semver",
|
||||
"semver 0.9.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee"
|
||||
dependencies = [
|
||||
"semver 0.11.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1182,9 +1203,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
|||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "2.3.1"
|
||||
version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
|
||||
checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"core-foundation",
|
||||
|
|
@ -1195,9 +1216,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "security-framework-sys"
|
||||
version = "2.3.0"
|
||||
version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284"
|
||||
checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
|
|
@ -1229,7 +1250,16 @@ version = "0.9.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
|
||||
dependencies = [
|
||||
"semver-parser",
|
||||
"semver-parser 0.7.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6"
|
||||
dependencies = [
|
||||
"semver-parser 0.10.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1239,19 +1269,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.126"
|
||||
name = "semver-parser"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
|
||||
checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7"
|
||||
dependencies = [
|
||||
"pest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.126"
|
||||
version = "1.0.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
|
||||
checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1260,9 +1299,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.64"
|
||||
version = "1.0.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
|
||||
checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"ryu",
|
||||
|
|
@ -1299,15 +1338,15 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d"
|
|||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "0.3.5"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27"
|
||||
checksum = "533494a8f9b724d33625ab53c6c4800f7cc445895924a8ef649222dcb76e938b"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.3"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
|
||||
checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
|
|
@ -1317,9 +1356,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
|
|||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.4.0"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2"
|
||||
checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
|
|
@ -1347,7 +1386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5"
|
||||
dependencies = [
|
||||
"discard",
|
||||
"rustc_version",
|
||||
"rustc_version 0.2.3",
|
||||
"stdweb-derive",
|
||||
"stdweb-internal-macros",
|
||||
"stdweb-internal-runtime",
|
||||
|
|
@ -1446,9 +1485,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "subprocess"
|
||||
version = "0.2.7"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "334b801f9ca2529ba9605109f1d2835f15aff94cd6cfe5afe6ce8cf71e5fd3c4"
|
||||
checksum = "055cf3ebc2981ad8f0a5a17ef6652f652d87831f79fddcba2ac57bcb9a0aa407"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
|
|
@ -1484,9 +1523,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
|
|||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.73"
|
||||
version = "1.0.76"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
|
||||
checksum = "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1596,9 +1635,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.2.0"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342"
|
||||
checksum = "5241dd6f21443a3606b432718b166d3cedc962fd4b8bea54a8bc7f514ebda986"
|
||||
dependencies = [
|
||||
"tinyvec_macros",
|
||||
]
|
||||
|
|
@ -1611,9 +1650,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
|
|||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.8.1"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98c8b05dc14c75ea83d63dd391100353789f5f24b8b3866542a5e85c8be8e985"
|
||||
checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"bytes",
|
||||
|
|
@ -1637,9 +1676,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.6.7"
|
||||
version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592"
|
||||
checksum = "08d3725d3efa29485e87311c5b699de63cde14b00ed4d256b8318aa30ca452cd"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
|
|
@ -1657,9 +1696,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
|
|||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.26"
|
||||
version = "0.1.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d"
|
||||
checksum = "84f96e095c0c82419687c20ddf5cb3eadb61f4e1405923c9dc8e53a1adacbda8"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"pin-project-lite",
|
||||
|
|
@ -1668,9 +1707,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tracing-core"
|
||||
version = "0.1.18"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052"
|
||||
checksum = "46125608c26121c81b0c6d693eab5a420e416da7e43c426d2e8f7df8da8a3acf"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
|
@ -1682,13 +1721,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.5"
|
||||
name = "ucd-trie"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0"
|
||||
dependencies = [
|
||||
"matches",
|
||||
]
|
||||
checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
|
|
@ -1707,9 +1749,9 @@ checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
|
|||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.8"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
|
||||
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
|
|
@ -1777,9 +1819,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.74"
|
||||
version = "0.2.78"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
|
||||
checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"serde",
|
||||
|
|
@ -1789,9 +1831,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-backend"
|
||||
version = "0.2.74"
|
||||
version = "0.2.78"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
|
||||
checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"lazy_static",
|
||||
|
|
@ -1804,9 +1846,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.24"
|
||||
version = "0.4.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1"
|
||||
checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
|
|
@ -1816,9 +1858,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.74"
|
||||
version = "0.2.78"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
|
||||
checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
|
|
@ -1826,9 +1868,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.74"
|
||||
version = "0.2.78"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
|
||||
checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1839,15 +1881,15 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.74"
|
||||
version = "0.2.78"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"
|
||||
checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc"
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.51"
|
||||
version = "0.3.55"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582"
|
||||
checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
|
|
|
|||
|
|
@ -29,7 +29,8 @@ your disk.
|
|||
|---|---|
|
||||
|`-h, --help`|Displays help information|
|
||||
|`-v, --verbose`|Activate Verbose output|
|
||||
|`-d, --depth`|Specify the level of depth to go to when visiting the website|
|
||||
|`-d, --depth`|Specify the level of depth to go to when visiting the website. Default is -1 (infinity)|
|
||||
|`--ext-depth`|Specify the level of depth to go to when visiting websites that have a different domain name. Default is 0 (ignore external links), -1 is infinity|
|
||||
|`-j, --jobs`|Number of threads to use|
|
||||
|`-o, --output`|Output directory where the downloaded files are written|
|
||||
|`-t, --tries`|Number of times to retry when the downloading of a page fails|
|
||||
|
|
|
|||
29
src/args.rs
29
src/args.rs
|
|
@ -33,10 +33,17 @@ pub struct Args {
|
|||
short,
|
||||
long,
|
||||
default_value = "-1",
|
||||
help = "Maximum recursion depth to reach when visiting. -1 is the default and will go as far as it can"
|
||||
help = "Maximum recursion depth to reach when visiting. Default is -1 (infinity)"
|
||||
)]
|
||||
pub depth: i32,
|
||||
|
||||
#[structopt(
|
||||
long,
|
||||
default_value = "0",
|
||||
help = "Maximum recursion depth to reach when visiting external domains. Default is 0. -1 means infinity"
|
||||
)]
|
||||
pub ext_depth: i32,
|
||||
|
||||
///Number of retries when downloading a page fails
|
||||
#[structopt(
|
||||
short,
|
||||
|
|
@ -81,21 +88,21 @@ pub struct Args {
|
|||
|
||||
/// Regex filter to limit saving pages to only matched ones
|
||||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
default_value = ".*",
|
||||
parse(try_from_str = parse_regex),
|
||||
help = "Regex filter to limit to only saving pages that match this expression"
|
||||
short,
|
||||
long,
|
||||
default_value = ".*",
|
||||
parse(try_from_str = parse_regex),
|
||||
help = "Regex filter to limit to only saving pages that match this expression"
|
||||
)]
|
||||
pub include: Regex,
|
||||
|
||||
/// Regex filter to limit saving pages to only matched ones
|
||||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
default_value = "$^",
|
||||
parse(try_from_str = parse_regex),
|
||||
help = "Regex filter to exclude saving pages that match this expression"
|
||||
short,
|
||||
long,
|
||||
default_value = "$^",
|
||||
parse(try_from_str = parse_regex),
|
||||
help = "Regex filter to exclude saving pages that match this expression"
|
||||
)]
|
||||
pub exclude: Regex,
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use symlink::symlink_auto;
|
||||
|
||||
use crate::{error, warn};
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
use super::response::{Response, ResponseData};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
|
|
@ -7,6 +6,8 @@ use url::Url;
|
|||
|
||||
use crate::warn;
|
||||
|
||||
use super::response::{Response, ResponseData};
|
||||
|
||||
const AUTH_CHUNK_SIZE: usize = 3;
|
||||
|
||||
///A Downloader to download web content
|
||||
|
|
@ -190,14 +191,14 @@ mod tests {
|
|||
assert_eq!(
|
||||
parse_auth(
|
||||
&["".to_string(), "pw".to_string()],
|
||||
&Url::parse("https://example.com/").unwrap()
|
||||
&Url::parse("https://example.com/").unwrap(),
|
||||
),
|
||||
Err("Invalid arguments supplied to auth".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
parse_auth(
|
||||
&["username".to_string()],
|
||||
&Url::parse("https://example.com/").unwrap()
|
||||
&Url::parse("https://example.com/").unwrap(),
|
||||
),
|
||||
Ok(("username".to_string(), None, "example.com".to_string()))
|
||||
);
|
||||
|
|
@ -209,7 +210,7 @@ mod tests {
|
|||
"h".to_string(),
|
||||
"t".to_string()
|
||||
],
|
||||
&Url::parse("https://example.com/").unwrap()
|
||||
&Url::parse("https://example.com/").unwrap(),
|
||||
),
|
||||
Ok(("un".to_string(), Some("pw".to_string()), "h".to_string()))
|
||||
)
|
||||
|
|
|
|||
121
src/scraper.rs
121
src/scraper.rs
|
|
@ -1,3 +1,10 @@
|
|||
use std::borrow::Borrow;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
use std::time;
|
||||
|
||||
use crossbeam::channel::{Receiver, Sender, TryRecvError};
|
||||
use crossbeam::thread;
|
||||
use encoding_rs::Encoding;
|
||||
|
|
@ -7,12 +14,7 @@ use rand::Rng;
|
|||
use regex::Regex;
|
||||
use url::Url;
|
||||
|
||||
use std::borrow::Borrow;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
use std::time;
|
||||
use crate::{error, info, warn};
|
||||
|
||||
use super::args;
|
||||
use super::disk;
|
||||
|
|
@ -21,8 +23,6 @@ use super::downloader;
|
|||
use super::response;
|
||||
use super::url_helper;
|
||||
|
||||
use crate::{error, info, warn};
|
||||
|
||||
/// Maximum number of empty recv() from the channel
|
||||
static MAX_EMPTY_RECEIVES: usize = 10;
|
||||
|
||||
|
|
@ -37,8 +37,8 @@ static SLEEP_DURATION: time::Duration = time::Duration::from_millis(SLEEP_MILLIS
|
|||
/// adds more as new URLs are found
|
||||
pub struct Scraper {
|
||||
args: args::Args,
|
||||
transmitter: Sender<(Url, i32)>,
|
||||
receiver: Receiver<(Url, i32)>,
|
||||
transmitter: Sender<(Url, i32, i32)>,
|
||||
receiver: Receiver<(Url, i32, i32)>,
|
||||
downloader: downloader::Downloader,
|
||||
visited_urls: Mutex<HashSet<String>>,
|
||||
path_map: Mutex<HashMap<String, String>>,
|
||||
|
|
@ -77,8 +77,8 @@ impl Scraper {
|
|||
}
|
||||
|
||||
/// Push a new URL into the channel
|
||||
fn push(transmitter: &Sender<(Url, i32)>, url: Url, depth: i32) {
|
||||
if let Err(e) = transmitter.send((url, depth)) {
|
||||
fn push(transmitter: &Sender<(Url, i32, i32)>, url: Url, depth: i32, ext_depth: i32) {
|
||||
if let Err(e) = transmitter.send((url, depth, ext_depth)) {
|
||||
error!("Couldn't push to channel ! {}", e);
|
||||
}
|
||||
}
|
||||
|
|
@ -140,9 +140,10 @@ impl Scraper {
|
|||
/// Proces an html file: add new url to the chanel and prepare for offline navigation
|
||||
fn handle_html(
|
||||
scraper: &Scraper,
|
||||
transmitter: &Sender<(Url, i32)>,
|
||||
transmitter: &Sender<(Url, i32, i32)>,
|
||||
url: &Url,
|
||||
depth: i32,
|
||||
ext_depth: i32,
|
||||
data: &[u8],
|
||||
http_charset: Option<String>,
|
||||
) -> Vec<u8> {
|
||||
|
|
@ -182,16 +183,32 @@ impl Scraper {
|
|||
|
||||
dom.find_urls_as_strings()
|
||||
.into_iter()
|
||||
.filter(|candidate| Scraper::should_visit(candidate, &url))
|
||||
.filter(|candidate| Scraper::should_visit(candidate))
|
||||
.for_each(|next_url| {
|
||||
let mut next_full_url = url.join(&next_url).unwrap();
|
||||
let url_to_parse = Scraper::normalize_url(next_url.clone());
|
||||
|
||||
let mut next_full_url = match url.join(url_to_parse.as_str()) {
|
||||
Ok(url) => url,
|
||||
Err(e) => panic!("Failed to parse url: {} | Error: {}", next_url, e),
|
||||
};
|
||||
|
||||
next_full_url.set_fragment(None);
|
||||
let path = url_helper::to_path(&next_full_url);
|
||||
|
||||
if scraper.map_url_path(&next_full_url, path.clone())
|
||||
&& (scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth)
|
||||
{
|
||||
Scraper::push(transmitter, next_full_url, depth + 1);
|
||||
if scraper.map_url_path(&next_full_url, path.clone()) {
|
||||
if !Scraper::is_on_another_domain(&next_url, &url) {
|
||||
// If we are determining for a local domain
|
||||
if scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth {
|
||||
Scraper::push(transmitter, next_full_url, depth + 1, ext_depth);
|
||||
}
|
||||
} else {
|
||||
// If we are determining for an external domain
|
||||
if scraper.args.ext_depth == INFINITE_DEPTH
|
||||
|| ext_depth < scraper.args.ext_depth
|
||||
{
|
||||
Scraper::push(transmitter, next_full_url, depth, ext_depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
scraper.fix_domtree(next_url, &source_path, &path);
|
||||
|
|
@ -207,7 +224,13 @@ impl Scraper {
|
|||
}
|
||||
|
||||
/// Process a single URL
|
||||
fn handle_url(scraper: &Scraper, transmitter: &Sender<(Url, i32)>, url: Url, depth: i32) {
|
||||
fn handle_url(
|
||||
scraper: &Scraper,
|
||||
transmitter: &Sender<(Url, i32, i32)>,
|
||||
url: Url,
|
||||
depth: i32,
|
||||
ext_depth: i32,
|
||||
) {
|
||||
match scraper.downloader.get(&url) {
|
||||
Ok(response) => {
|
||||
let data = match response.data {
|
||||
|
|
@ -216,6 +239,7 @@ impl Scraper {
|
|||
transmitter,
|
||||
&url,
|
||||
depth,
|
||||
ext_depth,
|
||||
&data,
|
||||
response.charset,
|
||||
),
|
||||
|
|
@ -263,7 +287,7 @@ impl Scraper {
|
|||
pub fn run(&mut self) {
|
||||
/* Push the origin URL and depth (0) through the channel */
|
||||
self.map_url_path(&self.args.origin, url_helper::to_path(&self.args.origin));
|
||||
Scraper::push(&self.transmitter, self.args.origin.clone(), 0);
|
||||
Scraper::push(&self.transmitter, self.args.origin.clone(), 0, 0);
|
||||
|
||||
thread::scope(|thread_scope| {
|
||||
for _ in 0..self.args.jobs {
|
||||
|
|
@ -285,9 +309,9 @@ impl Scraper {
|
|||
}
|
||||
TryRecvError::Disconnected => panic!("{}", e),
|
||||
},
|
||||
Ok((url, depth)) => {
|
||||
Ok((url, depth, ext_depth)) => {
|
||||
counter = 0;
|
||||
Scraper::handle_url(&self_clone, &tx, url, depth);
|
||||
Scraper::handle_url(&self_clone, &tx, url, depth, ext_depth);
|
||||
self_clone.sleep(&mut rng);
|
||||
}
|
||||
}
|
||||
|
|
@ -313,14 +337,14 @@ impl Scraper {
|
|||
std::thread::sleep(delay_duration);
|
||||
}
|
||||
|
||||
/// If a URL should be visited, or does it belong to another domain
|
||||
fn should_visit(url: &str, base: &Url) -> bool {
|
||||
/// If a URL should be visited (ignores `mail:`, `javascript:` and other pseudo-links)
|
||||
fn should_visit(url: &str) -> bool {
|
||||
match Url::parse(url) {
|
||||
/* The given candidate is a valid URL, and not a relative path to
|
||||
* the next one. Therefore, we have to check if this URL belongs
|
||||
* to the same domain as our current URL. If the candidate has the
|
||||
* same domain as our base, then we should visit it */
|
||||
Ok(not_ok) => not_ok.domain() == base.domain(),
|
||||
* the next one. Therefore, we have to check if this URL is valid.
|
||||
* If it is, we should visit it.
|
||||
*/
|
||||
Ok(not_ok) => not_ok.has_host() && !not_ok.cannot_be_a_base(),
|
||||
|
||||
/* Since we couldn't parse this "URL", then it must be a relative
|
||||
* path or a malformed URL. If the URL is malformed, then it will
|
||||
|
|
@ -328,14 +352,47 @@ impl Scraper {
|
|||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces `///` with `//`
|
||||
/// And `//` with `https://`
|
||||
/// Without this function, if url is `///<domain>.<extension>/`, the app crashes.
|
||||
fn normalize_url(url: String) -> String {
|
||||
if url.starts_with("///") {
|
||||
return url.replacen("///", "https://", 1);
|
||||
} else if url.starts_with("//") {
|
||||
return url.replacen("//", "https://", 1);
|
||||
}
|
||||
url
|
||||
}
|
||||
|
||||
/// If the URL leads to another domain
|
||||
fn is_on_another_domain(url: &str, base: &Url) -> bool {
|
||||
let real_url = Scraper::normalize_url(String::from(url));
|
||||
|
||||
match Url::parse(real_url.as_str()) {
|
||||
/* The given candidate is a valid URL, and not a relative path to
|
||||
* the next one. Therefore, we have to check if this URL belongs
|
||||
* to the same domain as our current URL. If the candidate has the
|
||||
* same domain as our base, and the depth condition is satisfied,
|
||||
* then we should visit it, */
|
||||
Ok(not_ok) => not_ok.domain() != base.domain(),
|
||||
|
||||
/* Since we couldn't parse this "URL", then it must be a relative
|
||||
* path or a malformed URL. If the URL is malformed, then it will
|
||||
* be handled during the join() call in run() */
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use regex::Regex;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_zero_delay_range() {
|
||||
let args = args::Args {
|
||||
|
|
@ -344,6 +401,7 @@ mod tests {
|
|||
jobs: 1,
|
||||
tries: 1,
|
||||
depth: 5,
|
||||
ext_depth: 0,
|
||||
delay: 0,
|
||||
user_agent: "suckit".to_string(),
|
||||
random_range: 0,
|
||||
|
|
@ -366,6 +424,7 @@ mod tests {
|
|||
jobs: 1,
|
||||
tries: 1,
|
||||
depth: 5,
|
||||
ext_depth: 0,
|
||||
delay: 2,
|
||||
user_agent: "suckit".to_string(),
|
||||
random_range: 5,
|
||||
|
|
|
|||
Loading…
Reference in a new issue