diff --git a/Cargo.lock b/Cargo.lock index 7f29e76..8d85269 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "aho-corasick" version = "0.7.18" @@ -55,15 +57,15 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bitflags" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bumpalo" -version = "3.7.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" +checksum = "d9df67f7bf9ef8498769f994239c45613ef0c5899415fb58e9add412d2c1a538" [[package]] name = "byteorder" @@ -73,15 +75,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" [[package]] name = "cc" -version = "1.0.68" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" +checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0" [[package]] name = "cfg-if" @@ -215,9 +217,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -286,13 +288,14 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.14" +version = "0.99.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc7b9cef1e351660e5443924e4f43ab25fbbed3e9a5f052df3677deb4d6b320" +checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df" dependencies = [ "convert_case", "proc-macro2", "quote", + "rustc_version 0.3.3", "syn", ] @@ -369,42 +372,42 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" +checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" +checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" [[package]] name = "futures-io" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" +checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" [[package]] name = "futures-sink" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" +checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" [[package]] name = "futures-task" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" +checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" [[package]] name = "futures-util" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" +checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" dependencies = [ "autocfg", "futures-core", @@ -449,9 +452,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" +checksum = "d7f3675cfef6a30c8031cf9e6493ebdc3bb3272a3fea3923c4210d1830e6a472" dependencies = [ "bytes", "fnv", @@ -506,9 +509,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" dependencies = [ "bytes", "fnv", @@ -517,9 +520,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5" dependencies = [ "bytes", "http", @@ -528,9 +531,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" +checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" [[package]] name = "httpdate" @@ -540,9 +543,9 @@ checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" [[package]] name = "hyper" -version = "0.14.9" +version = "0.14.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07d6baa1b441335f3ce5098ac421fb6547c46dda735ca1bc6d0153c838f9dd83" +checksum = "15d1cfb9e4f68655fa04c01f59edb405b6074a0f7118ea881e5026e4a1cd8593" dependencies = [ "bytes", "futures-channel", @@ -604,15 +607,15 @@ checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" [[package]] name = "itoa" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "js-sys" -version = "0.3.51" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" dependencies = [ "wasm-bindgen", ] @@ -637,9 +640,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.97" +version = "0.2.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" +checksum = "a2a5ac8f984bfcf3a823267e5fde638acc3325f6496633a5da6bb6eb2171e103" [[package]] name = "log" @@ -672,9 +675,9 @@ dependencies = [ [[package]] name = "matches" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "md5" @@ -684,9 +687,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "memoffset" @@ -727,9 +730,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" dependencies = [ "lazy_static", "libc", @@ -801,9 +804,9 @@ checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" [[package]] name = "openssl" -version = "0.10.35" +version = "0.10.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885" +checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a" dependencies = [ "bitflags", "cfg-if", @@ -821,9 +824,9 @@ checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "openssl-sys" -version = "0.9.65" +version = "0.9.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d" +checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058" dependencies = [ "autocfg", "cc", @@ -844,6 +847,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + [[package]] name = "phf" version = "0.8.0" @@ -960,9 +972,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.27" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" +checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" dependencies = [ "unicode-xid", ] @@ -1079,9 +1091,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" dependencies = [ "bitflags", ] @@ -1155,7 +1167,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", ] [[package]] @@ -1182,9 +1203,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "security-framework" -version = "2.3.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" dependencies = [ "bitflags", "core-foundation", @@ -1195,9 +1216,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.3.0" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" dependencies = [ "core-foundation-sys", "libc", @@ -1229,7 +1250,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser", + "semver-parser 0.7.0", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", ] [[package]] @@ -1239,19 +1269,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] -name = "serde" -version = "1.0.126" +name = "semver-parser" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + +[[package]] +name = "serde" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.126" +version = "1.0.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" +checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" dependencies = [ "proc-macro2", "quote", @@ -1260,9 +1299,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.64" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" dependencies = [ "itoa", "ryu", @@ -1299,15 +1338,15 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "siphasher" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27" +checksum = "533494a8f9b724d33625ab53c6c4800f7cc445895924a8ef649222dcb76e938b" [[package]] name = "slab" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" +checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" [[package]] name = "smallvec" @@ -1317,9 +1356,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "socket2" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" dependencies = [ "libc", "winapi", @@ -1347,7 +1386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -1446,9 +1485,9 @@ dependencies = [ [[package]] name = "subprocess" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "334b801f9ca2529ba9605109f1d2835f15aff94cd6cfe5afe6ce8cf71e5fd3c4" +checksum = "055cf3ebc2981ad8f0a5a17ef6652f652d87831f79fddcba2ac57bcb9a0aa407" dependencies = [ "libc", "winapi", @@ -1484,9 +1523,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" [[package]] name = "syn" -version = "1.0.73" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7" +checksum = "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84" dependencies = [ "proc-macro2", "quote", @@ -1596,9 +1635,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" +checksum = "5241dd6f21443a3606b432718b166d3cedc962fd4b8bea54a8bc7f514ebda986" dependencies = [ "tinyvec_macros", ] @@ -1611,9 +1650,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.8.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c8b05dc14c75ea83d63dd391100353789f5f24b8b3866542a5e85c8be8e985" +checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc" dependencies = [ "autocfg", "bytes", @@ -1637,9 +1676,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +checksum = "08d3725d3efa29485e87311c5b699de63cde14b00ed4d256b8318aa30ca452cd" dependencies = [ "bytes", "futures-core", @@ -1657,9 +1696,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.26" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" +checksum = "84f96e095c0c82419687c20ddf5cb3eadb61f4e1405923c9dc8e53a1adacbda8" dependencies = [ "cfg-if", "pin-project-lite", @@ -1668,9 +1707,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.18" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" +checksum = "46125608c26121c81b0c6d693eab5a420e416da7e43c426d2e8f7df8da8a3acf" dependencies = [ "lazy_static", ] @@ -1682,13 +1721,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] -name = "unicode-bidi" -version = "0.3.5" +name = "ucd-trie" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" -dependencies = [ - "matches", -] +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "unicode-bidi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085" [[package]] name = "unicode-normalization" @@ -1707,9 +1749,9 @@ checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" [[package]] name = "unicode-width" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" [[package]] name = "unicode-xid" @@ -1777,9 +1819,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasm-bindgen" -version = "0.2.74" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" dependencies = [ "cfg-if", "serde", @@ -1789,9 +1831,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.74" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" dependencies = [ "bumpalo", "lazy_static", @@ -1804,9 +1846,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.24" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" +checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" dependencies = [ "cfg-if", "js-sys", @@ -1816,9 +1858,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.74" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1826,9 +1868,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.74" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" dependencies = [ "proc-macro2", "quote", @@ -1839,15 +1881,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.74" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" [[package]] name = "web-sys" -version = "0.3.51" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/README.md b/README.md index 6f98413..d7731a3 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,8 @@ your disk. |---|---| |`-h, --help`|Displays help information| |`-v, --verbose`|Activate Verbose output| -|`-d, --depth`|Specify the level of depth to go to when visiting the website| +|`-d, --depth`|Specify the level of depth to go to when visiting the website. Default is -1 (infinity)| +|`--ext-depth`|Specify the level of depth to go to when visiting websites that have a different domain name. Default is 0 (ignore external links), -1 is infinity| |`-j, --jobs`|Number of threads to use| |`-o, --output`|Output directory where the downloaded files are written| |`-t, --tries`|Number of times to retry when the downloading of a page fails| diff --git a/src/args.rs b/src/args.rs index b7bc1b0..c8be299 100644 --- a/src/args.rs +++ b/src/args.rs @@ -33,10 +33,17 @@ pub struct Args { short, long, default_value = "-1", - help = "Maximum recursion depth to reach when visiting. -1 is the default and will go as far as it can" + help = "Maximum recursion depth to reach when visiting. Default is -1 (infinity)" )] pub depth: i32, + #[structopt( + long, + default_value = "0", + help = "Maximum recursion depth to reach when visiting external domains. Default is 0. -1 means infinity" + )] + pub ext_depth: i32, + ///Number of retries when downloading a page fails #[structopt( short, @@ -81,21 +88,21 @@ pub struct Args { /// Regex filter to limit saving pages to only matched ones #[structopt( - short, - long, - default_value = ".*", - parse(try_from_str = parse_regex), - help = "Regex filter to limit to only saving pages that match this expression" + short, + long, + default_value = ".*", + parse(try_from_str = parse_regex), + help = "Regex filter to limit to only saving pages that match this expression" )] pub include: Regex, /// Regex filter to limit saving pages to only matched ones #[structopt( - short, - long, - default_value = "$^", - parse(try_from_str = parse_regex), - help = "Regex filter to exclude saving pages that match this expression" + short, + long, + default_value = "$^", + parse(try_from_str = parse_regex), + help = "Regex filter to exclude saving pages that match this expression" )] pub exclude: Regex, diff --git a/src/disk.rs b/src/disk.rs index a944839..39c6846 100644 --- a/src/disk.rs +++ b/src/disk.rs @@ -1,6 +1,7 @@ use std::fs; use std::io::Write; use std::path::PathBuf; + use symlink::symlink_auto; use crate::{error, warn}; diff --git a/src/downloader.rs b/src/downloader.rs index 09c9f66..1496f56 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -1,4 +1,3 @@ -use super::response::{Response, ResponseData}; use std::collections::HashMap; use lazy_static::lazy_static; @@ -7,6 +6,8 @@ use url::Url; use crate::warn; +use super::response::{Response, ResponseData}; + const AUTH_CHUNK_SIZE: usize = 3; ///A Downloader to download web content @@ -190,14 +191,14 @@ mod tests { assert_eq!( parse_auth( &["".to_string(), "pw".to_string()], - &Url::parse("https://example.com/").unwrap() + &Url::parse("https://example.com/").unwrap(), ), Err("Invalid arguments supplied to auth".to_string()) ); assert_eq!( parse_auth( &["username".to_string()], - &Url::parse("https://example.com/").unwrap() + &Url::parse("https://example.com/").unwrap(), ), Ok(("username".to_string(), None, "example.com".to_string())) ); @@ -209,7 +210,7 @@ mod tests { "h".to_string(), "t".to_string() ], - &Url::parse("https://example.com/").unwrap() + &Url::parse("https://example.com/").unwrap(), ), Ok(("un".to_string(), Some("pw".to_string()), "h".to_string())) ) diff --git a/src/scraper.rs b/src/scraper.rs index b13d8cb..46488fd 100644 --- a/src/scraper.rs +++ b/src/scraper.rs @@ -1,3 +1,10 @@ +use std::borrow::Borrow; +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::Path; +use std::sync::Mutex; +use std::time; + use crossbeam::channel::{Receiver, Sender, TryRecvError}; use crossbeam::thread; use encoding_rs::Encoding; @@ -7,12 +14,7 @@ use rand::Rng; use regex::Regex; use url::Url; -use std::borrow::Borrow; -use std::collections::HashMap; -use std::collections::HashSet; -use std::path::Path; -use std::sync::Mutex; -use std::time; +use crate::{error, info, warn}; use super::args; use super::disk; @@ -21,8 +23,6 @@ use super::downloader; use super::response; use super::url_helper; -use crate::{error, info, warn}; - /// Maximum number of empty recv() from the channel static MAX_EMPTY_RECEIVES: usize = 10; @@ -37,8 +37,8 @@ static SLEEP_DURATION: time::Duration = time::Duration::from_millis(SLEEP_MILLIS /// adds more as new URLs are found pub struct Scraper { args: args::Args, - transmitter: Sender<(Url, i32)>, - receiver: Receiver<(Url, i32)>, + transmitter: Sender<(Url, i32, i32)>, + receiver: Receiver<(Url, i32, i32)>, downloader: downloader::Downloader, visited_urls: Mutex>, path_map: Mutex>, @@ -77,8 +77,8 @@ impl Scraper { } /// Push a new URL into the channel - fn push(transmitter: &Sender<(Url, i32)>, url: Url, depth: i32) { - if let Err(e) = transmitter.send((url, depth)) { + fn push(transmitter: &Sender<(Url, i32, i32)>, url: Url, depth: i32, ext_depth: i32) { + if let Err(e) = transmitter.send((url, depth, ext_depth)) { error!("Couldn't push to channel ! {}", e); } } @@ -140,9 +140,10 @@ impl Scraper { /// Proces an html file: add new url to the chanel and prepare for offline navigation fn handle_html( scraper: &Scraper, - transmitter: &Sender<(Url, i32)>, + transmitter: &Sender<(Url, i32, i32)>, url: &Url, depth: i32, + ext_depth: i32, data: &[u8], http_charset: Option, ) -> Vec { @@ -182,16 +183,32 @@ impl Scraper { dom.find_urls_as_strings() .into_iter() - .filter(|candidate| Scraper::should_visit(candidate, &url)) + .filter(|candidate| Scraper::should_visit(candidate)) .for_each(|next_url| { - let mut next_full_url = url.join(&next_url).unwrap(); + let url_to_parse = Scraper::normalize_url(next_url.clone()); + + let mut next_full_url = match url.join(url_to_parse.as_str()) { + Ok(url) => url, + Err(e) => panic!("Failed to parse url: {} | Error: {}", next_url, e), + }; + next_full_url.set_fragment(None); let path = url_helper::to_path(&next_full_url); - if scraper.map_url_path(&next_full_url, path.clone()) - && (scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth) - { - Scraper::push(transmitter, next_full_url, depth + 1); + if scraper.map_url_path(&next_full_url, path.clone()) { + if !Scraper::is_on_another_domain(&next_url, &url) { + // If we are determining for a local domain + if scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth { + Scraper::push(transmitter, next_full_url, depth + 1, ext_depth); + } + } else { + // If we are determining for an external domain + if scraper.args.ext_depth == INFINITE_DEPTH + || ext_depth < scraper.args.ext_depth + { + Scraper::push(transmitter, next_full_url, depth, ext_depth + 1); + } + } } scraper.fix_domtree(next_url, &source_path, &path); @@ -207,7 +224,13 @@ impl Scraper { } /// Process a single URL - fn handle_url(scraper: &Scraper, transmitter: &Sender<(Url, i32)>, url: Url, depth: i32) { + fn handle_url( + scraper: &Scraper, + transmitter: &Sender<(Url, i32, i32)>, + url: Url, + depth: i32, + ext_depth: i32, + ) { match scraper.downloader.get(&url) { Ok(response) => { let data = match response.data { @@ -216,6 +239,7 @@ impl Scraper { transmitter, &url, depth, + ext_depth, &data, response.charset, ), @@ -263,7 +287,7 @@ impl Scraper { pub fn run(&mut self) { /* Push the origin URL and depth (0) through the channel */ self.map_url_path(&self.args.origin, url_helper::to_path(&self.args.origin)); - Scraper::push(&self.transmitter, self.args.origin.clone(), 0); + Scraper::push(&self.transmitter, self.args.origin.clone(), 0, 0); thread::scope(|thread_scope| { for _ in 0..self.args.jobs { @@ -285,9 +309,9 @@ impl Scraper { } TryRecvError::Disconnected => panic!("{}", e), }, - Ok((url, depth)) => { + Ok((url, depth, ext_depth)) => { counter = 0; - Scraper::handle_url(&self_clone, &tx, url, depth); + Scraper::handle_url(&self_clone, &tx, url, depth, ext_depth); self_clone.sleep(&mut rng); } } @@ -313,14 +337,14 @@ impl Scraper { std::thread::sleep(delay_duration); } - /// If a URL should be visited, or does it belong to another domain - fn should_visit(url: &str, base: &Url) -> bool { + /// If a URL should be visited (ignores `mail:`, `javascript:` and other pseudo-links) + fn should_visit(url: &str) -> bool { match Url::parse(url) { /* The given candidate is a valid URL, and not a relative path to - * the next one. Therefore, we have to check if this URL belongs - * to the same domain as our current URL. If the candidate has the - * same domain as our base, then we should visit it */ - Ok(not_ok) => not_ok.domain() == base.domain(), + * the next one. Therefore, we have to check if this URL is valid. + * If it is, we should visit it. + */ + Ok(not_ok) => not_ok.has_host() && !not_ok.cannot_be_a_base(), /* Since we couldn't parse this "URL", then it must be a relative * path or a malformed URL. If the URL is malformed, then it will @@ -328,14 +352,47 @@ impl Scraper { Err(_) => true, } } + + /// Replaces `///` with `//` + /// And `//` with `https://` + /// Without this function, if url is `///./`, the app crashes. + fn normalize_url(url: String) -> String { + if url.starts_with("///") { + return url.replacen("///", "https://", 1); + } else if url.starts_with("//") { + return url.replacen("//", "https://", 1); + } + url + } + + /// If the URL leads to another domain + fn is_on_another_domain(url: &str, base: &Url) -> bool { + let real_url = Scraper::normalize_url(String::from(url)); + + match Url::parse(real_url.as_str()) { + /* The given candidate is a valid URL, and not a relative path to + * the next one. Therefore, we have to check if this URL belongs + * to the same domain as our current URL. If the candidate has the + * same domain as our base, and the depth condition is satisfied, + * then we should visit it, */ + Ok(not_ok) => not_ok.domain() != base.domain(), + + /* Since we couldn't parse this "URL", then it must be a relative + * path or a malformed URL. If the URL is malformed, then it will + * be handled during the join() call in run() */ + Err(_) => false, + } + } } #[cfg(test)] mod tests { - use super::*; - use regex::Regex; use std::path::PathBuf; + use regex::Regex; + + use super::*; + #[test] fn test_zero_delay_range() { let args = args::Args { @@ -344,6 +401,7 @@ mod tests { jobs: 1, tries: 1, depth: 5, + ext_depth: 0, delay: 0, user_agent: "suckit".to_string(), random_range: 0, @@ -366,6 +424,7 @@ mod tests { jobs: 1, tries: 1, depth: 5, + ext_depth: 0, delay: 2, user_agent: "suckit".to_string(), random_range: 5,