mirror of
https://github.com/Skallwar/suckit.git
synced 2026-09-10 07:16:19 -04:00
Respect website architecture (#110)
* scraper: Stop flat archive creation * scrapper: Fix_domtree create only relative path * tests: Fix to_path() tests * tests: Fix tests * scraper: Fix doc Co-authored-by: CohenArthur <arthur.cohen@epita.fr> * misc: Cleanup * Update tests/fixtures/mod.rs Co-authored-by: CohenArthur <arthur.cohen@epita.fr> Co-authored-by: CohenArthur <arthur.cohen@epita.fr>
This commit is contained in:
Esteban Blanc
GitHub
parent
299247a107
commit
96e1f2bf52
428
Cargo.lock
generated
428
Cargo.lock
generated
|
|
@ -15,7 +15,7 @@ version = "0.11.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
|
||||
dependencies = [
|
||||
"winapi 0.3.9",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -32,7 +32,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
|||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"winapi 0.3.9",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -67,21 +67,15 @@ checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820"
|
|||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.3.4"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
|
||||
checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "0.5.6"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16"
|
||||
checksum = "ad1f8e949d755f9d79112b5bb46938e0ef9d3804a0b16dfab13aafcaa5f0fa72"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
|
|
@ -110,8 +104,8 @@ dependencies = [
|
|||
"libc",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"time 0.1.44",
|
||||
"winapi 0.3.9",
|
||||
"time 0.1.43",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -143,14 +137,14 @@ checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd"
|
|||
dependencies = [
|
||||
"atty",
|
||||
"lazy_static",
|
||||
"winapi 0.3.9",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const_fn"
|
||||
version = "0.4.4"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826"
|
||||
checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6"
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
|
|
@ -159,7 +153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
"time 0.2.23",
|
||||
"time 0.2.24",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
|
|
@ -175,7 +169,7 @@ dependencies = [
|
|||
"publicsuffix",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time 0.2.23",
|
||||
"time 0.2.24",
|
||||
"url",
|
||||
]
|
||||
|
||||
|
|
@ -373,22 +367,6 @@ dependencies = [
|
|||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fuchsia-zircon"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"fuchsia-zircon-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fuchsia-zircon-sys"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
|
||||
|
||||
[[package]]
|
||||
name = "futf"
|
||||
version = "0.1.4"
|
||||
|
|
@ -401,51 +379,51 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.8"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64"
|
||||
checksum = "f01c61843314e95f96cc9245702248733a3a3d744e43e2e755e3c7af8348a0a9"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.8"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748"
|
||||
checksum = "db8d3b0917ff63a2a96173133c02818fac4a746b0a57569d3baca9ec0e945e08"
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.8"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb"
|
||||
checksum = "e37c1a51b037b80922864b8eed90692c5cd8abd4c71ce49b77146caa47f3253b"
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.8"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d"
|
||||
checksum = "f6adabac1290109cfa089f79192fb6244ad2c3f1cc2281f3e1dd987592b71feb"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.8"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d"
|
||||
checksum = "a92a0843a2ff66823a8f7c77bffe9a09be2b64e533562c412d63075643ec0038"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.8"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2"
|
||||
checksum = "036a2107cdeb57f6d7322f1b6c363dad67cd63ca3b7d1b925bdf75bd5d96cda9"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project 1.0.2",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"slab",
|
||||
]
|
||||
|
|
@ -478,16 +456,16 @@ checksum = "4060f4657be78b8e766215b02b18a2e862d83745545de804638e2b545e81aee6"
|
|||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"libc",
|
||||
"wasi 0.10.0+wasi-snapshot-preview1",
|
||||
"wasi 0.10.1+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.2.7"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535"
|
||||
checksum = "6b67e66362108efccd8ac053abafc8b7a8d86a37e6e48fc4f6f7485eb5e9e6a5"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"bytes",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
|
|
@ -541,22 +519,22 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84129d298a6d57d246960ff8eb831ca4af3f96d29e2e28848dae275408658e26"
|
||||
checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"bytes",
|
||||
"fnv",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b"
|
||||
checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"bytes",
|
||||
"http",
|
||||
]
|
||||
|
||||
|
|
@ -574,11 +552,11 @@ checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47"
|
|||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.13.9"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf"
|
||||
checksum = "12219dc884514cb4a6a03737f4413c0e01c23a1b059b0156004b23f1e19dccbe"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
|
|
@ -588,7 +566,7 @@ dependencies = [
|
|||
"httparse",
|
||||
"httpdate",
|
||||
"itoa",
|
||||
"pin-project 1.0.2",
|
||||
"pin-project 1.0.4",
|
||||
"socket2",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
|
|
@ -598,15 +576,15 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.4.3"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed"
|
||||
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"bytes",
|
||||
"hyper",
|
||||
"native-tls",
|
||||
"tokio",
|
||||
"tokio-tls",
|
||||
"tokio-native-tls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -630,24 +608,6 @@ dependencies = [
|
|||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iovec"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.3.0"
|
||||
|
|
@ -669,16 +629,6 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kernel32-sys"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
|
||||
dependencies = [
|
||||
"winapi 0.2.8",
|
||||
"winapi-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kuchiki"
|
||||
version = "0.8.1"
|
||||
|
|
@ -699,18 +649,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
|||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.81"
|
||||
version = "0.2.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
|
|
@ -771,45 +712,27 @@ version = "0.3.16"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
|
||||
|
||||
[[package]]
|
||||
name = "mime_guess"
|
||||
version = "2.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212"
|
||||
dependencies = [
|
||||
"mime",
|
||||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.6.23"
|
||||
version = "0.7.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4"
|
||||
checksum = "e50ae3f04d169fcc9bde0b547d1c205219b7157e07ded9c5aff03e0637cb3ed7"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"fuchsia-zircon",
|
||||
"fuchsia-zircon-sys",
|
||||
"iovec",
|
||||
"kernel32-sys",
|
||||
"libc",
|
||||
"log",
|
||||
"miow",
|
||||
"net2",
|
||||
"slab",
|
||||
"winapi 0.2.8",
|
||||
"ntapi",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miow"
|
||||
version = "0.2.2"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d"
|
||||
checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897"
|
||||
dependencies = [
|
||||
"kernel32-sys",
|
||||
"net2",
|
||||
"winapi 0.2.8",
|
||||
"ws2_32-sys",
|
||||
"socket2",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -830,17 +753,6 @@ dependencies = [
|
|||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "net2"
|
||||
version = "0.2.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"libc",
|
||||
"winapi 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "new_debug_unreachable"
|
||||
version = "1.0.4"
|
||||
|
|
@ -853,6 +765,15 @@ version = "0.1.14"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.44"
|
||||
|
|
@ -922,29 +843,10 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.1"
|
||||
name = "pathdiff"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"winapi 0.3.9",
|
||||
]
|
||||
checksum = "877630b3de15c0b64cc52f659345724fbf6bdad9bd9566699fc53688f3c34a34"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
|
|
@ -1017,11 +919,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "pin-project"
|
||||
version = "1.0.2"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7"
|
||||
checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2"
|
||||
dependencies = [
|
||||
"pin-project-internal 1.0.2",
|
||||
"pin-project-internal 1.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1037,9 +939,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "pin-project-internal"
|
||||
version = "1.0.2"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f"
|
||||
checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1048,15 +950,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.1.11"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c"
|
||||
checksum = "ba36e0a6cc5a4c645073f4984f1ed55d09f5857d4de7c14550baa81a39ef5a17"
|
||||
|
||||
[[package]]
|
||||
name = "pin-utils"
|
||||
|
|
@ -1242,9 +1138,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
|
|||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.4.2"
|
||||
version = "1.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c"
|
||||
checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
|
|
@ -1254,9 +1150,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.21"
|
||||
version = "0.6.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189"
|
||||
checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
|
|
@ -1264,17 +1160,17 @@ version = "0.5.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
|
||||
dependencies = [
|
||||
"winapi 0.3.9",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.10.10"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c"
|
||||
checksum = "fd281b1030aa675fb90aa994d07187645bb3c8fc756ca766e7c3070b439de9de"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes 0.5.6",
|
||||
"bytes",
|
||||
"cookie",
|
||||
"cookie_store",
|
||||
"encoding_rs",
|
||||
|
|
@ -1289,15 +1185,14 @@ dependencies = [
|
|||
"lazy_static",
|
||||
"log",
|
||||
"mime",
|
||||
"mime_guess",
|
||||
"native-tls",
|
||||
"percent-encoding",
|
||||
"pin-project-lite 0.2.0",
|
||||
"pin-project-lite",
|
||||
"serde",
|
||||
"serde_urlencoded",
|
||||
"time 0.2.23",
|
||||
"time 0.2.24",
|
||||
"tokio",
|
||||
"tokio-tls",
|
||||
"tokio-native-tls",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
|
|
@ -1327,7 +1222,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"winapi 0.3.9",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1437,28 +1332,6 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serial_test"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0bccbcf40c8938196944a3da0e133e031a33f4d6b72db3bda3cc556e361905d"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"parking_lot",
|
||||
"serial_test_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serial_test_derive"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "servo_arc"
|
||||
version = "0.1.1"
|
||||
|
|
@ -1489,9 +1362,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
|
|||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.6.0"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a55ca5f3b68e41c979bf8c46a6f1da892ca4db8f94023ce0bd32407573b1ac0"
|
||||
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
|
|
@ -1501,7 +1374,7 @@ checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e"
|
|||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"libc",
|
||||
"winapi 0.3.9",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1623,32 +1496,41 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "subprocess"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69b9ad6c3e1b525a55872a4d2f2d404b3c959b7bbcbfd83c364580f68ed157bd"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "suckit"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bytes 0.6.0",
|
||||
"chrono",
|
||||
"colored",
|
||||
"crossbeam",
|
||||
"encoding_rs",
|
||||
"kuchiki",
|
||||
"lazy_static",
|
||||
"percent-encoding",
|
||||
"pathdiff",
|
||||
"rand 0.8.1",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"serial_test",
|
||||
"structopt",
|
||||
"subprocess",
|
||||
"tiny_http",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.57"
|
||||
version = "1.0.58"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4211ce9909eb971f111059df92c45640aad50a619cf55cd76476be803c4c68e6"
|
||||
checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1666,14 +1548,14 @@ dependencies = [
|
|||
"rand 0.7.3",
|
||||
"redox_syscall",
|
||||
"remove_dir_all",
|
||||
"winapi 0.3.9",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.4.1"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "707feda9f2582d5d680d733e38755547a3e8fb471e7ba11452ecfd9ce93a5d3b"
|
||||
checksum = "a9ef557cb397a4f0a5a3a628f06515f78563f2209e64d47055d9dc6052bf5e33"
|
||||
dependencies = [
|
||||
"futf",
|
||||
"mac",
|
||||
|
|
@ -1697,29 +1579,28 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
|
|||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.0.1"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
|
||||
checksum = "bb9bc092d0d51e76b2b19d9d85534ffc9ec2db959a2523cdae0697e2972cd447"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.1.44"
|
||||
version = "0.1.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
|
||||
checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"wasi 0.10.0+wasi-snapshot-preview1",
|
||||
"winapi 0.3.9",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.2.23"
|
||||
version = "0.2.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b"
|
||||
checksum = "273d3ed44dca264b0d6b3665e8d48fb515042d42466fad93d2a45b90ec4058f7"
|
||||
dependencies = [
|
||||
"const_fn",
|
||||
"libc",
|
||||
|
|
@ -1727,7 +1608,7 @@ dependencies = [
|
|||
"stdweb",
|
||||
"time-macros",
|
||||
"version_check",
|
||||
"winapi 0.3.9",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1783,44 +1664,53 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
|
|||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "0.2.24"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48"
|
||||
checksum = "d258221f566b6c803c7b4714abadc080172b272090cdc5e244a6d4dd13c3a6bd"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"iovec",
|
||||
"lazy_static",
|
||||
"autocfg",
|
||||
"bytes",
|
||||
"libc",
|
||||
"memchr",
|
||||
"mio",
|
||||
"num_cpus",
|
||||
"pin-project-lite 0.1.11",
|
||||
"slab",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-tls"
|
||||
version = "0.3.1"
|
||||
name = "tokio-native-tls"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343"
|
||||
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
|
||||
dependencies = [
|
||||
"native-tls",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.3.1"
|
||||
name = "tokio-stream"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499"
|
||||
checksum = "e4cdeb73537e63f98adcd73138af75e3f368ccaecffaa29d7eb61b9f5a440457"
|
||||
dependencies = [
|
||||
"bytes 0.5.6",
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36135b7e7da911f5f8b9331209f7fab4cc13498f3fff52f72a710c78187e3148"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"log",
|
||||
"pin-project-lite 0.1.11",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1836,8 +1726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"log",
|
||||
"pin-project-lite 0.2.0",
|
||||
"pin-project-lite",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
|
|
@ -1866,15 +1755,6 @@ version = "0.2.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
|
||||
dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.4"
|
||||
|
|
@ -1965,9 +1845,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
|
|||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.10.0+wasi-snapshot-preview1"
|
||||
version = "0.10.1+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
|
||||
checksum = "93c6c3420963c5c64bca373b25e77acb562081b9bb4dd5bb864187742186cea9"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
|
|
@ -2047,12 +1927,6 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
|
|
@ -2063,12 +1937,6 @@ dependencies = [
|
|||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-build"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
|
|
@ -2087,15 +1955,5 @@ version = "0.7.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
|
||||
dependencies = [
|
||||
"winapi 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ws2_32-sys"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
|
||||
dependencies = [
|
||||
"winapi 0.2.8",
|
||||
"winapi-build",
|
||||
"winapi",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -24,18 +24,17 @@ include = [
|
|||
[dependencies]
|
||||
structopt = "^0.3"
|
||||
crossbeam = "^0.8"
|
||||
reqwest = { version = "^0.10", features = ["blocking", "cookies"] }
|
||||
reqwest = { version = "^0.11", features = ["blocking", "cookies"] }
|
||||
kuchiki = "^0.8"
|
||||
colored = "2.0"
|
||||
chrono = "^0.4"
|
||||
bytes = "^0.6"
|
||||
percent-encoding = "^2.1"
|
||||
url = "^2.2"
|
||||
rand = "^0.8"
|
||||
regex = "^1.4"
|
||||
encoding_rs = "^0.8"
|
||||
lazy_static = "1.4.0"
|
||||
pathdiff = "^0.2"
|
||||
|
||||
[dev-dependencies]
|
||||
tiny_http = "^0.7"
|
||||
serial_test = "^0.5"
|
||||
subprocess = "^0.2"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ use crossbeam::channel::{Receiver, Sender, TryRecvError};
|
|||
use crossbeam::thread;
|
||||
use encoding_rs::Encoding;
|
||||
use lazy_static::lazy_static;
|
||||
use pathdiff;
|
||||
use rand::Rng;
|
||||
use regex::Regex;
|
||||
use url::Url;
|
||||
|
|
@ -9,6 +10,7 @@ use url::Url;
|
|||
use std::borrow::Borrow;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::process;
|
||||
use std::sync::Mutex;
|
||||
use std::time;
|
||||
|
|
@ -82,15 +84,14 @@ impl Scraper {
|
|||
}
|
||||
}
|
||||
|
||||
/// Fix the URLs contained in the DOM-tree so they point to each other
|
||||
fn fix_domtree(&self, old_url_str: &mut String, new_url: &Url) {
|
||||
let path_map = self.path_map.lock().unwrap();
|
||||
let path = path_map.get(new_url.as_str()).unwrap();
|
||||
/// Fix the URLs contained in the DOM-tree so they point to each other relatively
|
||||
fn fix_domtree(&self, dom_url: &mut String, source_path: &str, dest_path: &str) {
|
||||
let source_path_parent = Path::new(source_path).parent().unwrap().to_str().unwrap(); //Unwrap should be safe, there will alway be at least .../index.html
|
||||
let diff_path = pathdiff::diff_paths(dest_path, source_path_parent).unwrap();
|
||||
let relative_path = diff_path.as_path().to_str().unwrap();
|
||||
|
||||
let new_url_str = url_helper::encode(path);
|
||||
|
||||
old_url_str.clear();
|
||||
old_url_str.push_str(&new_url_str);
|
||||
dom_url.clear();
|
||||
dom_url.push_str(&relative_path);
|
||||
}
|
||||
|
||||
/// Find the charset of the webpage. ``data`` is not a String as this might not be utf8.
|
||||
|
|
@ -166,6 +167,13 @@ impl Scraper {
|
|||
};
|
||||
|
||||
let dom = dom::Dom::new(&String::from_utf8_lossy(&utf8_data).into_owned());
|
||||
let source_path = scraper
|
||||
.path_map
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(url.as_str())
|
||||
.unwrap()
|
||||
.clone();
|
||||
|
||||
dom.find_urls_as_strings()
|
||||
.into_iter()
|
||||
|
|
@ -174,7 +182,7 @@ impl Scraper {
|
|||
let next_full_url = url.join(&next_url).unwrap();
|
||||
let path = url_helper::to_path(&next_full_url);
|
||||
|
||||
if scraper.map_url_path(&next_full_url, path)
|
||||
if scraper.map_url_path(&next_full_url, path.clone())
|
||||
&& (scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth)
|
||||
{
|
||||
let mut next_full_download_url = next_full_url.clone();
|
||||
|
|
@ -182,7 +190,7 @@ impl Scraper {
|
|||
Scraper::push(transmitter, next_full_download_url, depth + 1);
|
||||
}
|
||||
|
||||
scraper.fix_domtree(next_url, &next_full_url);
|
||||
scraper.fix_domtree(next_url, &source_path, &path);
|
||||
});
|
||||
|
||||
let utf8_data = dom.serialize().into_bytes();
|
||||
|
|
|
|||
|
|
@ -1,34 +1,16 @@
|
|||
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
|
||||
use url::Url;
|
||||
|
||||
///Max file name size supported by the file system
|
||||
const FILE_NAME_MAX_LENGTH: usize = 255;
|
||||
///Characters that need to be replaced by encode()
|
||||
const FRAGMENT: &AsciiSet = &CONTROLS.add(b'?');
|
||||
|
||||
///Encode special character with '%' representation
|
||||
pub fn encode(path: &str) -> String {
|
||||
utf8_percent_encode(path, FRAGMENT).to_string()
|
||||
}
|
||||
|
||||
///Convert an Url to the corresponding path
|
||||
/// Convert an Url to the corresponding path
|
||||
pub fn to_path(url: &Url) -> String {
|
||||
let fragment = url.fragment();
|
||||
let mut url = url.clone();
|
||||
url.set_fragment(None);
|
||||
let domain = url.host_str().unwrap();
|
||||
let path = url.path();
|
||||
|
||||
let url = url.as_str().split("://").collect::<Vec<&str>>()[1];
|
||||
|
||||
let mut url = url.replace('/', "_").replace('.', "_");
|
||||
if url.len() >= FILE_NAME_MAX_LENGTH {
|
||||
url = url.split_at(FILE_NAME_MAX_LENGTH).0.to_string(); //Shrink too long file name
|
||||
let mut path = format!("{}{}", domain, path);
|
||||
if path.ends_with("/") {
|
||||
path = format!("{}index.html", path);
|
||||
}
|
||||
let url = url.trim_end_matches('_'); //Remaining '/'
|
||||
|
||||
match fragment {
|
||||
Some(fragment) => format!("{}#{}", url.to_string(), fragment),
|
||||
None => url.to_string(),
|
||||
}
|
||||
path
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -37,22 +19,22 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn url_to_path() {
|
||||
let str = super::to_path(&Url::parse("https://lwn.net/Kernel/index.html").unwrap());
|
||||
|
||||
assert_eq!(str, "lwn.net/Kernel/index.html");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path_index() {
|
||||
let str = super::to_path(&Url::parse("https://lwn.net/Kernel/").unwrap());
|
||||
|
||||
assert_eq!(str, "lwn_net_Kernel");
|
||||
assert_eq!(str, "lwn.net/Kernel/index.html");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path_fragment() {
|
||||
let str = super::to_path(&Url::parse("https://lwn.net/Kernel/#fragment").unwrap());
|
||||
|
||||
assert_eq!(str, "lwn_net_Kernel#fragment");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path_long() {
|
||||
let str = super::to_path(&Url::parse("https://e8v0pez1lofdxoxgg5vwrnaqkjuvpowp9wtgc2eknlfpjdwmmfti8fcwyjzfdgys3nrgyqyeqjkulpyg9kfiqajza2bwxkinhhpohyrnnoy2bak374tcaxh1ycpboolmx8so9yq9kbcj5wu5cgymqndeqasdak0nvl0ijka6fkkmhhvt43l73bn38rewicd4h1ff2omhpni752jtqyzsjub5coh8dlnr3i35udmkzhxo4db3is9gnqmf3hl.comtest").unwrap());
|
||||
|
||||
assert_eq!(str, "e8v0pez1lofdxoxgg5vwrnaqkjuvpowp9wtgc2eknlfpjdwmmfti8fcwyjzfdgys3nrgyqyeqjkulpyg9kfiqajza2bwxkinhhpohyrnnoy2bak374tcaxh1ycpboolmx8so9yq9kbcj5wu5cgymqndeqasdak0nvl0ijka6fkkmhhvt43l73bn38rewicd4h1ff2omhpni752jtqyzsjub5coh8dlnr3i35udmkzhxo4db3is9gnqmf3hl_com");
|
||||
assert_eq!(str, "lwn.net/Kernel/index.html");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use std::process::Stdio;
|
|||
use std::sync::Once;
|
||||
|
||||
const PAGE: &'static str = "tests/fixtures/index.html";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
static START: Once = Once::new();
|
||||
|
||||
#[test]
|
||||
|
|
@ -42,7 +43,7 @@ fn auth_different_host() {
|
|||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let paths = read_dir(output_dir).unwrap();
|
||||
let paths = read_dir(format!("{}/{}", output_dir, IP)).unwrap();
|
||||
// Only the initial invalid response file should be present
|
||||
assert_eq!(paths.count(), 1);
|
||||
|
||||
|
|
@ -69,9 +70,11 @@ fn auth_valid() {
|
|||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let paths = read_dir(output_dir).unwrap();
|
||||
let paths = read_dir(format!("{}/{}", output_dir, IP)).unwrap();
|
||||
// Should load multiple paths, not just the invalid auth response
|
||||
assert!(paths.count() > 1);
|
||||
let paths_count = paths.count();
|
||||
println!("Paths.count() = {}", paths_count);
|
||||
assert!(paths_count > 1);
|
||||
|
||||
std::fs::remove_dir_all(output_dir).unwrap();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use std::process::{Command, Stdio};
|
|||
use std::sync::Once;
|
||||
|
||||
const PAGE_META: &'static str = "tests/fixtures/charset_test_html.html";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
static START: Once = Once::new();
|
||||
|
||||
#[test]
|
||||
|
|
@ -17,6 +18,7 @@ fn test_html_charset_found() {
|
|||
});
|
||||
|
||||
let output_dir = "charset_html_found";
|
||||
let file_dir = format!("{}/{}", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[fixtures::HTTP_ADDR, "-o", output_dir])
|
||||
.stdout(Stdio::inherit())
|
||||
|
|
@ -25,7 +27,7 @@ fn test_html_charset_found() {
|
|||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let file_path = fs::read_dir(output_dir)
|
||||
let file_path = fs::read_dir(file_dir)
|
||||
.unwrap()
|
||||
.next()
|
||||
.unwrap()
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use std::process::{Command, Stdio};
|
|||
use std::sync::Once;
|
||||
|
||||
const PAGE_NO_META: &'static str = "tests/fixtures/charset_test_html_no_meta.html";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
static START: Once = Once::new();
|
||||
|
||||
#[test]
|
||||
|
|
@ -17,6 +18,7 @@ fn test_html_charset_not_found() {
|
|||
});
|
||||
|
||||
let output_dir = "charset_html_not_found";
|
||||
let file_dir = format!("{}/{}", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[fixtures::HTTP_ADDR, "-o", output_dir])
|
||||
.stdout(Stdio::inherit())
|
||||
|
|
@ -25,7 +27,7 @@ fn test_html_charset_not_found() {
|
|||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let file_path = fs::read_dir(output_dir)
|
||||
let file_path = fs::read_dir(file_dir)
|
||||
.unwrap()
|
||||
.next()
|
||||
.unwrap()
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use std::sync::Once;
|
|||
use lazy_static::lazy_static;
|
||||
|
||||
const PAGE_NO_META: &'static str = "tests/fixtures/charset_test_html_no_meta.html";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
static START: Once = Once::new();
|
||||
|
||||
lazy_static! {
|
||||
|
|
@ -24,6 +25,7 @@ fn test_http_charset_found() {
|
|||
});
|
||||
|
||||
let output_dir = "charset_html_found";
|
||||
let file_dir = format!("{}/{}", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[fixtures::HTTP_ADDR, "-o", output_dir])
|
||||
.stdout(Stdio::inherit())
|
||||
|
|
@ -32,7 +34,7 @@ fn test_http_charset_found() {
|
|||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let file_path = fs::read_dir(output_dir)
|
||||
let file_path = fs::read_dir(file_dir)
|
||||
.unwrap()
|
||||
.next()
|
||||
.unwrap()
|
||||
|
|
|
|||
|
|
@ -6,9 +6,8 @@ use std::fs;
|
|||
use std::process::{Command, Stdio};
|
||||
use std::sync::Once;
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
|
||||
const PAGE_NO_META: &'static str = "tests/fixtures/charset_test_html_no_meta.html";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
static START: Once = Once::new();
|
||||
|
||||
#[test]
|
||||
|
|
@ -19,6 +18,7 @@ fn test_http_charset_found() {
|
|||
});
|
||||
|
||||
let output_dir = "charset_html_found";
|
||||
let file_dir = format!("{}/{}", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[fixtures::HTTP_ADDR, "-o", output_dir])
|
||||
.stdout(Stdio::inherit())
|
||||
|
|
@ -27,7 +27,7 @@ fn test_http_charset_found() {
|
|||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let file_path = fs::read_dir(output_dir)
|
||||
let file_path = fs::read_dir(file_dir)
|
||||
.unwrap()
|
||||
.next()
|
||||
.unwrap()
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use std::process::Stdio;
|
|||
use std::sync::Once;
|
||||
|
||||
const PAGE: &'static str = "tests/fixtures/index.html";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
static START: Once = Once::new();
|
||||
|
||||
#[test]
|
||||
|
|
@ -27,8 +28,17 @@ fn test_include_exclude() {
|
|||
// Test to use include flag for downloading pages only matching the given pattern.
|
||||
fn include_filter() {
|
||||
let output_dir = "w2";
|
||||
let files_dir = format!("{}/{}/", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[fixtures::HTTP_ADDR, "-o", "w2", "-i", "mp[3-4]", "-j", "16"])
|
||||
.args(&[
|
||||
fixtures::HTTP_ADDR,
|
||||
"-o",
|
||||
output_dir,
|
||||
"-i",
|
||||
"mp[3-4]",
|
||||
"-j",
|
||||
"16",
|
||||
])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
|
|
@ -36,10 +46,11 @@ fn include_filter() {
|
|||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let paths = read_dir(output_dir).unwrap();
|
||||
let paths = read_dir(&files_dir).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
paths.count(),
|
||||
get_file_count_with_pattern("*_mp3", output_dir).unwrap()
|
||||
get_file_count_with_pattern(".mp3", &files_dir).unwrap()
|
||||
);
|
||||
|
||||
std::fs::remove_dir_all(output_dir).unwrap();
|
||||
|
|
@ -48,6 +59,7 @@ fn include_filter() {
|
|||
// Test demonstrating usage of multiple include patterns for downloading pages only matching the given pattern.
|
||||
fn include_multiple_filters() {
|
||||
let output_dir = "w1";
|
||||
let files_dir = format!("{}/{}/", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[
|
||||
fixtures::HTTP_ADDR,
|
||||
|
|
@ -64,9 +76,9 @@ fn include_multiple_filters() {
|
|||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let paths = read_dir(output_dir).unwrap();
|
||||
let mp3_count = get_file_count_with_pattern("*_mp3", output_dir).unwrap();
|
||||
let txt_count = get_file_count_with_pattern("*_txt", output_dir).unwrap();
|
||||
let paths = read_dir(&files_dir).unwrap();
|
||||
let mp3_count = get_file_count_with_pattern(".mp3", &files_dir).unwrap();
|
||||
let txt_count = get_file_count_with_pattern(".txt", &files_dir).unwrap();
|
||||
assert_eq!(paths.count(), mp3_count + txt_count);
|
||||
|
||||
std::fs::remove_dir_all(output_dir).unwrap();
|
||||
|
|
@ -75,6 +87,7 @@ fn include_multiple_filters() {
|
|||
// Test to use exclude flag for excluding pages matching the given pattern.
|
||||
fn exclude_filter() {
|
||||
let output_dir = "w3";
|
||||
let files_dir = format!("{}/{}/", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[
|
||||
fixtures::HTTP_ADDR,
|
||||
|
|
@ -92,9 +105,9 @@ fn exclude_filter() {
|
|||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let paths = read_dir(output_dir).unwrap();
|
||||
let mp3_count = get_file_count_with_pattern("*_mp3", output_dir).unwrap();
|
||||
let txt_count = get_file_count_with_pattern("*_txt", output_dir).unwrap();
|
||||
let paths = read_dir(&files_dir).unwrap();
|
||||
let mp3_count = get_file_count_with_pattern(".mp3", &files_dir).unwrap();
|
||||
let txt_count = get_file_count_with_pattern(".txt", &files_dir).unwrap();
|
||||
let index_file = 1;
|
||||
assert_eq!(paths.count(), mp3_count + txt_count + index_file);
|
||||
|
||||
|
|
|
|||
53
tests/fixtures/mod.rs
vendored
53
tests/fixtures/mod.rs
vendored
|
|
@ -1,7 +1,7 @@
|
|||
use std::fs::File;
|
||||
use std::process::Command;
|
||||
use std::process::Stdio;
|
||||
use std::thread;
|
||||
|
||||
use subprocess::Exec;
|
||||
use tiny_http::{Header, Response, Server};
|
||||
|
||||
pub const HTTP_ADDR: &'static str = "http://0.0.0.0:8000";
|
||||
|
|
@ -62,41 +62,24 @@ fn check_auth_credentials(auth_header: Option<&Header>) -> bool {
|
|||
}
|
||||
|
||||
pub fn get_file_count_with_pattern(pattern: &str, dir: &str) -> Result<usize, ()> {
|
||||
// Command being run: `ls | grep .mp3 | wc -w`
|
||||
let mut du_output_child = Command::new("ls")
|
||||
.args(&[dir])
|
||||
.stdout(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
// Command being run: `ls | grep pattern | wc -w`
|
||||
|
||||
if let Some(du_output) = du_output_child.stdout.take() {
|
||||
let mut sort_output_child = Command::new("egrep")
|
||||
.arg(pattern)
|
||||
.stdin(du_output)
|
||||
.stdout(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
du_output_child.wait().unwrap();
|
||||
|
||||
if let Some(sort_output) = sort_output_child.stdout.take() {
|
||||
let head_output_child = Command::new("wc")
|
||||
.args(&["-w"])
|
||||
.stdin(sort_output)
|
||||
.stdout(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let head_stdout = head_output_child.wait_with_output().unwrap();
|
||||
sort_output_child.wait().unwrap();
|
||||
return Ok(String::from_utf8(head_stdout.stdout)
|
||||
.unwrap()
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap());
|
||||
}
|
||||
let cmd = {
|
||||
// Pipe ('|') is overloaded here and does a real pipe
|
||||
Exec::shell(format!("ls {}", dir))
|
||||
| Exec::shell(format!("grep '{}'", pattern))
|
||||
| Exec::shell("wc -l")
|
||||
}
|
||||
.capture();
|
||||
|
||||
match cmd {
|
||||
Ok(capture_data) => {
|
||||
let stdout = capture_data.stdout_str();
|
||||
let count = stdout.trim().parse::<usize>().unwrap();
|
||||
Ok(count)
|
||||
}
|
||||
_ => Err(()),
|
||||
}
|
||||
Err(())
|
||||
}
|
||||
|
||||
pub fn do_vecs_match<T: PartialEq>(a: &Vec<T>, b: &Vec<T>) -> bool {
|
||||
|
|
|
|||
Loading…
Reference in a new issue