mirror of
https://github.com/Skallwar/suckit.git
synced 2026-09-10 15:26:24 -04:00
Compare commits
No commits in common. "master" and "v0.1.0" have entirely different histories.
20
.github/dependabot.yml
vendored
20
.github/dependabot.yml
vendored
|
|
@ -1,20 +0,0 @@
|
|||
version: 2
|
||||
updates:
|
||||
# ATM this only create pr to the lockfile
|
||||
# - package-ecosystem: "cargo"
|
||||
# directory: "/"
|
||||
# schedule:
|
||||
# interval: "daily"
|
||||
# time: "02:00" # UTC
|
||||
# labels:
|
||||
# - "domain: deps"
|
||||
# commit-message:
|
||||
# prefix: "robo(deps)"
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
labels:
|
||||
- "domain: ci"
|
||||
commit-message:
|
||||
prefix: "robo(ci)"
|
||||
46
.github/env.sh
vendored
46
.github/env.sh
vendored
|
|
@ -1,46 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [ $# != 1 ]; then
|
||||
echo "Usage: . $0 <rust-target>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
arch=""
|
||||
sysroot=""
|
||||
need_target_linker=""
|
||||
ubuntu_cross_pkg_list=""
|
||||
target_linker=""
|
||||
target_cc=""
|
||||
|
||||
case "$1" in
|
||||
aarch64*)
|
||||
arch="arm64"
|
||||
sysroot="/usr/lib/aarch64-linux-gnu/"
|
||||
ubuntu_cross_pkg_list="gcc-aarch64-linux-gnu"
|
||||
# YES they are the same but otherwise it fails
|
||||
target_linker="aarch64-linux-gnu-gcc"
|
||||
target_cc="aarch64-linux-gnu-gcc"
|
||||
|
||||
;;
|
||||
riscv64*)
|
||||
arch="riscv64"
|
||||
sysroot="/usr/lib/riscv64-linux-gnu/"
|
||||
ubuntu_cross_pkg_list="gcc-riscv64-linux-gnu"
|
||||
# YES they are the same but otherwise it fails
|
||||
target_linker="riscv64-linux-gnu-gcc"
|
||||
target_cc="riscv64-linux-gnu-gcc"
|
||||
|
||||
;;
|
||||
x86_64*)
|
||||
target_linker="gcc"
|
||||
target_cc="gcc"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo MULTILIB_ARCH=$arch
|
||||
echo PKG_CONFIG_SYSROOT_DIR=$sysroot
|
||||
echo TARGET_CC=$target_cc
|
||||
echo CARGO_TARGET_$(echo "$1" | tr 'a-z' 'A-Z' | tr '-' '_' )_LINKER=$target_linker
|
||||
echo UBUNTU_CROSS_PKG_LIST=$ubuntu_cross_pkg_list
|
||||
105
.github/workflows/build_and_test.yml
vendored
105
.github/workflows/build_and_test.yml
vendored
|
|
@ -13,7 +13,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- name: Lint
|
||||
run: |
|
||||
rustup component add rustfmt
|
||||
|
|
@ -21,106 +21,23 @@ jobs:
|
|||
fmt' (version $(cargo fmt --version))"; false)
|
||||
|
||||
tests:
|
||||
needs: [lint]
|
||||
name: Tests ${{ matrix.target }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
target:
|
||||
- x86_64-unknown-linux-gnu
|
||||
os:
|
||||
- ubuntu-latest
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# - name: Set environment variables
|
||||
# run: |
|
||||
# .github/env.sh ${{ matrix.target }} >> $GITHUB_ENV
|
||||
|
||||
# - name: Install qemu-user-static
|
||||
# if: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
|
||||
# run: |
|
||||
# sudo apt update
|
||||
# sudo apt install qemu-user-static
|
||||
|
||||
# - uses: ryankurte/action-apt@v0.3.0
|
||||
# if: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
|
||||
# with:
|
||||
# arch: ${{ env.MULTILIB_ARCH }}
|
||||
# packages: "libssl-dev:${{ env.MULTILIB_ARCH }}"
|
||||
|
||||
- name: Rustup setup
|
||||
uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
default: true
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
- name: Unit tests
|
||||
run: |
|
||||
cargo test --locked
|
||||
cargo test
|
||||
|
||||
build:
|
||||
needs: [lint]
|
||||
|
||||
name: Build ${{ matrix.target }}-${{ matrix.rust }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
target:
|
||||
- x86_64-unknown-linux-gnu
|
||||
- aarch64-unknown-linux-gnu
|
||||
- riscv64gc-unknown-linux-gnu
|
||||
rust:
|
||||
- stable
|
||||
- 1.70.0
|
||||
os:
|
||||
- ubuntu-latest
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint, tests]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Rustup setup
|
||||
uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: ${{ matrix.rust }}
|
||||
default: true
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
.github/env.sh ${{ matrix.target }} >> $GITHUB_ENV
|
||||
|
||||
- name: Install libssl (native)
|
||||
if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install libssl-dev
|
||||
|
||||
# - name: Install libssl (cross)
|
||||
# uses: ryankurte/action-apt@v0.3.0
|
||||
# if: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
|
||||
# with:
|
||||
# arch: ${{ env.MULTILIB_ARCH }}
|
||||
# packages: "${{ env.UBUNTU_CROSS_PKG_LIST }} libssl-dev:${{ env.MULTILIB_ARCH }}"
|
||||
|
||||
- name: Install libssl (cross)
|
||||
if: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
|
||||
run: |
|
||||
sudo dpkg --add-architecture ${{ env.MULTILIB_ARCH }}
|
||||
sudo sed -i'' -E 's/^(deb|deb-src) /\1 [arch=amd64,i386] /' /etc/apt/sources.list
|
||||
echo "deb [arch=${{ env.MULTILIB_ARCH }}] http://ports.ubuntu.com/ubuntu-ports/ $(lsb_release -cs) main restricted universe" | sudo tee /etc/apt/sources.list.d/${{ env.MULTILIB_ARCH }}.list
|
||||
echo "deb [arch=${{ env.MULTILIB_ARCH }}] http://ports.ubuntu.com/ubuntu-ports/ $(lsb_release -cs)-updates main restricted universe" | sudo tee -a /etc/apt/sources.list.d/${{ env.MULTILIB_ARCH }}.list
|
||||
echo "deb [arch=${{ env.MULTILIB_ARCH }}] http://ports.ubuntu.com/ubuntu-ports/ $(lsb_release -cs)-security main restricted universe" | sudo tee -a /etc/apt/sources.list.d/${{ env.MULTILIB_ARCH }}.list
|
||||
sudo apt update
|
||||
sudo apt install ${{ env.UBUNTU_CROSS_PKG_LIST }} libssl-dev:${{ env.MULTILIB_ARCH }}
|
||||
|
||||
#- name: Setup tmate session
|
||||
#uses: mxschmitt/action-tmate@v3
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
- name: Check
|
||||
run: cargo check --locked --target ${{ matrix.target }}
|
||||
|
||||
run: |
|
||||
cargo check
|
||||
- name: Build
|
||||
run: cargo build --locked --target ${{ matrix.target }}
|
||||
run: |
|
||||
cargo build
|
||||
|
|
|
|||
27
.github/workflows/clippy.yml
vendored
27
.github/workflows/clippy.yml
vendored
|
|
@ -1,27 +0,0 @@
|
|||
name: Clippy
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
clippy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Rustup setup
|
||||
uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
override: true
|
||||
|
||||
- uses: actions-rs/clippy-check@v1
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
args: --all-features
|
||||
|
||||
37
.github/workflows/coverage.yml
vendored
37
.github/workflows/coverage.yml
vendored
|
|
@ -1,37 +0,0 @@
|
|||
name: code-coverage
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Rust project
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install stable toolchain
|
||||
uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
override: true
|
||||
|
||||
- name: Run tarpaulin
|
||||
run: |
|
||||
cargo install cargo-tarpaulin
|
||||
cargo tarpaulin --engine llvm --out Xml -- --test-threads 1
|
||||
|
||||
- name: Upload to codecov.io
|
||||
uses: codecov/codecov-action@v3
|
||||
|
||||
- name: Archive code coverage results
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: code-coverage-report
|
||||
path: cobertura.xml
|
||||
78
.github/workflows/release.yml
vendored
78
.github/workflows/release.yml
vendored
|
|
@ -1,78 +0,0 @@
|
|||
name: Release upload artifact
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published, edited]
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: true
|
||||
matrix:
|
||||
target:
|
||||
- x86_64-unknown-linux-gnu
|
||||
- aarch64-unknown-linux-gnu
|
||||
- riscv64gc-unknown-linux-gnu
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Rustup setup
|
||||
uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
default: true
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
.github/env.sh ${{ matrix.target }} >> $GITHUB_ENV
|
||||
|
||||
- name: Install libssl (native)
|
||||
if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install libssl-dev
|
||||
|
||||
- name: Install libssl (cross)
|
||||
uses: ryankurte/action-apt@master
|
||||
if: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
|
||||
with:
|
||||
arch: ${{ env.MULTILIB_ARCH }}
|
||||
packages: "${{ env.UBUNTU_CROSS_PKG_LIST }} libssl-dev:${{ env.MULTILIB_ARCH }}"
|
||||
|
||||
- name: Get release upload url
|
||||
id: get_release
|
||||
uses: bruceadams/get-release@v1.3.2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
|
||||
- name: Build release
|
||||
run: |
|
||||
cargo build --release --locked --target ${{ matrix.target }}
|
||||
|
||||
- name: Compute sha512
|
||||
run: |
|
||||
sha512sum ./target/${{ matrix.target }}/release/suckit | cut -d " " -f 1 > suckit.sha512
|
||||
|
||||
- name: Upload release asset
|
||||
id: upload-release-asset
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ steps.get_release.outputs.upload_url}}
|
||||
asset_path: ./target/${{ matrix.target }}/release/suckit
|
||||
asset_name: suckit-${{ steps.get_release.outputs.tag_name }}-${{ matrix.target }}
|
||||
asset_content_type: application/x-elf
|
||||
|
||||
- name: Upload release asset checksum
|
||||
id: upload-release-asset-sha512
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ steps.get_release.outputs.upload_url}}
|
||||
asset_path: ./suckit.sha512
|
||||
asset_name: suckit-${{ steps.get_release.outputs.tag_name }}-${{ matrix.target }}.sha512
|
||||
asset_content_type: text/plain
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -6,6 +6,3 @@
|
|||
tags
|
||||
|
||||
.idea/
|
||||
|
||||
# Ignore local web server folder
|
||||
tests/local_server
|
||||
|
|
|
|||
1586
Cargo.lock
generated
1586
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
50
Cargo.toml
50
Cargo.toml
|
|
@ -1,9 +1,9 @@
|
|||
[package]
|
||||
name = "suckit"
|
||||
version = "0.2.0"
|
||||
version = "0.1.0"
|
||||
edition = "2018"
|
||||
authors = ["Esteban \"Skallwar\" Blanc <estblcsk@gmail.com>",
|
||||
"Arthur \"CohenArthur\" Cohen <cohenarthur.dev@gmail.com>"]
|
||||
"Arthur \"CohenArthur\" Cohen <arthur.cohen@epita.fr>"]
|
||||
license = "MIT OR Apache-2.0"
|
||||
homepage = "https://github.com/skallwar/suckit"
|
||||
repository = "https://github.com/skallwar/suckit"
|
||||
|
|
@ -19,39 +19,21 @@ include = [
|
|||
"src/*",
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
msrv = "1.67.0"
|
||||
|
||||
[lib]
|
||||
name = "suckit"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "suckit"
|
||||
path = "src/bin/suckit.rs"
|
||||
doc = false
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
structopt = "^0.3"
|
||||
crossbeam = "^0.8"
|
||||
reqwest = { version = "^0.11", features = ["blocking", "cookies"] }
|
||||
kuchiki = "^0.8"
|
||||
colored = "2.0"
|
||||
chrono = "^0.4"
|
||||
url = "^2.3"
|
||||
rand = "^0.8"
|
||||
regex = "^1.6"
|
||||
encoding_rs = "^0.8"
|
||||
lazy_static = "1.4.0"
|
||||
pathdiff = "^0.2"
|
||||
md5 = "^0.7"
|
||||
symlink = "^0.1.0"
|
||||
structopt = "0.3"
|
||||
crossbeam = "0.7"
|
||||
reqwest = { version = "0.10", features = ["blocking", "cookies"] }
|
||||
pretty_assertions = "0.6"
|
||||
kuchiki = "0.8"
|
||||
colored = "1.9"
|
||||
chrono = "0.4"
|
||||
bytes = "0.5"
|
||||
percent-encoding = "2.1"
|
||||
url = "2.1"
|
||||
rand = "0.7"
|
||||
regex = "1.3.7"
|
||||
|
||||
[dev-dependencies]
|
||||
tiny_http = "^0.12"
|
||||
subprocess = "^0.2"
|
||||
mktemp = "^0.5"
|
||||
portpicker = "^0.1"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
tiny_http = "0.7.0"
|
||||
|
|
|
|||
83
README.md
83
README.md
|
|
@ -1,18 +1,11 @@
|
|||
|
||||
[](https://codecov.io/gh/Skallwar/suckit)
|
||||
[](https://crates.io/crates/suckit)
|
||||
[](https://docs.rs/suckit)
|
||||
[](https://deps.rs/repo/github/Skallwar/suckit)
|
||||
[](https://opensource.org/licenses/Apache-2.0)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
|
||||
|
||||
# SuckIT
|
||||
|
||||
`SuckIT` allows you to recursively visit and download a website's content to
|
||||
your disk.
|
||||
|
||||
|
||||
|
||||
|
||||
# Features
|
||||
|
||||
|
|
@ -20,62 +13,22 @@ your disk.
|
|||
* [x] Uses multithreading
|
||||
* [x] Writes the website's content to your disk
|
||||
* [x] Enables offline navigation
|
||||
* [x] Offers random delays to avoid IP banning
|
||||
* [ ] Saves application state on CTRL-C for later pickup
|
||||
* [ ] Offers random delays to avoid IP banning
|
||||
|
||||
# Options
|
||||
```console
|
||||
USAGE:
|
||||
suckit [FLAGS] [OPTIONS] <url>
|
||||
|
||||
FLAGS:
|
||||
-c, --continue-on-error Flag to enable or disable exit on error
|
||||
--disable-certs-checks Dissable SSL certificates verification
|
||||
--dry-run Do everything without saving the files to the disk
|
||||
-h, --help Prints help information
|
||||
-V, --version Prints version information
|
||||
-v, --verbose Enable more information regarding the scraping process
|
||||
--visit-filter-is-download-filter Use the dowload filter in/exclude regexes for visiting as well
|
||||
|
||||
OPTIONS:
|
||||
-a, --auth <auth>...
|
||||
HTTP basic authentication credentials space-separated as "username password host". Can be repeated for
|
||||
multiple credentials as "u1 p1 h1 u2 p2 h2"
|
||||
--cookie <cookie>
|
||||
Cookie to send with each request, format: key1=value1;key2=value2 [default: ]
|
||||
|
||||
--delay <delay>
|
||||
Add a delay in seconds between downloads to reduce the likelihood of getting banned [default: 0]
|
||||
|
||||
-d, --depth <depth>
|
||||
Maximum recursion depth to reach when visiting. Default is -1 (infinity) [default: -1]
|
||||
|
||||
-e, --exclude-download <exclude-download>
|
||||
Regex filter to exclude saving pages that match this expression [default: $^]
|
||||
|
||||
--exclude-visit <exclude-visit>
|
||||
Regex filter to exclude visiting pages that match this expression [default: $^]
|
||||
|
||||
--ext-depth <ext-depth>
|
||||
Maximum recursion depth to reach when visiting external domains. Default is 0. -1 means infinity [default:
|
||||
0]
|
||||
-i, --include-download <include-download>
|
||||
Regex filter to limit to only saving pages that match this expression [default: .*]
|
||||
|
||||
--include-visit <include-visit>
|
||||
Regex filter to limit to only visiting pages that match this expression [default: .*]
|
||||
|
||||
-j, --jobs <jobs> Maximum number of threads to use concurrently [default: 1]
|
||||
-o, --output <output> Output directory
|
||||
--random-range <random-range>
|
||||
Generate an extra random delay between downloads, from 0 to this number. This is added to the base delay
|
||||
seconds [default: 0]
|
||||
-t, --tries <tries> Maximum amount of retries on download failure [default: 20]
|
||||
-u, --user-agent <user-agent> User agent to be used for sending requests [default: suckit]
|
||||
|
||||
ARGS:
|
||||
<url> Entry point of the scraping
|
||||
```
|
||||
|Option|Behavior|
|
||||
|---|---|
|
||||
|`-h, --help`|Displays help information|
|
||||
|`-v, --verbose`|Activate Verbose output|
|
||||
|`-d, --depth`|Specify the level of depth to go to when visiting the website|
|
||||
|`-j, --jobs`|Number of threads to use|
|
||||
|`-o, --output`|Output directory where the downloaded files are written|
|
||||
|`-t, --tries`|Number of times to retry when the downloading of a page fails|
|
||||
|`-u, --user-agent`|User agent to be used for sending requests|
|
||||
|`-i, --include`|Specify a regex to include pages that match this pattern|
|
||||
|`-e, --exclude`|Specify a regex to exclude pages that match this pattern|
|
||||
|
||||
# Example
|
||||
|
||||
|
|
@ -83,7 +36,7 @@ A common use case could be the following:
|
|||
|
||||
`suckit http://books.toscrape.com -j 8 -o /path/to/downloaded/pages/`
|
||||
|
||||
|
||||
[](https://asciinema.org/a/327889)
|
||||
|
||||
# Installation
|
||||
|
||||
|
|
@ -99,14 +52,6 @@ instructions on how to install Rust.
|
|||
|
||||
* Now, run it from anywhere with the `suckit` command.
|
||||
|
||||
### Arch Linux
|
||||
|
||||
`suckit` can be installed from available [AUR packages](https://aur.archlinux.org/packages/?O=0&SeB=b&K=suckit&outdated=&SB=n&SO=a&PP=50&do_Search=Go) using an [AUR helper](https://wiki.archlinux.org/index.php/AUR_helpers). For example,
|
||||
|
||||
```
|
||||
yay -S suckit
|
||||
```
|
||||
|
||||
__Want to contribute ? Feel free to
|
||||
[open an issue](https://github.com/Skallwar/suckit/issues/new) or
|
||||
[submit a PR](https://github.com/Skallwar/suckit/compare) !__
|
||||
|
|
|
|||
130
flake.lock
130
flake.lock
|
|
@ -1,130 +0,0 @@
|
|||
{
|
||||
"nodes": {
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1705309234,
|
||||
"narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils_2": {
|
||||
"inputs": {
|
||||
"systems": "systems_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1681202837,
|
||||
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1705774713,
|
||||
"narHash": "sha256-j6ADaDH9XiumUzkTPlFyCBcoWYhO83lfgiSqEJF2zcs=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "1b64fc1287991a9cce717a01c1973ef86cb1af0b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-23.11",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1681358109,
|
||||
"narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixpkgs-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"rust-overlay": "rust-overlay"
|
||||
}
|
||||
},
|
||||
"rust-overlay": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils_2",
|
||||
"nixpkgs": "nixpkgs_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1705889935,
|
||||
"narHash": "sha256-77KPBK5e0ACNzIgJDMuptTtEqKvHBxTO3ksqXHHVO+4=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "e36f66bb10b09f5189dc3b1706948eaeb9a1c555",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_2": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
29
flake.nix
29
flake.nix
|
|
@ -1,29 +0,0 @@
|
|||
{
|
||||
description = "SuckIT devshell";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
|
||||
rust-overlay.url = "github:oxalica/rust-overlay";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, rust-overlay, flake-utils, ... }:
|
||||
flake-utils.lib.eachDefaultSystem (system:
|
||||
let
|
||||
overlays = [ (import rust-overlay) ];
|
||||
pkgs = import nixpkgs {
|
||||
inherit system overlays;
|
||||
};
|
||||
in
|
||||
with pkgs;
|
||||
{
|
||||
devShell = mkShell {
|
||||
buildInputs = [
|
||||
rust-bin.stable.latest.default
|
||||
openssl
|
||||
pkg-config
|
||||
];
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 37 KiB |
87
src/args.rs
87
src/args.rs
|
|
@ -33,17 +33,10 @@ pub struct Args {
|
|||
short,
|
||||
long,
|
||||
default_value = "-1",
|
||||
help = "Maximum recursion depth to reach when visiting. Default is -1 (infinity)"
|
||||
help = "Maximum recursion depth to reach when visiting. -1 is the default and will go as far as it can"
|
||||
)]
|
||||
pub depth: i32,
|
||||
|
||||
#[structopt(
|
||||
long,
|
||||
default_value = "0",
|
||||
help = "Maximum recursion depth to reach when visiting external domains. Default is 0. -1 means infinity"
|
||||
)]
|
||||
pub ext_depth: i32,
|
||||
|
||||
///Number of retries when downloading a page fails
|
||||
#[structopt(
|
||||
short,
|
||||
|
|
@ -86,79 +79,29 @@ pub struct Args {
|
|||
)]
|
||||
pub user_agent: String,
|
||||
|
||||
/// Cookie header
|
||||
#[structopt(
|
||||
long,
|
||||
default_value = "",
|
||||
help = "Cookie to send with each request, format: key1=value1;key2=value2"
|
||||
)]
|
||||
pub cookie: String,
|
||||
|
||||
/// Regex filter to limit visiting pages to only matched ones
|
||||
#[structopt(
|
||||
long,
|
||||
default_value = ".*",
|
||||
parse(try_from_str = parse_regex),
|
||||
help = "Regex filter to limit to only visiting pages that match this expression"
|
||||
)]
|
||||
pub include_visit: Regex,
|
||||
|
||||
/// Regex filter to limit visiting pages to only matched ones
|
||||
#[structopt(
|
||||
long,
|
||||
default_value = "$^",
|
||||
parse(try_from_str = parse_regex),
|
||||
help = "Regex filter to exclude visiting pages that match this expression"
|
||||
)]
|
||||
pub exclude_visit: Regex,
|
||||
|
||||
/// Regex filter to limit saving pages to only matched ones
|
||||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
default_value = ".*",
|
||||
parse(try_from_str = parse_regex),
|
||||
help = "Regex filter to limit to only saving pages that match this expression"
|
||||
)]
|
||||
pub include_download: Regex,
|
||||
|
||||
/// Regex filter to limit saving pages to only matched ones
|
||||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
default_value = "$^",
|
||||
parse(try_from_str = parse_regex),
|
||||
help = "Regex filter to exclude saving pages that match this expression"
|
||||
)]
|
||||
pub exclude_download: Regex,
|
||||
|
||||
/// If set, set the visit filter to the values of the download filter
|
||||
#[structopt(
|
||||
long,
|
||||
help = "Use the dowload filter in/exclude regexes for visiting as well"
|
||||
)]
|
||||
pub visit_filter_is_download_filter: bool,
|
||||
|
||||
/// HTTP basic authentication credentials
|
||||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
use_delimiter = true,
|
||||
value_delimiter = " ",
|
||||
help = "HTTP basic authentication credentials space-separated as \"username password host\". Can be repeated for multiple credentials as \"u1 p1 h1 u2 p2 h2\""
|
||||
default_value = ".*",
|
||||
parse(try_from_str = parse_regex),
|
||||
help = "Regex filter to limit to only saving pages that match this expression"
|
||||
)]
|
||||
pub auth: Vec<String>,
|
||||
pub include: Regex,
|
||||
|
||||
/// Regex filter to limit saving pages to only matched ones
|
||||
#[structopt(
|
||||
short,
|
||||
long,
|
||||
default_value = "$^",
|
||||
parse(try_from_str = parse_regex),
|
||||
help = "Regex filter to exclude saving pages that match this expression"
|
||||
)]
|
||||
pub exclude: Regex,
|
||||
|
||||
/// Decides if we should bail out on download error (like, too many redirects)
|
||||
#[structopt(short, long, help = "Flag to enable or disable exit on error")]
|
||||
pub continue_on_error: bool,
|
||||
|
||||
/// If set, run without saving anything to the disk
|
||||
#[structopt(long, help = "Do everything without saving the files to the disk")]
|
||||
pub dry_run: bool,
|
||||
|
||||
#[structopt(long, help = "Dissable SSL certificates verification")]
|
||||
pub disable_certs_checks: bool,
|
||||
}
|
||||
|
||||
impl Args {
|
||||
|
|
|
|||
|
|
@ -1,10 +0,0 @@
|
|||
use suckit::args::Args;
|
||||
use suckit::scraper::Scraper;
|
||||
|
||||
fn main() {
|
||||
let args = Args::collect();
|
||||
|
||||
let mut scraper = Scraper::new(args);
|
||||
|
||||
scraper.run();
|
||||
}
|
||||
36
src/disk.rs
36
src/disk.rs
|
|
@ -1,11 +1,7 @@
|
|||
use pathdiff;
|
||||
use std::fs;
|
||||
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use symlink::symlink_file;
|
||||
|
||||
use crate::{error, warn};
|
||||
|
||||
///Save content in a file
|
||||
|
|
@ -16,8 +12,11 @@ pub fn save_file(file_name: &str, content: &[u8], path: &Option<PathBuf>) {
|
|||
};
|
||||
|
||||
if let Some(parent) = path.parent() {
|
||||
if let Err(err) = fs::create_dir_all(parent) {
|
||||
error!("Couldn't create folder {}: {}", parent.display(), err);
|
||||
match fs::create_dir_all(parent) {
|
||||
Err(err) => {
|
||||
error!("Couldn't create folder {}: {}", parent.display(), err);
|
||||
}
|
||||
Ok(()) => (),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -33,33 +32,16 @@ pub fn save_file(file_name: &str, content: &[u8], path: &Option<PathBuf>) {
|
|||
|
||||
///Create a symlink
|
||||
pub fn symlink(source: &str, destination: &str, path: &Option<PathBuf>) {
|
||||
let source = match path {
|
||||
Some(path) => path.join(source),
|
||||
None => PathBuf::from(source),
|
||||
};
|
||||
|
||||
if let Some(parent) = source.parent() {
|
||||
match fs::create_dir_all(parent) {
|
||||
Err(err) => {
|
||||
error!("Couldn't create folder {}: {}", parent.display(), err);
|
||||
}
|
||||
Ok(()) => (),
|
||||
}
|
||||
}
|
||||
|
||||
let destination = match path {
|
||||
Some(path) => path.join(destination),
|
||||
None => PathBuf::from(destination),
|
||||
};
|
||||
|
||||
let target = pathdiff::diff_paths(&destination, &source.parent().unwrap()).unwrap();
|
||||
|
||||
if let Err(err) = symlink_file(&target, &source) {
|
||||
if let Err(_) = std::os::unix::fs::symlink(source, &destination) {
|
||||
warn!(
|
||||
"Couldn't create symlink\n{} -> {}:\n{:#?}",
|
||||
source.display(),
|
||||
target.display(),
|
||||
err,
|
||||
"{} is already present, coulnd't create a symlink to {}",
|
||||
destination.display(),
|
||||
source,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,88 +1,24 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
use reqwest::header::{HeaderMap, HeaderValue, COOKIE};
|
||||
use super::response::{Response, ResponseData};
|
||||
use url::Url;
|
||||
|
||||
use crate::warn;
|
||||
|
||||
use super::response::{Response, ResponseData};
|
||||
|
||||
const AUTH_CHUNK_SIZE: usize = 3;
|
||||
|
||||
///A Downloader to download web content
|
||||
pub struct Downloader {
|
||||
client: reqwest::blocking::Client,
|
||||
tries: usize,
|
||||
auth_map: HashMap<String, (String, Option<String>)>,
|
||||
}
|
||||
|
||||
/// Parse HTTP authentication credentials from string iterable
|
||||
fn parse_auth(auth: &[String], origin: &Url) -> Result<(String, Option<String>, String), String> {
|
||||
// Convert any empty strings to None
|
||||
let auth: Vec<Option<String>> = auth
|
||||
.iter()
|
||||
.map(|s| match s.as_ref() {
|
||||
"" => None,
|
||||
s => Some(s.to_string()),
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Match on auth values and origin host, defaulting to the origin host if host not provided
|
||||
match (auth.as_slice(), origin.host_str()) {
|
||||
([Some(username)], Some(origin_host)) => {
|
||||
Ok((username.to_string(), None, origin_host.to_string()))
|
||||
}
|
||||
([Some(username), password], Some(origin_host)) => Ok((
|
||||
username.to_string(),
|
||||
password.clone(),
|
||||
origin_host.to_string(),
|
||||
)),
|
||||
([Some(username), password, None, ..], Some(origin_host)) => Ok((
|
||||
username.to_string(),
|
||||
password.clone(),
|
||||
origin_host.to_string(),
|
||||
)),
|
||||
([Some(username), password, Some(host), ..], _) => {
|
||||
Ok((username.to_string(), password.clone(), host.to_string()))
|
||||
}
|
||||
_ => Err("Invalid arguments supplied to auth".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
impl Downloader {
|
||||
/// Create a new Downloader
|
||||
pub fn new(
|
||||
tries: usize,
|
||||
user_agent: &str,
|
||||
cookie: &str,
|
||||
disable_certs_checks: bool,
|
||||
auth: &[String],
|
||||
origin: &Url,
|
||||
) -> Downloader {
|
||||
// Create a mapping of hosts to username, password tuples for authentication
|
||||
let mut auth_map = HashMap::new();
|
||||
// Iterate over the auth string in chunks of 3 items each for (username, password, host)
|
||||
for auth_chunk in auth.chunks(AUTH_CHUNK_SIZE) {
|
||||
// Throwing the error with panic! for now if parsing fails
|
||||
let (username, password, host) = parse_auth(auth_chunk, origin).unwrap();
|
||||
auth_map.insert(host, (username, password));
|
||||
}
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(COOKIE, HeaderValue::from_str(cookie).unwrap());
|
||||
|
||||
pub fn new(tries: usize, user_agent: &str) -> Downloader {
|
||||
Downloader {
|
||||
client: reqwest::blocking::ClientBuilder::new()
|
||||
.default_headers(headers)
|
||||
.danger_accept_invalid_certs(disable_certs_checks)
|
||||
.cookie_store(true)
|
||||
.user_agent(user_agent)
|
||||
.build()
|
||||
.unwrap(),
|
||||
tries,
|
||||
auth_map,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -103,64 +39,30 @@ impl Downloader {
|
|||
}
|
||||
}
|
||||
|
||||
/// Load HTTP auth credentials in a username, password tuple based on the host string
|
||||
fn get_auth(&self, url: &Url) -> Option<&(String, Option<String>)> {
|
||||
if let Some(host) = url.host_str() {
|
||||
self.auth_map.get(&host.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
///Download the content at this url
|
||||
fn make_request(&self, url: &Url) -> Result<Response, reqwest::Error> {
|
||||
let req = self.client.get(url.clone());
|
||||
let req = match self.get_auth(url) {
|
||||
Some((username, password)) => req.basic_auth(username, password.clone()),
|
||||
None => req,
|
||||
};
|
||||
match req.send() {
|
||||
match self.client.get(url.clone()).send() {
|
||||
Ok(mut data) => {
|
||||
lazy_static! {
|
||||
static ref DATA_TYPE_REGEX: Regex =
|
||||
Regex::new(r#"^.*(\b[a-z]+/[a-z-+\.]+).*$"#).unwrap();
|
||||
static ref CHARSET_REGEX: Regex =
|
||||
Regex::new(r#"^.*charset\s*=\s*["']?([^"'\s;]+).*$"#).unwrap();
|
||||
}
|
||||
let data_type = match data.headers().get("content-type") {
|
||||
Some(data_type) => data_type.to_str().unwrap(),
|
||||
None => "text/html",
|
||||
};
|
||||
|
||||
let (data_type, charset): (String, Option<String>) =
|
||||
match data.headers().get("content-type") {
|
||||
Some(content_type_header) => {
|
||||
let content_type = content_type_header.to_str().unwrap();
|
||||
let data_type_captures =
|
||||
DATA_TYPE_REGEX.captures_iter(content_type).next();
|
||||
let data_type = data_type_captures
|
||||
.map_or(String::from("text/html"), |first| {
|
||||
first.get(1).unwrap().as_str().to_lowercase()
|
||||
});
|
||||
let charset_captures = CHARSET_REGEX.captures_iter(content_type).next();
|
||||
let charset = charset_captures
|
||||
.map(|first| first.get(1).unwrap().as_str().to_lowercase());
|
||||
(data_type, charset)
|
||||
}
|
||||
None => (String::from("text/html"), None),
|
||||
};
|
||||
|
||||
let filename = if !Downloader::is_html(&data_type) {
|
||||
let filename = if !Downloader::is_html(data_type) {
|
||||
Downloader::get_filename(data.headers())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut raw_data: Vec<u8> = Vec::new();
|
||||
data.copy_to(&mut raw_data).unwrap();
|
||||
let response_data = if Downloader::is_html(&data_type) {
|
||||
ResponseData::Html(raw_data)
|
||||
let data = if Downloader::is_html(data_type) {
|
||||
ResponseData::Html(data.text().unwrap())
|
||||
} else {
|
||||
let mut raw_data: Vec<u8> = Vec::new();
|
||||
data.copy_to(&mut raw_data).unwrap();
|
||||
ResponseData::Other(raw_data)
|
||||
};
|
||||
|
||||
Ok(Response::new(response_data, filename, charset))
|
||||
Ok(Response::new(data, filename))
|
||||
}
|
||||
|
||||
Err(e) => {
|
||||
|
|
@ -191,39 +93,9 @@ mod tests {
|
|||
#[test]
|
||||
fn test_download_url() {
|
||||
let url: Url = Url::parse("https://lwn.net").unwrap();
|
||||
match Downloader::new(1, "suckit", "", false, &[], &url).get(&url) {
|
||||
match Downloader::new(1, "suckit").get(&url) {
|
||||
Err(e) => assert!(false, "Fail to download lwn.net: {:?}", e),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_auth() {
|
||||
assert_eq!(
|
||||
parse_auth(
|
||||
&["".to_string(), "pw".to_string()],
|
||||
&Url::parse("https://example.com/").unwrap(),
|
||||
),
|
||||
Err("Invalid arguments supplied to auth".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
parse_auth(
|
||||
&["username".to_string()],
|
||||
&Url::parse("https://example.com/").unwrap(),
|
||||
),
|
||||
Ok(("username".to_string(), None, "example.com".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_auth(
|
||||
&[
|
||||
"un".to_string(),
|
||||
"pw".to_string(),
|
||||
"h".to_string(),
|
||||
"t".to_string()
|
||||
],
|
||||
&Url::parse("https://example.com/").unwrap(),
|
||||
),
|
||||
Ok(("un".to_string(), Some("pw".to_string()), "h".to_string()))
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
pub mod args;
|
||||
pub mod disk;
|
||||
pub mod dom;
|
||||
pub mod downloader;
|
||||
pub mod logger;
|
||||
pub mod response;
|
||||
pub mod scraper;
|
||||
pub mod url_helper;
|
||||
|
|
@ -39,7 +39,7 @@ impl Logger {
|
|||
"ERROR".red(),
|
||||
message
|
||||
);
|
||||
panic!("{}", message)
|
||||
panic!(message)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
18
src/main.rs
Normal file
18
src/main.rs
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
mod args;
|
||||
mod disk;
|
||||
mod dom;
|
||||
mod downloader;
|
||||
mod logger;
|
||||
mod response;
|
||||
mod scraper;
|
||||
mod url_helper;
|
||||
|
||||
use scraper::Scraper;
|
||||
|
||||
fn main() {
|
||||
let args = args::Args::collect();
|
||||
|
||||
let mut scraper = Scraper::new(args);
|
||||
|
||||
scraper.run();
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
/// Separates HTML responses and other content (PDFs, images...)
|
||||
pub enum ResponseData {
|
||||
Html(Vec<u8>),
|
||||
Html(String),
|
||||
Other(Vec<u8>),
|
||||
}
|
||||
|
||||
|
|
@ -8,16 +8,11 @@ pub enum ResponseData {
|
|||
pub struct Response {
|
||||
pub data: ResponseData,
|
||||
pub filename: Option<String>,
|
||||
pub charset: Option<String>,
|
||||
}
|
||||
|
||||
impl Response {
|
||||
///Create a new Response
|
||||
pub fn new(data: ResponseData, filename: Option<String>, charset: Option<String>) -> Response {
|
||||
Response {
|
||||
data,
|
||||
filename,
|
||||
charset,
|
||||
}
|
||||
pub fn new(data: ResponseData, filename: Option<String>) -> Response {
|
||||
Response { data, filename }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
337
src/scraper.rs
337
src/scraper.rs
|
|
@ -1,28 +1,25 @@
|
|||
use std::borrow::Borrow;
|
||||
use crossbeam::channel::{Receiver, Sender, TryRecvError};
|
||||
use crossbeam::thread;
|
||||
use url::Url;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::process;
|
||||
use std::sync::Mutex;
|
||||
use std::time;
|
||||
|
||||
use crossbeam::channel::{Receiver, Sender, TryRecvError};
|
||||
use crossbeam::thread;
|
||||
use encoding_rs::Encoding;
|
||||
use lazy_static::lazy_static;
|
||||
use pathdiff;
|
||||
use rand::Rng;
|
||||
use regex::Regex;
|
||||
use url::Url;
|
||||
|
||||
use crate::{error, info, warn};
|
||||
use super::downloader;
|
||||
|
||||
use super::args;
|
||||
use super::disk;
|
||||
use super::dom;
|
||||
use super::downloader;
|
||||
use super::response;
|
||||
use super::url_helper;
|
||||
|
||||
use crate::{error, info};
|
||||
|
||||
/// Maximum number of empty recv() from the channel
|
||||
static MAX_EMPTY_RECEIVES: usize = 10;
|
||||
|
||||
|
|
@ -30,15 +27,15 @@ static MAX_EMPTY_RECEIVES: usize = 10;
|
|||
static INFINITE_DEPTH: i32 = -1;
|
||||
|
||||
/// Sleep duration on empty recv()
|
||||
static SLEEP_MILLIS: u64 = 500;
|
||||
static SLEEP_MILLIS: u64 = 100;
|
||||
static SLEEP_DURATION: time::Duration = time::Duration::from_millis(SLEEP_MILLIS);
|
||||
|
||||
/// Producer and Consumer data structure. Handles the incoming requests and
|
||||
/// adds more as new URLs are found
|
||||
pub struct Scraper {
|
||||
args: args::Args,
|
||||
transmitter: Sender<(Url, i32, i32)>,
|
||||
receiver: Receiver<(Url, i32, i32)>,
|
||||
transmitter: Sender<(Url, i32)>,
|
||||
receiver: Receiver<(Url, i32)>,
|
||||
downloader: downloader::Downloader,
|
||||
visited_urls: Mutex<HashSet<String>>,
|
||||
path_map: Mutex<HashMap<String, String>>,
|
||||
|
|
@ -49,21 +46,8 @@ impl Scraper {
|
|||
pub fn new(args: args::Args) -> Scraper {
|
||||
let (tx, rx) = crossbeam::channel::unbounded();
|
||||
|
||||
let mut args = args;
|
||||
if args.visit_filter_is_download_filter {
|
||||
args.include_visit = args.include_download.clone();
|
||||
args.exclude_visit = args.exclude_download.clone();
|
||||
}
|
||||
|
||||
Scraper {
|
||||
downloader: downloader::Downloader::new(
|
||||
args.tries,
|
||||
&args.user_agent,
|
||||
&args.cookie,
|
||||
args.disable_certs_checks,
|
||||
&args.auth,
|
||||
&args.origin,
|
||||
),
|
||||
downloader: downloader::Downloader::new(args.tries, &args.user_agent),
|
||||
args,
|
||||
transmitter: tx,
|
||||
receiver: rx,
|
||||
|
|
@ -85,175 +69,60 @@ impl Scraper {
|
|||
}
|
||||
|
||||
/// Push a new URL into the channel
|
||||
fn push(transmitter: &Sender<(Url, i32, i32)>, url: Url, depth: i32, ext_depth: i32) {
|
||||
if let Err(e) = transmitter.send((url, depth, ext_depth)) {
|
||||
fn push(transmitter: &Sender<(Url, i32)>, url: Url, depth: i32) {
|
||||
if let Err(e) = transmitter.send((url, depth)) {
|
||||
error!("Couldn't push to channel ! {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
/// Fix the URLs contained in the DOM-tree so they point to each other relatively
|
||||
fn fix_domtree(&self, dom_url: &mut String, source_path: &str, dest_path: &str) {
|
||||
let source_path_parent = Path::new(source_path).parent().unwrap().to_str().unwrap(); //Unwrap should be safe, there will alway be at least .../index.html
|
||||
let diff_path = pathdiff::diff_paths(dest_path, source_path_parent).unwrap();
|
||||
let relative_path = diff_path.as_path().to_str().unwrap();
|
||||
/// Fix the URLs contained in the DOM-tree so they point to each other
|
||||
fn fix_domtree(&self, old_url_str: &mut String, new_url: &Url) {
|
||||
let path_map = self.path_map.lock().unwrap();
|
||||
let path = path_map.get(new_url.as_str()).unwrap();
|
||||
|
||||
dom_url.clear();
|
||||
dom_url.push_str(relative_path);
|
||||
let new_url_str = url_helper::encode(path);
|
||||
|
||||
old_url_str.clear();
|
||||
old_url_str.push_str(&new_url_str);
|
||||
}
|
||||
|
||||
/// Find the charset of the webpage. ``data`` is not a String as this might not be utf8.
|
||||
/// Returned String is lower cased
|
||||
/// This is a hack and should be check in case of a bug
|
||||
fn find_charset(data: &[u8], http_charset: Option<String>) -> Option<String> {
|
||||
lazy_static! {
|
||||
static ref CHARSET_REGEX: Regex =
|
||||
Regex::new(r#"<meta.*charset\s*=\s*["']?([^"'\s;>]+).*>"#).unwrap();
|
||||
}
|
||||
|
||||
// We don't know the real charset yet. We hope that the charset is ASCII
|
||||
// compatible, because Rust String are in UTF-8 (also ASCII compatible).
|
||||
let data_utf8 = unsafe { String::from_utf8_unchecked(Vec::from(data)) };
|
||||
let captures = CHARSET_REGEX.captures_iter(&data_utf8).next();
|
||||
|
||||
// We use the first one, hopping we are in the <head> of the page... or if nothing is found
|
||||
// we used the http charset (if any).
|
||||
captures
|
||||
.map(|first| first.get(1).unwrap().as_str().to_lowercase())
|
||||
.or(http_charset)
|
||||
}
|
||||
|
||||
/// Proceed to convert the data in utf8.
|
||||
fn charset_convert(
|
||||
data: &[u8],
|
||||
charset_source: &'static Encoding,
|
||||
charset_dest: &'static Encoding,
|
||||
) -> Vec<u8> {
|
||||
let decode_result = charset_source.decode(data);
|
||||
let decode_bytes = decode_result.0.borrow();
|
||||
|
||||
let encode_result = charset_dest.encode(decode_bytes);
|
||||
|
||||
encode_result.0.into_owned()
|
||||
}
|
||||
|
||||
/// Check if the charset require conversion
|
||||
fn needs_charset_conversion(charset: &str) -> bool {
|
||||
!matches!(charset, "utf-8")
|
||||
}
|
||||
|
||||
/// Proces an html file: add new url to the chanel and prepare for offline navigation
|
||||
///Proces an html file: add new url to the chanel and prepare for offline navigation
|
||||
fn handle_html(
|
||||
scraper: &Scraper,
|
||||
transmitter: &Sender<(Url, i32, i32)>,
|
||||
transmitter: &Sender<(Url, i32)>,
|
||||
url: &Url,
|
||||
depth: i32,
|
||||
ext_depth: i32,
|
||||
data: &[u8],
|
||||
http_charset: Option<String>,
|
||||
data: &str,
|
||||
) -> Vec<u8> {
|
||||
let charset_source_str = match Self::find_charset(data, http_charset) {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
warn!("Charset not found for {}, defaulting to UTF-8", url);
|
||||
String::from("utf-8")
|
||||
}
|
||||
};
|
||||
|
||||
let need_charset_conversion = Self::needs_charset_conversion(&charset_source_str);
|
||||
|
||||
let charset_source = match encoding_rs::Encoding::for_label(charset_source_str.as_bytes()) {
|
||||
Some(encoder) => encoder,
|
||||
None => {
|
||||
warn!(
|
||||
"Charset {} not supported for {}, defaulting to UTF-8",
|
||||
charset_source_str, url
|
||||
);
|
||||
encoding_rs::UTF_8
|
||||
}
|
||||
};
|
||||
let charset_utf8 = encoding_rs::UTF_8;
|
||||
let utf8_data = if need_charset_conversion {
|
||||
Self::charset_convert(data, charset_source, charset_utf8)
|
||||
} else {
|
||||
Vec::from(data)
|
||||
};
|
||||
|
||||
let dom = dom::Dom::new(&String::from_utf8_lossy(&utf8_data));
|
||||
let source_path = match scraper.path_map.lock().unwrap().get(url.as_str()) {
|
||||
Some(path) => path.clone(),
|
||||
None => error!("Url {} was not found in the path map", url.as_str()),
|
||||
};
|
||||
let dom = dom::Dom::new(data);
|
||||
|
||||
dom.find_urls_as_strings()
|
||||
.into_iter()
|
||||
.filter(|candidate| Scraper::should_visit(scraper, candidate))
|
||||
.filter(|candidate| Scraper::should_visit(candidate, &url))
|
||||
.for_each(|next_url| {
|
||||
let url_to_parse = Scraper::normalize_url(next_url.clone());
|
||||
let next_full_url = url.join(&next_url).unwrap();
|
||||
let path = url_helper::to_path(&next_full_url);
|
||||
|
||||
let next_full_url = match url.join(url_to_parse.as_str()) {
|
||||
Ok(url) => url,
|
||||
Err(e) => {
|
||||
warn!("Failed to parse url: {} | Error: {}", next_url, e);
|
||||
return;
|
||||
},
|
||||
};
|
||||
|
||||
let path = url_helper::to_path(&next_full_url, true);
|
||||
let path_no_fragments = url_helper::to_path(&next_full_url, false);
|
||||
|
||||
// We only add urls without fragments to avoid duplication
|
||||
if scraper.map_url_path(&next_full_url, path_no_fragments.clone()) {
|
||||
if !Scraper::is_on_another_domain(next_url, url) {
|
||||
// If we are determining for a local domain
|
||||
if scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth {
|
||||
Scraper::push(transmitter, next_full_url, depth + 1, ext_depth);
|
||||
}
|
||||
} else {
|
||||
// If we are determining for an external domain
|
||||
if scraper.args.ext_depth == INFINITE_DEPTH
|
||||
|| ext_depth < scraper.args.ext_depth
|
||||
{
|
||||
Scraper::push(transmitter, next_full_url, depth, ext_depth + 1);
|
||||
}
|
||||
}
|
||||
if scraper.map_url_path(&next_full_url, path)
|
||||
&& (scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth)
|
||||
{
|
||||
Scraper::push(transmitter, next_full_url.clone(), depth + 1);
|
||||
}
|
||||
|
||||
scraper.fix_domtree(next_url, &source_path, &path);
|
||||
scraper.fix_domtree(next_url, &next_full_url);
|
||||
});
|
||||
|
||||
let utf8_data = dom.serialize().into_bytes();
|
||||
|
||||
if need_charset_conversion {
|
||||
Self::charset_convert(&utf8_data, charset_utf8, charset_source)
|
||||
} else {
|
||||
utf8_data
|
||||
}
|
||||
dom.serialize().into_bytes()
|
||||
}
|
||||
|
||||
/// Process a single URL
|
||||
fn handle_url(
|
||||
scraper: &Scraper,
|
||||
transmitter: &Sender<(Url, i32, i32)>,
|
||||
url: Url,
|
||||
depth: i32,
|
||||
ext_depth: i32,
|
||||
) {
|
||||
let download_filter_matches = !scraper.args.exclude_download.is_match(url.as_str())
|
||||
&& scraper.args.include_download.is_match(url.as_str());
|
||||
// download the page even if the download filter does not match,
|
||||
// so its links can be discovered and added to the queue
|
||||
fn handle_url(scraper: &Scraper, transmitter: &Sender<(Url, i32)>, url: Url, depth: i32) {
|
||||
match scraper.downloader.get(&url) {
|
||||
Ok(response) => {
|
||||
let data = match response.data {
|
||||
response::ResponseData::Html(data) => Scraper::handle_html(
|
||||
scraper,
|
||||
transmitter,
|
||||
&url,
|
||||
depth,
|
||||
ext_depth,
|
||||
&data,
|
||||
response.charset,
|
||||
),
|
||||
response::ResponseData::Html(data) => {
|
||||
Scraper::handle_html(scraper, transmitter, &url, depth, &data)
|
||||
}
|
||||
response::ResponseData::Other(data) => data,
|
||||
};
|
||||
|
||||
|
|
@ -262,7 +131,9 @@ impl Scraper {
|
|||
let path_map = scraper.path_map.lock().unwrap();
|
||||
let path = path_map.get(url.as_str()).unwrap();
|
||||
|
||||
if !scraper.args.dry_run && download_filter_matches {
|
||||
if !scraper.args.exclude.is_match(url.as_str())
|
||||
&& scraper.args.include.is_match(url.as_str())
|
||||
{
|
||||
match response.filename {
|
||||
Some(filename) => {
|
||||
disk::save_file(&filename, &data, &scraper.args.output);
|
||||
|
|
@ -276,10 +147,9 @@ impl Scraper {
|
|||
}
|
||||
}
|
||||
Err(e) => {
|
||||
println!("Couldn't download a page, {:?}", e);
|
||||
if !scraper.args.continue_on_error {
|
||||
error!("Couldn't download a page, {:?}", e);
|
||||
} else {
|
||||
warn!("Couldn't download a page, {:?}", e);
|
||||
process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -287,22 +157,15 @@ impl Scraper {
|
|||
scraper.visited_urls.lock().unwrap().insert(url.to_string());
|
||||
|
||||
if scraper.args.verbose {
|
||||
if download_filter_matches {
|
||||
info!("Downloaded: {}", url);
|
||||
} else {
|
||||
info!("Visited: {}", url);
|
||||
}
|
||||
info!("Visited: {}", url);
|
||||
}
|
||||
}
|
||||
|
||||
/// Run through the channel and complete it
|
||||
pub fn run(&mut self) {
|
||||
/* Push the origin URL and depth (0) through the channel */
|
||||
self.map_url_path(
|
||||
&self.args.origin,
|
||||
url_helper::to_path(&self.args.origin, false),
|
||||
);
|
||||
Scraper::push(&self.transmitter, self.args.origin.clone(), 0, 0);
|
||||
self.map_url_path(&self.args.origin, url_helper::to_path(&self.args.origin));
|
||||
Scraper::push(&self.transmitter, self.args.origin.clone(), 0);
|
||||
|
||||
thread::scope(|thread_scope| {
|
||||
for _ in 0..self.args.jobs {
|
||||
|
|
@ -324,9 +187,9 @@ impl Scraper {
|
|||
}
|
||||
TryRecvError::Disconnected => panic!("{}", e),
|
||||
},
|
||||
Ok((url, depth, ext_depth)) => {
|
||||
Ok((url, depth)) => {
|
||||
counter = 0;
|
||||
Scraper::handle_url(self_clone, &tx, url, depth, ext_depth);
|
||||
Scraper::handle_url(&self_clone, &tx, url, depth);
|
||||
self_clone.sleep(&mut rng);
|
||||
}
|
||||
}
|
||||
|
|
@ -347,22 +210,19 @@ impl Scraper {
|
|||
}
|
||||
|
||||
// delay_range+1 because gen_range is exclusive on the upper limit
|
||||
let rand_delay_secs = rng.gen_range(0..random_range + 1);
|
||||
let rand_delay_secs = rng.gen_range(0, random_range + 1);
|
||||
let delay_duration = time::Duration::from_secs(base_delay + rand_delay_secs);
|
||||
std::thread::sleep(delay_duration);
|
||||
}
|
||||
|
||||
/// If a URL should be visited (ignores `mail:`, `javascript:` and other pseudo-links)
|
||||
fn should_visit(scraper: &Scraper, url: &str) -> bool {
|
||||
if scraper.args.exclude_visit.is_match(url) || !scraper.args.include_visit.is_match(url) {
|
||||
return false;
|
||||
}
|
||||
/// If a URL should be visited, or does it belong to another domain
|
||||
fn should_visit(url: &str, base: &Url) -> bool {
|
||||
match Url::parse(url) {
|
||||
/* The given candidate is a valid URL, and not a relative path to
|
||||
* the next one. Therefore, we have to check if this URL is valid.
|
||||
* If it is, we should visit it.
|
||||
*/
|
||||
Ok(not_ok) => not_ok.has_host() && !not_ok.cannot_be_a_base(),
|
||||
* the next one. Therefore, we have to check if this URL belongs
|
||||
* to the same domain as our current URL. If the candidate has the
|
||||
* same domain as our base, then we should visit it */
|
||||
Ok(not_ok) => not_ok.domain() == base.domain(),
|
||||
|
||||
/* Since we couldn't parse this "URL", then it must be a relative
|
||||
* path or a malformed URL. If the URL is malformed, then it will
|
||||
|
|
@ -370,46 +230,13 @@ impl Scraper {
|
|||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces `///` with `//`
|
||||
/// And `//` with `https://`
|
||||
/// Without this function, if url is `///<domain>.<extension>/`, the app crashes.
|
||||
fn normalize_url(url: String) -> String {
|
||||
if url.starts_with("///") {
|
||||
return url.replacen("///", "https://", 1);
|
||||
} else if url.starts_with("//") {
|
||||
return url.replacen("//", "https://", 1);
|
||||
}
|
||||
url
|
||||
}
|
||||
|
||||
/// If the URL leads to another domain
|
||||
fn is_on_another_domain(url: &str, base: &Url) -> bool {
|
||||
let real_url = Scraper::normalize_url(String::from(url));
|
||||
|
||||
match Url::parse(real_url.as_str()) {
|
||||
/* The given candidate is a valid URL, and not a relative path to
|
||||
* the next one. Therefore, we have to check if this URL belongs
|
||||
* to the same domain as our current URL. If the candidate has the
|
||||
* same domain as our base, and the depth condition is satisfied,
|
||||
* then we should visit it, */
|
||||
Ok(not_ok) => not_ok.domain() != base.domain(),
|
||||
|
||||
/* Since we couldn't parse this "URL", then it must be a relative
|
||||
* path or a malformed URL. If the URL is malformed, then it will
|
||||
* be handled during the join() call in run() */
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::PathBuf;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
use super::*;
|
||||
use regex::Regex;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[test]
|
||||
fn test_zero_delay_range() {
|
||||
|
|
@ -419,21 +246,13 @@ mod tests {
|
|||
jobs: 1,
|
||||
tries: 1,
|
||||
depth: 5,
|
||||
ext_depth: 0,
|
||||
delay: 0,
|
||||
user_agent: "suckit".to_string(),
|
||||
random_range: 0,
|
||||
verbose: true,
|
||||
include_visit: Regex::new(".*").unwrap(),
|
||||
exclude_visit: Regex::new("^$").unwrap(),
|
||||
include_download: Regex::new("jpg").unwrap(),
|
||||
exclude_download: Regex::new("png").unwrap(),
|
||||
visit_filter_is_download_filter: false,
|
||||
auth: Vec::new(),
|
||||
include: Regex::new("jpg").unwrap(),
|
||||
exclude: Regex::new("png").unwrap(),
|
||||
continue_on_error: true,
|
||||
dry_run: false,
|
||||
disable_certs_checks: false,
|
||||
cookie: "".to_string(),
|
||||
};
|
||||
|
||||
let _ = Scraper::new(args);
|
||||
|
|
@ -447,47 +266,15 @@ mod tests {
|
|||
jobs: 1,
|
||||
tries: 1,
|
||||
depth: 5,
|
||||
ext_depth: 0,
|
||||
delay: 2,
|
||||
user_agent: "suckit".to_string(),
|
||||
random_range: 5,
|
||||
verbose: true,
|
||||
include_visit: Regex::new(".*").unwrap(),
|
||||
exclude_visit: Regex::new("^$").unwrap(),
|
||||
include_download: Regex::new("jpg").unwrap(),
|
||||
exclude_download: Regex::new("png").unwrap(),
|
||||
visit_filter_is_download_filter: false,
|
||||
auth: Vec::new(),
|
||||
include: Regex::new("jpg").unwrap(),
|
||||
exclude: Regex::new("png").unwrap(),
|
||||
continue_on_error: true,
|
||||
dry_run: false,
|
||||
disable_certs_checks: false,
|
||||
cookie: "".to_string(),
|
||||
};
|
||||
|
||||
let _ = Scraper::new(args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_charset_parsing_double_quotes() {
|
||||
assert_eq!(
|
||||
Scraper::find_charset(b"<meta charset=\"UTF-8\">", None),
|
||||
Some("utf-8".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
Scraper::find_charset(b"<meta charset=\"windows-1252\">", None),
|
||||
Some("windows-1252".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_charset_parsing_single_quotes() {
|
||||
assert_eq!(
|
||||
Scraper::find_charset(b"<meta charset=\'UTF-8\'>", None),
|
||||
Some("utf-8".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
Scraper::find_charset(b"<meta charset=\'windows-1252\'>", None),
|
||||
Some("windows-1252".to_string())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,141 +1,44 @@
|
|||
use std::path::Path;
|
||||
|
||||
use md5;
|
||||
use std::borrow::Cow;
|
||||
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
|
||||
use url::Url;
|
||||
|
||||
///Max file name size supported by the file system
|
||||
const FILE_NAME_MAX_LENGTH: usize = 255;
|
||||
///Characters that need to be replaced by encode()
|
||||
const FRAGMENT: &AsciiSet = &CONTROLS.add(b'?');
|
||||
|
||||
/// Convert an Url to the corresponding path
|
||||
pub fn to_path(url: &Url, with_fragment: bool) -> String {
|
||||
let url_domain = url.host_str().unwrap();
|
||||
///Encode special character with '%' representation
|
||||
pub fn encode(path: &str) -> String {
|
||||
utf8_percent_encode(path, FRAGMENT).to_string()
|
||||
}
|
||||
|
||||
let mut url_path_and_query = url.path().to_string();
|
||||
if let Some(query) = url.query() {
|
||||
url_path_and_query.push_str("__querystring__");
|
||||
url_path_and_query.push_str(query);
|
||||
///Convert an Url to the corresponding path
|
||||
pub fn to_path(url: &Url) -> String {
|
||||
let url = url.as_str().split("://").collect::<Vec<&str>>()[1];
|
||||
|
||||
let mut url = url.replace('/', "_").replace('.', "_");
|
||||
if url.len() >= FILE_NAME_MAX_LENGTH {
|
||||
url = url.split_at(FILE_NAME_MAX_LENGTH).0.to_string(); //Shrink too long file name
|
||||
}
|
||||
let url = url.trim_end_matches('_'); //Remaining '/'
|
||||
|
||||
let path = Path::new(&url_path_and_query);
|
||||
let mut filename = path.file_name().map_or(String::from(""), |filename| {
|
||||
filename.to_str().unwrap().to_string()
|
||||
});
|
||||
let mut parent = path
|
||||
.parent()
|
||||
.map_or("", |filename| filename.to_str().unwrap())
|
||||
.to_string();
|
||||
|
||||
// Ensure the folder names are not too long
|
||||
parent = parent
|
||||
.split('/')
|
||||
.map(|str| {
|
||||
if str.len() > FILE_NAME_MAX_LENGTH {
|
||||
Cow::Owned(format!("{:x}", md5::compute(str)))
|
||||
} else {
|
||||
Cow::Borrowed(str)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<Cow<str>>>()
|
||||
.join("/");
|
||||
|
||||
if url_path_and_query.ends_with('/') {
|
||||
filename = "index.html".to_string();
|
||||
parent = url_path_and_query.trim_end_matches('/').to_string();
|
||||
} else if Path::new(&filename).extension().is_none() {
|
||||
parent = url_path_and_query.trim_end_matches('/').to_string();
|
||||
filename = "index_no_slash.html".to_string();
|
||||
}
|
||||
|
||||
if filename.len() > FILE_NAME_MAX_LENGTH {
|
||||
let digest = md5::compute(filename);
|
||||
filename = format!("{:x}.html", digest);
|
||||
}
|
||||
|
||||
match (url.fragment(), with_fragment) {
|
||||
(Some(fragment), true) => format!("{}{}/{}#{}", url_domain, parent, filename, fragment),
|
||||
(_, _) => format!("{}{}/{}", url_domain, parent, filename),
|
||||
}
|
||||
url.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn url_to_path_domain_only() {
|
||||
let str = super::to_path(&Url::parse("https://lwn.net/").unwrap(), false);
|
||||
|
||||
assert_eq!(str, "lwn.net/index.html");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path_domain_only_no_slash() {
|
||||
let str = super::to_path(&Url::parse("https://lwn.net").unwrap(), false);
|
||||
|
||||
assert_eq!(str, "lwn.net/index.html");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path() {
|
||||
let str = super::to_path(
|
||||
&Url::parse("https://lwn.net/Kernel/index.html").unwrap(),
|
||||
false,
|
||||
);
|
||||
let str = super::to_path(&Url::parse("https://lwn.net/Kernel/").unwrap());
|
||||
|
||||
assert_eq!(str, "lwn.net/Kernel/index.html");
|
||||
assert_eq!(str, "lwn_net_Kernel");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path_index() {
|
||||
let str = super::to_path(&Url::parse("https://lwn.net/Kernel/").unwrap(), false);
|
||||
fn url_to_path_long() {
|
||||
let str = super::to_path(&Url::parse("https://e8v0pez1lofdxoxgg5vwrnaqkjuvpowp9wtgc2eknlfpjdwmmfti8fcwyjzfdgys3nrgyqyeqjkulpyg9kfiqajza2bwxkinhhpohyrnnoy2bak374tcaxh1ycpboolmx8so9yq9kbcj5wu5cgymqndeqasdak0nvl0ijka6fkkmhhvt43l73bn38rewicd4h1ff2omhpni752jtqyzsjub5coh8dlnr3i35udmkzhxo4db3is9gnqmf3hl.comtest").unwrap());
|
||||
|
||||
assert_eq!(str, "lwn.net/Kernel/index.html");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path_index_no_slash() {
|
||||
let str = super::to_path(&Url::parse("https://lwn.net/Kernel").unwrap(), false);
|
||||
|
||||
assert_eq!(str, "lwn.net/Kernel/index_no_slash.html");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path_fragment() {
|
||||
let str = super::to_path(
|
||||
&Url::parse("https://lwn.net/Kernel/#fragment").unwrap(),
|
||||
true,
|
||||
);
|
||||
|
||||
assert_eq!(str, "lwn.net/Kernel/index.html#fragment");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path_no_fragment() {
|
||||
let str = super::to_path(
|
||||
&Url::parse("https://lwn.net/Kernel/#fragment").unwrap(),
|
||||
false,
|
||||
);
|
||||
|
||||
assert_eq!(str, "lwn.net/Kernel/index.html");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path_to_long_md5() {
|
||||
let str = super::to_path(&Url::parse("https://lwn.net/Kernel/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.html").unwrap(), false);
|
||||
|
||||
assert_eq!(str, "lwn.net/Kernel/5ca82767de71fe8930587e82bb994903.html");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_to_path_querystrings() {
|
||||
let str = super::to_path(
|
||||
&Url::parse(
|
||||
"https://google.com/foobar/platform-redirect/?next=/configuration/releases/",
|
||||
)
|
||||
.unwrap(),
|
||||
false,
|
||||
);
|
||||
assert_eq!(str, "google.com/foobar/platform-redirect/__querystring__next=/configuration/releases/index.html");
|
||||
assert_eq!(str, "e8v0pez1lofdxoxgg5vwrnaqkjuvpowp9wtgc2eknlfpjdwmmfti8fcwyjzfdgys3nrgyqyeqjkulpyg9kfiqajza2bwxkinhhpohyrnnoy2bak374tcaxh1ycpboolmx8so9yq9kbcj5wu5cgymqndeqasdak0nvl0ijka6fkkmhhvt43l73bn38rewicd4h1ff2omhpni752jtqyzsjub5coh8dlnr3i35udmkzhxo4db3is9gnqmf3hl_com");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 265 KiB After Width: | Height: | Size: 265 KiB |
|
|
@ -1,64 +0,0 @@
|
|||
//! Tests for using --auth flags for suckit
|
||||
|
||||
mod fixtures;
|
||||
|
||||
use std::fs::read_dir;
|
||||
use std::process::Command;
|
||||
use std::process::Stdio;
|
||||
|
||||
const PAGE: &'static str = "tests/fixtures/";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
|
||||
// Shouldn't supply credentials to a non-matching host
|
||||
#[test]
|
||||
fn auth_different_host() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, true, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[
|
||||
&url,
|
||||
"-o",
|
||||
output_dir,
|
||||
"-a",
|
||||
"username password example.com",
|
||||
])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let paths = read_dir(format!("{}/{}", output_dir, IP)).unwrap();
|
||||
|
||||
// Only the initial invalid response file should be present
|
||||
assert_eq!(paths.count(), 1);
|
||||
}
|
||||
|
||||
// Should authenticate with credentials to host (defaulting to origin host)
|
||||
#[test]
|
||||
fn auth_valid() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir, "-a", "username password"])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
|
||||
let paths = read_dir(format!("{}/{}", output_dir, IP)).unwrap();
|
||||
// Should load multiple paths, not just the invalid auth response
|
||||
assert!(paths.count() > 1);
|
||||
}
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
//! Test for charset detection/conversion
|
||||
|
||||
mod fixtures;
|
||||
|
||||
use std::fs;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::sync::Once;
|
||||
|
||||
const PAGE: &'static str = "tests/fixtures";
|
||||
const PAGE_META: &'static str = "tests/fixtures/charset_test_html.html";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
|
||||
#[test]
|
||||
fn test_html_charset_found() {
|
||||
// Spawn a single instance of a local http server usable by all tests in this module.
|
||||
let addr = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
let file_dir = format!("{}/{}", output_dir, IP);
|
||||
let url = format!("http://{}/charset_test_html.html", addr);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let file_path = fs::read_dir(file_dir)
|
||||
.unwrap()
|
||||
.next()
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.path(); // There is only one file in the directory
|
||||
|
||||
let data_source = fs::read(PAGE_META).unwrap();
|
||||
let data_downloaded = fs::read(file_path).unwrap();
|
||||
|
||||
assert!(fixtures::do_vecs_match(&data_source, &data_downloaded));
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
//! Test for charset detection/conversion
|
||||
|
||||
mod fixtures;
|
||||
|
||||
use std::fs;
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
const PAGE: &'static str = "tests/fixtures/";
|
||||
const PAGE_NO_META: &'static str = "tests/fixtures/charset_test_html_no_meta.html";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
|
||||
#[test]
|
||||
fn test_html_charset_not_found() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
// Spawn a single instance of a local http server usable by all tests in this module.
|
||||
let file_dir = format!("{}/{}", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
|
||||
let file_path = fs::read_dir(file_dir)
|
||||
.unwrap()
|
||||
.next()
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.path(); // There is only one file in the directory
|
||||
|
||||
let data_source = fs::read(PAGE_NO_META).unwrap();
|
||||
let data_downloaded = fs::read(file_path).unwrap();
|
||||
|
||||
assert!(!fixtures::do_vecs_match(&data_source, &data_downloaded));
|
||||
}
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
//! Test for charset detection/conversion
|
||||
|
||||
mod fixtures;
|
||||
|
||||
use std::fs;
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
|
||||
const PAGE: &'static str = "tests/fixtures/";
|
||||
const PAGE_NO_META: &'static str = "tests/fixtures/charset_test_html_no_meta.html";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
|
||||
lazy_static! {
|
||||
static ref CHARSET_HEADER: Vec<(&'static str, &'static str)> =
|
||||
vec![("Content-Type", "charset=windows-1252")];
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_charset_found() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, Some(&CHARSET_HEADER));
|
||||
let url = format!("http://{}/charset_test_html_no_meta.html", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let file_dir = format!("{}/{}", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
|
||||
let file_path = fs::read_dir(file_dir)
|
||||
.unwrap()
|
||||
.next()
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.path(); // There is only one file in the directory
|
||||
|
||||
let data_source = fs::read(PAGE_NO_META).unwrap();
|
||||
let data_downloaded = fs::read(file_path).unwrap();
|
||||
|
||||
assert!(fixtures::do_vecs_match(&data_source, &data_downloaded));
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
//! Test for charset detection/conversion
|
||||
|
||||
mod fixtures;
|
||||
|
||||
use std::fs;
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
const PAGE: &'static str = "tests/fixtures/";
|
||||
const PAGE_NO_META: &'static str = "tests/fixtures/charset_test_html_no_meta.html";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
|
||||
#[test]
|
||||
fn test_http_charset_found() {
|
||||
// Spawn a single instance of a local http server usable by all tests in this module.
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let file_dir = format!("{}/{}", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
|
||||
let file_path = fs::read_dir(file_dir)
|
||||
.unwrap()
|
||||
.next()
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.path(); // There is only one file in the directory
|
||||
|
||||
let data_source = fs::read(PAGE_NO_META).unwrap();
|
||||
let data_downloaded = fs::read(file_path).unwrap();
|
||||
|
||||
assert!(!fixtures::do_vecs_match(&data_source, &data_downloaded));
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
//! Tests for using --ext-depth
|
||||
|
||||
mod fixtures;
|
||||
|
||||
use fixtures::get_file_count_with_pattern;
|
||||
use std::fs::read_dir;
|
||||
use std::process::Command;
|
||||
use std::process::Stdio;
|
||||
use std::sync::Once;
|
||||
|
||||
const PAGE: &'static str = "tests/fixtures/";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
|
||||
// Test to use include flag for downloading pages only matching the given pattern.
|
||||
#[test]
|
||||
fn with_external() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let local = format!("{}/{}/", output_dir, IP);
|
||||
let external = format!("{}/{}/", output_dir, "google.com");
|
||||
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir, "-d", "0", "--ext-depth", "1"])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let path_local = read_dir(&local).unwrap();
|
||||
let path_external = read_dir(&external).unwrap();
|
||||
|
||||
assert_eq!(path_local.count() + path_external.count(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_external() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let external = format!("{}/{}/", output_dir, "google.com");
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir, "-d", "0", "--ext-depth", "0"])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let path_external = read_dir(&external);
|
||||
|
||||
assert!(path_external.is_err());
|
||||
}
|
||||
189
tests/filters.rs
189
tests/filters.rs
|
|
@ -8,133 +8,28 @@ use std::process::Command;
|
|||
use std::process::Stdio;
|
||||
use std::sync::Once;
|
||||
|
||||
const PAGE: &'static str = "tests/fixtures/";
|
||||
const IP: &'static str = "0.0.0.0";
|
||||
const SUCKIT: &'static str = "target/debug/suckit";
|
||||
const ADDR: &'static str = "http://0.0.0.0:8000";
|
||||
static START: Once = Once::new();
|
||||
|
||||
#[test]
|
||||
fn visit_filter_is_download_filter() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
fn test_include_exclude() {
|
||||
// Spawn a single instance of a local http server usable by all tests in this module.
|
||||
START.call_once(|| {
|
||||
fixtures::spawn_local_http_server();
|
||||
});
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let files_dir = format!("{}/{}/", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[
|
||||
&url,
|
||||
"-o",
|
||||
output_dir,
|
||||
"-v",
|
||||
"-e",
|
||||
"no_download_no_visit.html",
|
||||
"--visit-filter-is-download-filter",
|
||||
])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let result = cmd.wait_with_output().unwrap();
|
||||
let stdout_str = unsafe { String::from_utf8_unchecked(result.stdout) };
|
||||
assert!(result.status.success());
|
||||
|
||||
let paths = read_dir(&files_dir).unwrap();
|
||||
|
||||
assert!(!stdout_str.contains("should_not_get_visited.html"));
|
||||
}
|
||||
|
||||
// Test to use include flag for visiting pages only matching the given pattern.
|
||||
#[test]
|
||||
fn visit_include_filter() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let files_dir = format!("{}/{}/", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir, "--include-visit", "mp[3-4]"])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let paths = read_dir(&files_dir).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
paths.count() - 1, // minus one because of index.html which is downloaded unconditionally
|
||||
get_file_count_with_pattern(".mp3", &files_dir).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
// Test demonstrating usage of multiple include patterns for visiting pages only matching the given pattern.
|
||||
#[test]
|
||||
fn visit_include_multiple_filters() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let files_dir = format!("{}/{}/", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir, "--include-visit", "(mp[3-4])|(txt)"])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let paths = read_dir(&files_dir).unwrap();
|
||||
let mp3_count = get_file_count_with_pattern(".mp3", &files_dir).unwrap();
|
||||
let txt_count = get_file_count_with_pattern(".txt", &files_dir).unwrap();
|
||||
assert_eq!(
|
||||
paths.count() - 1, // minus one because of index.html which is downloaded unconditionally
|
||||
mp3_count + txt_count
|
||||
);
|
||||
}
|
||||
|
||||
// Test to use exclude flag for excluding pages matching the given pattern.
|
||||
#[test]
|
||||
fn visit_exclude_filter() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let files_dir = format!("{}/{}/", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir, "--exclude-visit", "jpe?g"])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
|
||||
let jpeg_count = get_file_count_with_pattern(".jpe?g", &files_dir).unwrap();
|
||||
assert_eq!(jpeg_count, 0);
|
||||
// Tests below are grouped together as they depend on the local_http_server above.
|
||||
include_filter();
|
||||
include_multiple_filters();
|
||||
exclude_filter();
|
||||
}
|
||||
|
||||
// Test to use include flag for downloading pages only matching the given pattern.
|
||||
#[test]
|
||||
fn download_include_filter() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let files_dir = format!("{}/{}/", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir, "-i", "mp[3-4]"])
|
||||
fn include_filter() {
|
||||
let output_dir = "w2";
|
||||
let mut cmd = Command::new(SUCKIT)
|
||||
.args(&[ADDR, "-o", "w2", "-i", "mp[3-4]", "-j", "16"])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
|
|
@ -142,53 +37,39 @@ fn download_include_filter() {
|
|||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let paths = read_dir(&files_dir).unwrap();
|
||||
|
||||
let paths = read_dir(output_dir).unwrap();
|
||||
assert_eq!(
|
||||
paths.count(),
|
||||
get_file_count_with_pattern(".mp3", &files_dir).unwrap()
|
||||
get_file_count_with_pattern("*_mp3", output_dir).unwrap()
|
||||
);
|
||||
|
||||
std::fs::remove_dir_all(output_dir).unwrap();
|
||||
}
|
||||
|
||||
// Test demonstrating usage of multiple include patterns for downloading pages only matching the given pattern.
|
||||
#[test]
|
||||
fn download_include_multiple_filters() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let files_dir = format!("{}/{}/", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir, "-i", "(mp[3-4])|(txt)"])
|
||||
fn include_multiple_filters() {
|
||||
let output_dir = "w1";
|
||||
let mut cmd = Command::new(SUCKIT)
|
||||
.args(&[ADDR, "-o", output_dir, "-i", "(mp[3-4])|(txt)", "-j", "16"])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
|
||||
let paths = read_dir(&files_dir).unwrap();
|
||||
let mp3_count = get_file_count_with_pattern(".mp3", &files_dir).unwrap();
|
||||
let txt_count = get_file_count_with_pattern(".txt", &files_dir).unwrap();
|
||||
let paths = read_dir(output_dir).unwrap();
|
||||
let mp3_count = get_file_count_with_pattern("*_mp3", output_dir).unwrap();
|
||||
let txt_count = get_file_count_with_pattern("*_txt", output_dir).unwrap();
|
||||
assert_eq!(paths.count(), mp3_count + txt_count);
|
||||
|
||||
std::fs::remove_dir_all(output_dir).unwrap();
|
||||
}
|
||||
|
||||
// Test to use exclude flag for excluding pages matching the given pattern.
|
||||
#[test]
|
||||
fn download_exclude_filter() {
|
||||
let ip = fixtures::spawn_local_http_server(PAGE, false, None);
|
||||
let url = format!("http://{}", ip);
|
||||
|
||||
let tempdir = mktemp::Temp::new_dir().unwrap();
|
||||
let output_dir = tempdir.to_str().unwrap();
|
||||
|
||||
let files_dir = format!("{}/{}/", output_dir, IP);
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
|
||||
.args(&[&url, "-o", output_dir, "-e", "jpe?g"])
|
||||
fn exclude_filter() {
|
||||
let output_dir = "w3";
|
||||
let mut cmd = Command::new(SUCKIT)
|
||||
.args(&[ADDR, "-o", output_dir, "-e", "jpe?g", "-j", "16"])
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
|
|
@ -196,7 +77,11 @@ fn download_exclude_filter() {
|
|||
|
||||
let status = cmd.wait().unwrap();
|
||||
assert!(status.success());
|
||||
let paths = read_dir(&files_dir).unwrap();
|
||||
let jpeg_count = get_file_count_with_pattern(".jpe?g", &files_dir).unwrap();
|
||||
assert_eq!(jpeg_count, 0);
|
||||
let paths = read_dir(output_dir).unwrap();
|
||||
let mp3_count = get_file_count_with_pattern("*_mp3", output_dir).unwrap();
|
||||
let txt_count = get_file_count_with_pattern("*_txt", output_dir).unwrap();
|
||||
let index_file = 1;
|
||||
assert_eq!(paths.count(), mp3_count + txt_count + index_file);
|
||||
|
||||
std::fs::remove_dir_all(output_dir).unwrap();
|
||||
}
|
||||
|
|
|
|||
13
tests/fixtures/charset_test_html.html
vendored
13
tests/fixtures/charset_test_html.html
vendored
|
|
@ -1,13 +0,0 @@
|
|||
<!-- This file is encoded in 'windows-1252' charset with no newline at the end.
|
||||
Be careful when modifying it. To save it correctly on:
|
||||
- vim: :write ++enc=windows-1252
|
||||
This file should not have a newline. Don't forget to run ``truncate -s -1``
|
||||
in case there is one.
|
||||
--><html><head>
|
||||
<meta content="text/html; charset=windows-1252" http-equiv="Content-Type">
|
||||
<title>Gamle Gjerpen - Ei Bygdebok for nett.</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<p>Gamle Valebø Redigering av sidene pågår.</p>
|
||||
</body></html>
|
||||
12
tests/fixtures/charset_test_html_no_meta.html
vendored
12
tests/fixtures/charset_test_html_no_meta.html
vendored
|
|
@ -1,12 +0,0 @@
|
|||
<!-- This file is encoded in 'windows-1252' charset with no newline at the end.
|
||||
Be careful when modifying it. To save it correctly on:
|
||||
- vim: :write ++enc=windows-1252
|
||||
This file should not have a newline. Don't forget to run ``truncate -s -1``
|
||||
in case there is one.
|
||||
--><html><head>
|
||||
<title>Gamle Gjerpen - Ei Bygdebok for nett.</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<p>Gamle Valebø Redigering av sidene pågår.</p>
|
||||
</body></html>
|
||||
2
tests/fixtures/index.html
vendored
2
tests/fixtures/index.html
vendored
|
|
@ -9,7 +9,5 @@
|
|||
<a href="jpeg.jpg" download>JPG File</a>
|
||||
<a href="mp3.mp3" download="">MP3 File</a>
|
||||
<a href="file.txt" download="">Text File</a>
|
||||
<a href="https://google.com" download="">Google</a>
|
||||
<a href="no_download_no_visit.html">No download</a>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
117
tests/fixtures/mod.rs
vendored
117
tests/fixtures/mod.rs
vendored
|
|
@ -1,95 +1,56 @@
|
|||
use std::fs::File;
|
||||
use std::process::Command;
|
||||
use std::process::Stdio;
|
||||
use std::thread;
|
||||
use tiny_http::{Response, Server};
|
||||
|
||||
use portpicker;
|
||||
use subprocess::Exec;
|
||||
use tiny_http::{Header, Response, Server};
|
||||
const PAGE: &'static str = "tests/fixtures/index.html";
|
||||
|
||||
const AUTH_HEADER: &str = "Authorization";
|
||||
const AUTH_CREDENTIALS: &str = "Basic dXNlcm5hbWU6cGFzc3dvcmQ="; // base64-encoded "username:password"
|
||||
|
||||
pub fn spawn_local_http_server(
|
||||
page: &'static str,
|
||||
requires_auth: bool,
|
||||
headers: Option<&'static Vec<(&'static str, &'static str)>>,
|
||||
) -> String {
|
||||
let port = portpicker::pick_unused_port().unwrap();
|
||||
let addr = format!("0.0.0.0:{}", port);
|
||||
let server = Server::http(&addr).unwrap();
|
||||
pub fn spawn_local_http_server() {
|
||||
let server = Server::http("0.0.0.0:8000").unwrap();
|
||||
println!("Spawning http server");
|
||||
thread::spawn(move || {
|
||||
for request in server.incoming_requests() {
|
||||
// Authenticate request from headers if provided
|
||||
let auth_header = request
|
||||
.headers()
|
||||
.iter()
|
||||
.find(|h| h.field.equiv(AUTH_HEADER));
|
||||
let valid_auth = check_auth_credentials(auth_header);
|
||||
|
||||
let mut response = if requires_auth && !valid_auth {
|
||||
let mut response = Response::from_string("Invalid auth").with_status_code(401);
|
||||
let h = Header::from_bytes("WWW-Authenticate", r#"Basic realm="Test""#).unwrap();
|
||||
response.add_header(h);
|
||||
response.boxed()
|
||||
} else {
|
||||
let file = match request.url() {
|
||||
"/" => format!("{}{}", page, "index.html"),
|
||||
other => format!("{}{}", page, other),
|
||||
};
|
||||
// panic!("File = {}", file);
|
||||
Response::from_file(File::open(file).unwrap()).boxed()
|
||||
};
|
||||
|
||||
match headers {
|
||||
Some(vec) => {
|
||||
let mut key_vec: Vec<u8> = vec![];
|
||||
let mut value_vec: Vec<u8> = vec![];
|
||||
for (key, value) in vec {
|
||||
key_vec.extend_from_slice(key.as_bytes());
|
||||
value_vec.extend_from_slice(value.as_bytes());
|
||||
}
|
||||
|
||||
let h = Header::from_bytes(key_vec, value_vec).unwrap();
|
||||
response.add_header(h);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
let response = Response::from_file(File::open(PAGE).unwrap());
|
||||
request.respond(response).unwrap();
|
||||
}
|
||||
});
|
||||
|
||||
return addr;
|
||||
}
|
||||
|
||||
fn check_auth_credentials(auth_header: Option<&Header>) -> bool {
|
||||
match auth_header {
|
||||
None => false,
|
||||
Some(header) => header.value.as_str() == AUTH_CREDENTIALS,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_file_count_with_pattern(pattern: &str, dir: &str) -> Result<usize, ()> {
|
||||
// Command being run: `ls | grep pattern | wc -w`
|
||||
// Command being run: `ls | grep .mp3 | wc -w`
|
||||
let mut du_output_child = Command::new("ls")
|
||||
.args(&[dir])
|
||||
.stdout(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let cmd = {
|
||||
// Pipe ('|') is overloaded here and does a real pipe
|
||||
Exec::shell(format!("ls {}", dir))
|
||||
| Exec::shell(format!("grep '{}'", pattern))
|
||||
| Exec::shell("wc -l")
|
||||
}
|
||||
.capture();
|
||||
if let Some(du_output) = du_output_child.stdout.take() {
|
||||
let mut sort_output_child = Command::new("egrep")
|
||||
.arg(pattern)
|
||||
.stdin(du_output)
|
||||
.stdout(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
match cmd {
|
||||
Ok(capture_data) => {
|
||||
let stdout = capture_data.stdout_str();
|
||||
let count = stdout.trim().parse::<usize>().unwrap();
|
||||
Ok(count)
|
||||
du_output_child.wait().unwrap();
|
||||
|
||||
if let Some(sort_output) = sort_output_child.stdout.take() {
|
||||
let head_output_child = Command::new("wc")
|
||||
.args(&["-w"])
|
||||
.stdin(sort_output)
|
||||
.stdout(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let head_stdout = head_output_child.wait_with_output().unwrap();
|
||||
sort_output_child.wait().unwrap();
|
||||
return Ok(String::from_utf8(head_stdout.stdout)
|
||||
.unwrap()
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap());
|
||||
}
|
||||
_ => Err(()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn do_vecs_match<T: PartialEq>(a: &Vec<T>, b: &Vec<T>) -> bool {
|
||||
let matching = a.iter().zip(b.iter()).filter(|&(a, b)| a == b).count();
|
||||
matching == a.len() && matching == b.len()
|
||||
Err(())
|
||||
}
|
||||
|
|
|
|||
8
tests/fixtures/no_download_no_visit.html
vendored
8
tests/fixtures/no_download_no_visit.html
vendored
|
|
@ -1,8 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head></head>
|
||||
|
||||
<body>
|
||||
<a href="should_not_get_visited.html">Link</a>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
# Create the local's server directory if necessary
|
||||
if [[ ! -d local_server ]]; then
|
||||
mkdir local_server
|
||||
fi
|
||||
|
||||
cd local_server
|
||||
|
||||
# Clone repository if necessary
|
||||
if [[ ! -d linux ]]; then
|
||||
# Clone a big repository
|
||||
git clone https://github.com/torvalds/linux
|
||||
|
||||
cd linux
|
||||
|
||||
# Checkout the 5.9 release
|
||||
git checkout v5.9
|
||||
|
||||
# No need for the .git directory. It creates a ton of files that take too long
|
||||
# to scrape
|
||||
rm -rf .git
|
||||
|
||||
# Get back to local_server
|
||||
cd ..
|
||||
fi
|
||||
|
||||
printf "\nWEBSERVER UP\n"
|
||||
|
||||
# Start up the local python server
|
||||
python3 -m http.server
|
||||
|
|
@ -1 +1,16 @@
|
|||
1, 1, 1
|
||||
92.01026630401611, 45.85903334617615, 24.938209056854248
|
||||
92.68794660568237, 45.51665663719177, 36.877896785736084
|
||||
94.26381759643554, 48.01623530387879, 39.034280157089235
|
||||
94.51657667160035, 47.43879041671753, 25.296046495437622
|
||||
94.91624474525452, 47.318924236297605, 38.720614719390866
|
||||
94.61654992103577, 57.59067335128784, 24.995196104049683
|
||||
95.28456358909607, 46.86330976486206, 24.708560609817503
|
||||
77.68792352676391, 39.08626494407654, 19.765260219573975
|
||||
67.09948434829712, 34.11661868095398, 17.574066400527954
|
||||
68.2605634689331, 33.728125953674315, 17.433270835876463
|
||||
70.19220662117004, 41.964057207107544, 17.660782480239867
|
||||
68.06500535011291, 34.10616636276245, 28.643333148956298
|
||||
67.92092838287354, 33.87600998878479, 17.644397974014282
|
||||
67.94186737537385, 33.967066860198976, 17.586638736724854
|
||||
68.20248901844025, 37.70992383956909, 22.916122150421142
|
||||
|
|
|
|||
|
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
# Number of tests for each bench-set
|
||||
TEST_RETRIES = 20
|
||||
TEST_RETRIES = 10
|
||||
|
||||
# File to store the results
|
||||
FILENAME = "speed.csv"
|
||||
|
|
@ -9,8 +9,8 @@ FILENAME = "speed.csv"
|
|||
# Path to the suckit binary
|
||||
SUCKIT = "suckit"
|
||||
|
||||
# URL to download: localhost
|
||||
URL = "http://0.0.0.0:8000"
|
||||
# URL to download
|
||||
URL = "http://books.toscrape.com"
|
||||
|
||||
# Path to store the downloaded data
|
||||
PATH = "/tmp/suckit_speed"
|
||||
|
|
@ -23,18 +23,6 @@ import subprocess
|
|||
import time
|
||||
from termcolor import colored
|
||||
|
||||
def start_webserver():
|
||||
print("Launching webserver")
|
||||
|
||||
webserver_pid = subprocess.Popen(["./local_server_setup.sh"], stdout = subprocess.PIPE)
|
||||
|
||||
while webserver_pid.stdout.readline() != b"WEBSERVER UP\n":
|
||||
print("Waiting on webserver...", end = "\r")
|
||||
|
||||
print("Webserver launched")
|
||||
|
||||
return webserver_pid
|
||||
|
||||
def parse_args():
|
||||
global FILENAME
|
||||
global SUCKIT
|
||||
|
|
@ -132,14 +120,4 @@ def main():
|
|||
shutil.rmtree(PATH)
|
||||
|
||||
if __name__ == "__main__":
|
||||
webserver_pid = None
|
||||
|
||||
try:
|
||||
webserver_pid = start_webserver()
|
||||
|
||||
main()
|
||||
|
||||
# Terminate the webserver
|
||||
webserver_pid.kill()
|
||||
except: # Kill the webserver if an exception occurs
|
||||
webserver_pid.kill()
|
||||
main()
|
||||
|
|
|
|||
Loading…
Reference in a new issue