fix: limit folder names length to avoid crashes

fix: avoid panic when encountering malformed URLs
build(deps): bump mio from 0.8.10 to 0.8.11
2026-09-10 15:26:24 -04:00 · 2026-03-12 11:18:03 +01:00 · 2026-03-12 11:18:03 +01:00 · 2024-03-05 09:38:09 +01:00 · 2024-01-24 16:48:59 +01:00 · 2024-01-24 16:48:59 +01:00
40 changed files with 2933 additions and 875 deletions
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@ -0,0 +1,20 @@
+version: 2
+updates:
+  # ATM this only create pr to the lockfile
+  # - package-ecosystem: "cargo"
+  #   directory: "/"
+  #   schedule:
+  #     interval: "daily"
+  #     time: "02:00" # UTC
+  #   labels:
+  #     - "domain: deps"
+  #   commit-message:
+  #     prefix: "robo(deps)"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+      - "domain: ci"
+    commit-message:
+      prefix: "robo(ci)"
--- a/.github/env.sh
+++ b/.github/env.sh
@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -e
+
+if [ $# != 1 ]; then
+	echo "Usage: . $0 <rust-target>"
+	exit 1
+fi
+
+arch=""
+sysroot=""
+need_target_linker=""
+ubuntu_cross_pkg_list=""
+target_linker=""
+target_cc=""
+
+case "$1" in
+	aarch64*)
+		arch="arm64"
+		sysroot="/usr/lib/aarch64-linux-gnu/"
+		ubuntu_cross_pkg_list="gcc-aarch64-linux-gnu"
+		# YES they are the same but otherwise it fails
+		target_linker="aarch64-linux-gnu-gcc"
+		target_cc="aarch64-linux-gnu-gcc"
+
+	;;
+	riscv64*)
+		arch="riscv64"
+		sysroot="/usr/lib/riscv64-linux-gnu/"
+		ubuntu_cross_pkg_list="gcc-riscv64-linux-gnu"
+		# YES they are the same but otherwise it fails
+		target_linker="riscv64-linux-gnu-gcc"
+		target_cc="riscv64-linux-gnu-gcc"
+
+	;;
+	x86_64*)
+		target_linker="gcc"
+		target_cc="gcc"
+	;;
+esac
+
+echo MULTILIB_ARCH=$arch
+echo PKG_CONFIG_SYSROOT_DIR=$sysroot
+echo TARGET_CC=$target_cc
+echo CARGO_TARGET_$(echo "$1" | tr 'a-z' 'A-Z' | tr '-' '_' )_LINKER=$target_linker
+echo UBUNTU_CROSS_PKG_LIST=$ubuntu_cross_pkg_list
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@ -13,7 +13,7 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
    - name: Lint
      run: |
          rustup component add rustfmt
@ -21,23 +21,106 @@ jobs:
          fmt' (version $(cargo fmt --version))"; false)

  tests:
-    runs-on: ubuntu-latest
+    needs: [lint]
+    name: Tests ${{ matrix.target }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        target:
+          - x86_64-unknown-linux-gnu
+        os:
+          - ubuntu-latest

    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+
+    # - name: Set environment variables
+    #   run: |
+    #       .github/env.sh ${{ matrix.target }} >> $GITHUB_ENV
+
+    # - name: Install qemu-user-static
+    #   if: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
+    #   run: |
+    #     sudo apt update
+    #     sudo apt install qemu-user-static
+
+    # - uses: ryankurte/action-apt@v0.3.0
+    #   if: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
+    #   with:
+    #     arch: ${{ env.MULTILIB_ARCH }}
+    #     packages: "libssl-dev:${{ env.MULTILIB_ARCH }}"
+
+    - name: Rustup setup
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        default: true
+        target: ${{ matrix.target }}
+
    - name: Unit tests
      run: |
-          cargo test
+          cargo test --locked

  build:
-    runs-on: ubuntu-latest
-    needs: [lint, tests]
+    needs: [lint]
+
+    name: Build ${{ matrix.target }}-${{ matrix.rust }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        target:
+          - x86_64-unknown-linux-gnu
+          - aarch64-unknown-linux-gnu
+          - riscv64gc-unknown-linux-gnu
+        rust:
+          - stable
+          - 1.70.0
+        os:
+          - ubuntu-latest

    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+
+    - name: Rustup setup
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        default: true
+        target: ${{ matrix.target }}
+
+    - name: Set environment variables
+      run: |
+          .github/env.sh ${{ matrix.target }} >> $GITHUB_ENV
+
+    - name: Install libssl (native)
+      if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
+      run: |
+        sudo apt update
+        sudo apt install libssl-dev
+
+    # - name: Install libssl (cross)
+    #   uses: ryankurte/action-apt@v0.3.0
+    #   if: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
+    #   with:
+    #     arch: ${{ env.MULTILIB_ARCH }}
+    #     packages: "${{ env.UBUNTU_CROSS_PKG_LIST }} libssl-dev:${{ env.MULTILIB_ARCH }}"
+
+    - name: Install libssl (cross)
+      if: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
+      run: |
+        sudo dpkg --add-architecture ${{ env.MULTILIB_ARCH }}
+        sudo sed -i'' -E 's/^(deb|deb-src) /\1 [arch=amd64,i386] /' /etc/apt/sources.list
+        echo "deb [arch=${{ env.MULTILIB_ARCH }}] http://ports.ubuntu.com/ubuntu-ports/ $(lsb_release -cs) main restricted universe" | sudo tee /etc/apt/sources.list.d/${{ env.MULTILIB_ARCH }}.list
+        echo "deb [arch=${{ env.MULTILIB_ARCH }}] http://ports.ubuntu.com/ubuntu-ports/ $(lsb_release -cs)-updates main restricted universe" | sudo tee -a /etc/apt/sources.list.d/${{ env.MULTILIB_ARCH }}.list
+        echo "deb [arch=${{ env.MULTILIB_ARCH }}] http://ports.ubuntu.com/ubuntu-ports/ $(lsb_release -cs)-security main restricted universe" | sudo tee -a /etc/apt/sources.list.d/${{ env.MULTILIB_ARCH }}.list
+        sudo apt update
+        sudo apt install ${{ env.UBUNTU_CROSS_PKG_LIST }} libssl-dev:${{ env.MULTILIB_ARCH }}
+
+          #- name: Setup tmate session
+          #uses: mxschmitt/action-tmate@v3
+
    - name: Check
-      run: |
-          cargo check
+      run: cargo check --locked --target ${{ matrix.target }}
+
    - name: Build
-      run: |
-          cargo build
+      run: cargo build --locked --target ${{ matrix.target }}
--- a/.github/workflows/clippy.yml
+++ b/.github/workflows/clippy.yml
@ -0,0 +1,27 @@
+name: Clippy
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Rustup setup
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features
+
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@ -0,0 +1,37 @@
+name: code-coverage
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  check:
+    name: Rust project
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+
+      - name: Run tarpaulin
+        run: |
+          cargo install cargo-tarpaulin
+          cargo tarpaulin --engine llvm --out Xml -- --test-threads 1
+
+      - name: Upload to codecov.io
+        uses: codecov/codecov-action@v3
+
+      - name: Archive code coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: code-coverage-report
+          path: cobertura.xml
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -0,0 +1,78 @@
+name: Release upload artifact
+
+on:
+  release:
+    types: [published, edited]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+          - x86_64-unknown-linux-gnu
+          - aarch64-unknown-linux-gnu
+          - riscv64gc-unknown-linux-gnu
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Rustup setup
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          default: true
+          target: ${{ matrix.target }}
+
+      - name: Set environment variables
+        run: |
+          .github/env.sh ${{ matrix.target }} >> $GITHUB_ENV
+
+      - name: Install libssl (native)
+        if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
+        run: |
+          sudo apt update
+          sudo apt install libssl-dev
+
+      - name: Install libssl (cross)
+        uses: ryankurte/action-apt@master
+        if: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
+        with:
+          arch: ${{ env.MULTILIB_ARCH }}
+          packages: "${{ env.UBUNTU_CROSS_PKG_LIST }} libssl-dev:${{ env.MULTILIB_ARCH }}"
+
+      - name: Get release upload url
+        id: get_release
+        uses: bruceadams/get-release@v1.3.2
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Build release
+        run: |
+          cargo build --release --locked --target ${{ matrix.target }}
+
+      - name: Compute sha512
+        run: |
+          sha512sum ./target/${{ matrix.target }}/release/suckit | cut -d " " -f 1 > suckit.sha512
+
+      - name: Upload release asset
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.get_release.outputs.upload_url}}
+          asset_path: ./target/${{ matrix.target }}/release/suckit
+          asset_name: suckit-${{ steps.get_release.outputs.tag_name }}-${{ matrix.target }}
+          asset_content_type: application/x-elf
+
+      - name: Upload release asset checksum
+        id: upload-release-asset-sha512
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.get_release.outputs.upload_url}}
+          asset_path: ./suckit.sha512
+          asset_name: suckit-${{ steps.get_release.outputs.tag_name }}-${{ matrix.target }}.sha512
+          asset_content_type: text/plain
--- a/.gitignore
+++ b/.gitignore
@ -6,3 +6,6 @@
 tags

 .idea/
+
+# Ignore local web server folder
+tests/local_server
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@ -1,9 +1,9 @@
 [package]
 name = "suckit"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2018"
 authors = ["Esteban \"Skallwar\" Blanc <estblcsk@gmail.com>",
-	   "Arthur \"CohenArthur\" Cohen <arthur.cohen@epita.fr>"]
+	   "Arthur \"CohenArthur\" Cohen <cohenarthur.dev@gmail.com>"]
 license = "MIT OR Apache-2.0"
 homepage = "https://github.com/skallwar/suckit"
 repository = "https://github.com/skallwar/suckit"
@ -19,21 +19,39 @@ include = [
 	"src/*",
 ]

-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[package.metadata]
+msrv = "1.67.0"
+
+[lib]
+name = "suckit"
+path = "src/lib.rs"
+
+[[bin]]
+name = "suckit"
+path = "src/bin/suckit.rs"
+doc = false

 [dependencies]
-structopt = "0.3"
-crossbeam = "0.7"
-reqwest = { version = "0.10", features = ["blocking", "cookies"] }
-pretty_assertions = "0.6"
-kuchiki = "0.8"
-colored = "1.9"
-chrono = "0.4"
-bytes = "0.5"
-percent-encoding = "2.1"
-url = "2.1"
-rand = "0.7"
-regex = "1.3.7"
+structopt = "^0.3"
+crossbeam = "^0.8"
+reqwest = { version = "^0.11", features = ["blocking", "cookies"] }
+kuchiki = "^0.8"
+colored = "2.0"
+chrono = "^0.4"
+url = "^2.3"
+rand = "^0.8"
+regex = "^1.6"
+encoding_rs = "^0.8"
+lazy_static = "1.4.0"
+pathdiff = "^0.2"
+md5 = "^0.7"
+symlink = "^0.1.0"

 [dev-dependencies]
-tiny_http = "0.7.0"
+tiny_http = "^0.12"
+subprocess = "^0.2"
+mktemp = "^0.5"
+portpicker = "^0.1"
+
+[profile.release]
+lto = true
--- a/README.md
+++ b/README.md
@ -1,11 +1,18 @@
 ![Build and test](https://github.com/Skallwar/suckit/workflows/Build%20and%20test/badge.svg)
+[![codecov](https://codecov.io/gh/Skallwar/suckit/branch/master/graph/badge.svg?token=ZLD369AY2G)](https://codecov.io/gh/Skallwar/suckit)
+[![Crates.io](https://img.shields.io/crates/v/suckit.svg)](https://crates.io/crates/suckit)
+[![Docs](https://docs.rs/suckit/badge.svg)](https://docs.rs/suckit)
+[![Deps](https://deps.rs/repo/github/Skallwar/suckit/status.svg)](https://deps.rs/repo/github/Skallwar/suckit)
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![License](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+![MSRV](https://img.shields.io/badge/MSRV-1.70.0-blue)

 # SuckIT

 `SuckIT` allows you to recursively visit and download a website's content to
 your disk.

-![SuckIT Logo](suckit_logo.png)
+![SuckIT Logo](media/suckit_logo.png)

 # Features

@ -13,22 +20,62 @@ your disk.
 * [x] Uses multithreading
 * [x] Writes the website's content to your disk
 * [x] Enables offline navigation
+* [x] Offers random delays to avoid IP banning
 * [ ] Saves application state on CTRL-C for later pickup
-* [ ] Offers random delays to avoid IP banning

 # Options
+```console
+USAGE:
+    suckit [FLAGS] [OPTIONS] <url>

-|Option|Behavior|
-|---|---|
-|`-h, --help`|Displays help information|
-|`-v, --verbose`|Activate Verbose output|
-|`-d, --depth`|Specify the level of depth to go to when visiting the website|
-|`-j, --jobs`|Number of threads to use|
-|`-o, --output`|Output directory where the downloaded files are written|
-|`-t, --tries`|Number of times to retry when the downloading of a page fails|
-|`-u, --user-agent`|User agent to be used for sending requests|
-|`-i, --include`|Specify a regex to include pages that match this pattern|
-|`-e, --exclude`|Specify a regex to exclude pages that match this pattern|
+FLAGS:
+    -c, --continue-on-error                  Flag to enable or disable exit on error
+        --disable-certs-checks               Dissable SSL certificates verification
+        --dry-run                            Do everything without saving the files to the disk
+    -h, --help                               Prints help information
+    -V, --version                            Prints version information
+    -v, --verbose                            Enable more information regarding the scraping process
+        --visit-filter-is-download-filter    Use the dowload filter in/exclude regexes for visiting as well
+
+OPTIONS:
+    -a, --auth <auth>...
+            HTTP basic authentication credentials space-separated as "username password host". Can be repeated for
+            multiple credentials as "u1 p1 h1 u2 p2 h2"
+        --cookie <cookie>
+            Cookie to send with each request, format: key1=value1;key2=value2 [default: ]
+
+        --delay <delay>
+            Add a delay in seconds between downloads to reduce the likelihood of getting banned [default: 0]
+
+    -d, --depth <depth>
+            Maximum recursion depth to reach when visiting. Default is -1 (infinity) [default: -1]
+
+    -e, --exclude-download <exclude-download>
+            Regex filter to exclude saving pages that match this expression [default: $^]
+
+        --exclude-visit <exclude-visit>
+            Regex filter to exclude visiting pages that match this expression [default: $^]
+
+        --ext-depth <ext-depth>
+            Maximum recursion depth to reach when visiting external domains. Default is 0. -1 means infinity [default:
+            0]
+    -i, --include-download <include-download>
+            Regex filter to limit to only saving pages that match this expression [default: .*]
+
+        --include-visit <include-visit>
+            Regex filter to limit to only visiting pages that match this expression [default: .*]
+
+    -j, --jobs <jobs>                            Maximum number of threads to use concurrently [default: 1]
+    -o, --output <output>                        Output directory
+        --random-range <random-range>
+            Generate an extra random delay between downloads, from 0 to this number. This is added to the base delay
+            seconds [default: 0]
+    -t, --tries <tries>                          Maximum amount of retries on download failure [default: 20]
+    -u, --user-agent <user-agent>                User agent to be used for sending requests [default: suckit]
+
+ARGS:
+    <url>    Entry point of the scraping
+```

 # Example

@ -36,7 +83,7 @@ A common use case could be the following:

 `suckit http://books.toscrape.com -j 8 -o /path/to/downloaded/pages/`

-[![asciicast](https://asciinema.org/a/327889.svg)](https://asciinema.org/a/327889)
+![asciicast](media/suckit-adjusted-120cols-40rows-100ms.svg)

 # Installation

@ -52,6 +99,14 @@ instructions on how to install Rust.

 * Now, run it from anywhere with the `suckit` command.

+### Arch Linux
+
+`suckit` can be installed from available [AUR packages](https://aur.archlinux.org/packages/?O=0&SeB=b&K=suckit&outdated=&SB=n&SO=a&PP=50&do_Search=Go) using an [AUR helper](https://wiki.archlinux.org/index.php/AUR_helpers). For example,
+
+```
+yay -S suckit
+```
+
 __Want to contribute ? Feel free to
 [open an issue](https://github.com/Skallwar/suckit/issues/new) or
 [submit a PR](https://github.com/Skallwar/suckit/compare) !__
--- a/flake.lock
+++ b/flake.lock
@ -0,0 +1,130 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1705309234,
+        "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1681202837,
+        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1705774713,
+        "narHash": "sha256-j6ADaDH9XiumUzkTPlFyCBcoWYhO83lfgiSqEJF2zcs=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "1b64fc1287991a9cce717a01c1973ef86cb1af0b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1681358109,
+        "narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs",
+        "rust-overlay": "rust-overlay"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs_2"
+      },
+      "locked": {
+        "lastModified": 1705889935,
+        "narHash": "sha256-77KPBK5e0ACNzIgJDMuptTtEqKvHBxTO3ksqXHHVO+4=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "e36f66bb10b09f5189dc3b1706948eaeb9a1c555",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
--- a/flake.nix
+++ b/flake.nix
@ -0,0 +1,29 @@
+{
+  description = "SuckIT devshell";
+
+  inputs = {
+    nixpkgs.url      = "github:NixOS/nixpkgs/nixos-23.11";
+    rust-overlay.url = "github:oxalica/rust-overlay";
+    flake-utils.url  = "github:numtide/flake-utils";
+  };
+
+  outputs = { self, nixpkgs, rust-overlay, flake-utils, ... }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        overlays = [ (import rust-overlay) ];
+        pkgs = import nixpkgs {
+          inherit system overlays;
+        };
+      in
+      with pkgs;
+      {
+        devShell = mkShell {
+          buildInputs = [
+            rust-bin.stable.latest.default
+            openssl
+            pkg-config
+          ];
+        };
+      }
+    );
+}
--- a/media/suckit-adjusted-120cols-40rows-100ms.svg
+++ b/media/suckit-adjusted-120cols-40rows-100ms.svg
--- a/media/suckit_logo.png
+++ b/media/suckit_logo.png
--- a/src/args.rs
+++ b/src/args.rs
@ -33,10 +33,17 @@ pub struct Args {
        short,
        long,
        default_value = "-1",
-        help = "Maximum recursion depth to reach when visiting. -1 is the default and will go as far as it can"
+        help = "Maximum recursion depth to reach when visiting. Default is -1 (infinity)"
    )]
    pub depth: i32,

+    #[structopt(
+        long,
+        default_value = "0",
+        help = "Maximum recursion depth to reach when visiting external domains. Default is 0. -1 means infinity"
+    )]
+    pub ext_depth: i32,
+
    ///Number of retries when downloading a page fails
    #[structopt(
        short,
@ -79,29 +86,79 @@ pub struct Args {
    )]
    pub user_agent: String,

-    /// Regex filter to limit saving pages to only matched ones
+    /// Cookie header
    #[structopt(
-        short,
        long,
-        default_value = ".*",
-        parse(try_from_str = parse_regex),
-        help = "Regex filter to limit to only saving pages that match this expression"
+        default_value = "",
+        help = "Cookie to send with each request, format: key1=value1;key2=value2"
    )]
-    pub include: Regex,
+    pub cookie: String,
+
+    /// Regex filter to limit visiting pages to only matched ones
+    #[structopt(
+    long,
+    default_value = ".*",
+    parse(try_from_str = parse_regex),
+    help = "Regex filter to limit to only visiting pages that match this expression"
+    )]
+    pub include_visit: Regex,
+
+    /// Regex filter to limit visiting pages to only matched ones
+    #[structopt(
+    long,
+    default_value = "$^",
+    parse(try_from_str = parse_regex),
+    help = "Regex filter to exclude visiting pages that match this expression"
+    )]
+    pub exclude_visit: Regex,

    /// Regex filter to limit saving pages to only matched ones
+    #[structopt(
+    short,
+    long,
+    default_value = ".*",
+    parse(try_from_str = parse_regex),
+    help = "Regex filter to limit to only saving pages that match this expression"
+    )]
+    pub include_download: Regex,
+
+    /// Regex filter to limit saving pages to only matched ones
+    #[structopt(
+    short,
+    long,
+    default_value = "$^",
+    parse(try_from_str = parse_regex),
+    help = "Regex filter to exclude saving pages that match this expression"
+    )]
+    pub exclude_download: Regex,
+
+    /// If set, set the visit filter to the values of the download filter
+    #[structopt(
+        long,
+        help = "Use the dowload filter in/exclude regexes for visiting as well"
+    )]
+    pub visit_filter_is_download_filter: bool,
+
+    /// HTTP basic authentication credentials
    #[structopt(
        short,
        long,
-        default_value = "$^",
-        parse(try_from_str = parse_regex),
-        help = "Regex filter to exclude saving pages that match this expression"
+        use_delimiter = true,
+        value_delimiter = " ",
+        help = "HTTP basic authentication credentials space-separated as \"username password host\". Can be repeated for multiple credentials as \"u1 p1 h1 u2 p2 h2\""
    )]
-    pub exclude: Regex,
+    pub auth: Vec<String>,

    /// Decides if we should bail out on download error (like, too many redirects)
    #[structopt(short, long, help = "Flag to enable or disable exit on error")]
    pub continue_on_error: bool,
+
+    /// If set, run without saving anything to the disk
+    #[structopt(long, help = "Do everything without saving the files to the disk")]
+    pub dry_run: bool,
+
+    #[structopt(long, help = "Dissable SSL certificates verification")]
+    pub disable_certs_checks: bool,
 }

 impl Args {
--- a/src/bin/suckit.rs
+++ b/src/bin/suckit.rs
@ -0,0 +1,10 @@
+use suckit::args::Args;
+use suckit::scraper::Scraper;
+
+fn main() {
+    let args = Args::collect();
+
+    let mut scraper = Scraper::new(args);
+
+    scraper.run();
+}
--- a/src/disk.rs
+++ b/src/disk.rs
@ -1,7 +1,11 @@
+use pathdiff;
 use std::fs;
+
 use std::io::Write;
 use std::path::PathBuf;

+use symlink::symlink_file;
+
 use crate::{error, warn};

 ///Save content in a file
@ -12,11 +16,8 @@ pub fn save_file(file_name: &str, content: &[u8], path: &Option<PathBuf>) {
    };

    if let Some(parent) = path.parent() {
-        match fs::create_dir_all(parent) {
-            Err(err) => {
-                error!("Couldn't create folder {}: {}", parent.display(), err);
-            }
-            Ok(()) => (),
+        if let Err(err) = fs::create_dir_all(parent) {
+            error!("Couldn't create folder {}: {}", parent.display(), err);
        }
    }

@ -32,16 +33,33 @@ pub fn save_file(file_name: &str, content: &[u8], path: &Option<PathBuf>) {

 ///Create a symlink
 pub fn symlink(source: &str, destination: &str, path: &Option<PathBuf>) {
+    let source = match path {
+        Some(path) => path.join(source),
+        None => PathBuf::from(source),
+    };
+
+    if let Some(parent) = source.parent() {
+        match fs::create_dir_all(parent) {
+            Err(err) => {
+                error!("Couldn't create folder {}: {}", parent.display(), err);
+            }
+            Ok(()) => (),
+        }
+    }
+
    let destination = match path {
        Some(path) => path.join(destination),
        None => PathBuf::from(destination),
    };

-    if let Err(_) = std::os::unix::fs::symlink(source, &destination) {
+    let target = pathdiff::diff_paths(&destination, &source.parent().unwrap()).unwrap();
+
+    if let Err(err) = symlink_file(&target, &source) {
        warn!(
-            "{} is already present, coulnd't create a symlink to {}",
-            destination.display(),
-            source,
+            "Couldn't create symlink\n{} -> {}:\n{:#?}",
+            source.display(),
+            target.display(),
+            err,
        );
    }
 }
--- a/src/downloader.rs
+++ b/src/downloader.rs
@ -1,24 +1,88 @@
-use super::response::{Response, ResponseData};
+use std::collections::HashMap;
+
+use lazy_static::lazy_static;
+use regex::Regex;
+use reqwest::header::{HeaderMap, HeaderValue, COOKIE};
 use url::Url;

 use crate::warn;

+use super::response::{Response, ResponseData};
+
+const AUTH_CHUNK_SIZE: usize = 3;
+
 ///A Downloader to download web content
 pub struct Downloader {
    client: reqwest::blocking::Client,
    tries: usize,
+    auth_map: HashMap<String, (String, Option<String>)>,
+}
+
+/// Parse HTTP authentication credentials from string iterable
+fn parse_auth(auth: &[String], origin: &Url) -> Result<(String, Option<String>, String), String> {
+    // Convert any empty strings to None
+    let auth: Vec<Option<String>> = auth
+        .iter()
+        .map(|s| match s.as_ref() {
+            "" => None,
+            s => Some(s.to_string()),
+        })
+        .collect();
+
+    // Match on auth values and origin host, defaulting to the origin host if host not provided
+    match (auth.as_slice(), origin.host_str()) {
+        ([Some(username)], Some(origin_host)) => {
+            Ok((username.to_string(), None, origin_host.to_string()))
+        }
+        ([Some(username), password], Some(origin_host)) => Ok((
+            username.to_string(),
+            password.clone(),
+            origin_host.to_string(),
+        )),
+        ([Some(username), password, None, ..], Some(origin_host)) => Ok((
+            username.to_string(),
+            password.clone(),
+            origin_host.to_string(),
+        )),
+        ([Some(username), password, Some(host), ..], _) => {
+            Ok((username.to_string(), password.clone(), host.to_string()))
+        }
+        _ => Err("Invalid arguments supplied to auth".to_string()),
+    }
 }

 impl Downloader {
    /// Create a new Downloader
-    pub fn new(tries: usize, user_agent: &str) -> Downloader {
+    pub fn new(
+        tries: usize,
+        user_agent: &str,
+        cookie: &str,
+        disable_certs_checks: bool,
+        auth: &[String],
+        origin: &Url,
+    ) -> Downloader {
+        // Create a mapping of hosts to username, password tuples for authentication
+        let mut auth_map = HashMap::new();
+        // Iterate over the auth string in chunks of 3 items each for (username, password, host)
+        for auth_chunk in auth.chunks(AUTH_CHUNK_SIZE) {
+            // Throwing the error with panic! for now if parsing fails
+            let (username, password, host) = parse_auth(auth_chunk, origin).unwrap();
+            auth_map.insert(host, (username, password));
+        }
+
+        let mut headers = HeaderMap::new();
+        headers.insert(COOKIE, HeaderValue::from_str(cookie).unwrap());
+
        Downloader {
            client: reqwest::blocking::ClientBuilder::new()
+                .default_headers(headers)
+                .danger_accept_invalid_certs(disable_certs_checks)
                .cookie_store(true)
                .user_agent(user_agent)
                .build()
                .unwrap(),
            tries,
+            auth_map,
        }
    }

@ -39,30 +103,64 @@ impl Downloader {
        }
    }

+    /// Load HTTP auth credentials in a username, password tuple based on the host string
+    fn get_auth(&self, url: &Url) -> Option<&(String, Option<String>)> {
+        if let Some(host) = url.host_str() {
+            self.auth_map.get(&host.to_string())
+        } else {
+            None
+        }
+    }
+
    ///Download the content at this url
    fn make_request(&self, url: &Url) -> Result<Response, reqwest::Error> {
-        match self.client.get(url.clone()).send() {
+        let req = self.client.get(url.clone());
+        let req = match self.get_auth(url) {
+            Some((username, password)) => req.basic_auth(username, password.clone()),
+            None => req,
+        };
+        match req.send() {
            Ok(mut data) => {
-                let data_type = match data.headers().get("content-type") {
-                    Some(data_type) => data_type.to_str().unwrap(),
-                    None => "text/html",
-                };
+                lazy_static! {
+                    static ref DATA_TYPE_REGEX: Regex =
+                        Regex::new(r#"^.*(\b[a-z]+/[a-z-+\.]+).*$"#).unwrap();
+                    static ref CHARSET_REGEX: Regex =
+                        Regex::new(r#"^.*charset\s*=\s*["']?([^"'\s;]+).*$"#).unwrap();
+                }

-                let filename = if !Downloader::is_html(data_type) {
+                let (data_type, charset): (String, Option<String>) =
+                    match data.headers().get("content-type") {
+                        Some(content_type_header) => {
+                            let content_type = content_type_header.to_str().unwrap();
+                            let data_type_captures =
+                                DATA_TYPE_REGEX.captures_iter(content_type).next();
+                            let data_type = data_type_captures
+                                .map_or(String::from("text/html"), |first| {
+                                    first.get(1).unwrap().as_str().to_lowercase()
+                                });
+                            let charset_captures = CHARSET_REGEX.captures_iter(content_type).next();
+                            let charset = charset_captures
+                                .map(|first| first.get(1).unwrap().as_str().to_lowercase());
+                            (data_type, charset)
+                        }
+                        None => (String::from("text/html"), None),
+                    };
+
+                let filename = if !Downloader::is_html(&data_type) {
                    Downloader::get_filename(data.headers())
                } else {
                    None
                };

-                let data = if Downloader::is_html(data_type) {
-                    ResponseData::Html(data.text().unwrap())
+                let mut raw_data: Vec<u8> = Vec::new();
+                data.copy_to(&mut raw_data).unwrap();
+                let response_data = if Downloader::is_html(&data_type) {
+                    ResponseData::Html(raw_data)
                } else {
-                    let mut raw_data: Vec<u8> = Vec::new();
-                    data.copy_to(&mut raw_data).unwrap();
                    ResponseData::Other(raw_data)
                };

-                Ok(Response::new(data, filename))
+                Ok(Response::new(response_data, filename, charset))
            }

            Err(e) => {
@ -93,9 +191,39 @@ mod tests {
    #[test]
    fn test_download_url() {
        let url: Url = Url::parse("https://lwn.net").unwrap();
-        match Downloader::new(1, "suckit").get(&url) {
+        match Downloader::new(1, "suckit", "", false, &[], &url).get(&url) {
            Err(e) => assert!(false, "Fail to download lwn.net: {:?}", e),
            _ => {}
        }
    }
+
+    #[test]
+    fn test_parse_auth() {
+        assert_eq!(
+            parse_auth(
+                &["".to_string(), "pw".to_string()],
+                &Url::parse("https://example.com/").unwrap(),
+            ),
+            Err("Invalid arguments supplied to auth".to_string())
+        );
+        assert_eq!(
+            parse_auth(
+                &["username".to_string()],
+                &Url::parse("https://example.com/").unwrap(),
+            ),
+            Ok(("username".to_string(), None, "example.com".to_string()))
+        );
+        assert_eq!(
+            parse_auth(
+                &[
+                    "un".to_string(),
+                    "pw".to_string(),
+                    "h".to_string(),
+                    "t".to_string()
+                ],
+                &Url::parse("https://example.com/").unwrap(),
+            ),
+            Ok(("un".to_string(), Some("pw".to_string()), "h".to_string()))
+        )
+    }
 }
--- a/src/lib.rs
+++ b/src/lib.rs
@ -0,0 +1,8 @@
+pub mod args;
+pub mod disk;
+pub mod dom;
+pub mod downloader;
+pub mod logger;
+pub mod response;
+pub mod scraper;
+pub mod url_helper;
--- a/src/logger.rs
+++ b/src/logger.rs
@ -39,7 +39,7 @@ impl Logger {
            "ERROR".red(),
            message
        );
-        panic!(message)
+        panic!("{}", message)
    }
 }

--- a/src/main.rs
+++ b/src/main.rs
@ -1,18 +0,0 @@
-mod args;
-mod disk;
-mod dom;
-mod downloader;
-mod logger;
-mod response;
-mod scraper;
-mod url_helper;
-
-use scraper::Scraper;
-
-fn main() {
-    let args = args::Args::collect();
-
-    let mut scraper = Scraper::new(args);
-
-    scraper.run();
-}
--- a/src/response.rs
+++ b/src/response.rs
@ -1,6 +1,6 @@
 /// Separates HTML responses and other content (PDFs, images...)
 pub enum ResponseData {
-    Html(String),
+    Html(Vec<u8>),
    Other(Vec<u8>),
 }

@ -8,11 +8,16 @@ pub enum ResponseData {
 pub struct Response {
    pub data: ResponseData,
    pub filename: Option<String>,
+    pub charset: Option<String>,
 }

 impl Response {
    ///Create a new Response
-    pub fn new(data: ResponseData, filename: Option<String>) -> Response {
-        Response { data, filename }
+    pub fn new(data: ResponseData, filename: Option<String>, charset: Option<String>) -> Response {
+        Response {
+            data,
+            filename,
+            charset,
+        }
    }
 }
--- a/src/scraper.rs
+++ b/src/scraper.rs
@ -1,25 +1,28 @@
-use crossbeam::channel::{Receiver, Sender, TryRecvError};
-use crossbeam::thread;
-use url::Url;
-
+use std::borrow::Borrow;
 use std::collections::HashMap;
 use std::collections::HashSet;
-use std::process;
+use std::path::Path;
 use std::sync::Mutex;
 use std::time;

+use crossbeam::channel::{Receiver, Sender, TryRecvError};
+use crossbeam::thread;
+use encoding_rs::Encoding;
+use lazy_static::lazy_static;
+use pathdiff;
 use rand::Rng;
+use regex::Regex;
+use url::Url;

-use super::downloader;
+use crate::{error, info, warn};

 use super::args;
 use super::disk;
 use super::dom;
+use super::downloader;
 use super::response;
 use super::url_helper;

-use crate::{error, info};
-
 /// Maximum number of empty recv() from the channel
 static MAX_EMPTY_RECEIVES: usize = 10;

@ -27,15 +30,15 @@ static MAX_EMPTY_RECEIVES: usize = 10;
 static INFINITE_DEPTH: i32 = -1;

 /// Sleep duration on empty recv()
-static SLEEP_MILLIS: u64 = 100;
+static SLEEP_MILLIS: u64 = 500;
 static SLEEP_DURATION: time::Duration = time::Duration::from_millis(SLEEP_MILLIS);

 /// Producer and Consumer data structure. Handles the incoming requests and
 /// adds more as new URLs are found
 pub struct Scraper {
    args: args::Args,
-    transmitter: Sender<(Url, i32)>,
-    receiver: Receiver<(Url, i32)>,
+    transmitter: Sender<(Url, i32, i32)>,
+    receiver: Receiver<(Url, i32, i32)>,
    downloader: downloader::Downloader,
    visited_urls: Mutex<HashSet<String>>,
    path_map: Mutex<HashMap<String, String>>,
@ -46,8 +49,21 @@ impl Scraper {
    pub fn new(args: args::Args) -> Scraper {
        let (tx, rx) = crossbeam::channel::unbounded();

+        let mut args = args;
+        if args.visit_filter_is_download_filter {
+            args.include_visit = args.include_download.clone();
+            args.exclude_visit = args.exclude_download.clone();
+        }
+
        Scraper {
-            downloader: downloader::Downloader::new(args.tries, &args.user_agent),
+            downloader: downloader::Downloader::new(
+                args.tries,
+                &args.user_agent,
+                &args.cookie,
+                args.disable_certs_checks,
+                &args.auth,
+                &args.origin,
+            ),
            args,
            transmitter: tx,
            receiver: rx,
@ -69,60 +85,175 @@ impl Scraper {
    }

    /// Push a new URL into the channel
-    fn push(transmitter: &Sender<(Url, i32)>, url: Url, depth: i32) {
-        if let Err(e) = transmitter.send((url, depth)) {
+    fn push(transmitter: &Sender<(Url, i32, i32)>, url: Url, depth: i32, ext_depth: i32) {
+        if let Err(e) = transmitter.send((url, depth, ext_depth)) {
            error!("Couldn't push to channel ! {}", e);
        }
    }

-    /// Fix the URLs contained in the DOM-tree so they point to each other
-    fn fix_domtree(&self, old_url_str: &mut String, new_url: &Url) {
-        let path_map = self.path_map.lock().unwrap();
-        let path = path_map.get(new_url.as_str()).unwrap();
+    /// Fix the URLs contained in the DOM-tree so they point to each other relatively
+    fn fix_domtree(&self, dom_url: &mut String, source_path: &str, dest_path: &str) {
+        let source_path_parent = Path::new(source_path).parent().unwrap().to_str().unwrap(); //Unwrap should be safe, there will alway be at least .../index.html
+        let diff_path = pathdiff::diff_paths(dest_path, source_path_parent).unwrap();
+        let relative_path = diff_path.as_path().to_str().unwrap();

-        let new_url_str = url_helper::encode(path);
-
-        old_url_str.clear();
-        old_url_str.push_str(&new_url_str);
+        dom_url.clear();
+        dom_url.push_str(relative_path);
    }

-    ///Proces an html file: add new url to the chanel and prepare for offline navigation
+    /// Find the charset of the webpage. ``data`` is not a String as this might not be utf8.
+    /// Returned String is lower cased
+    /// This is a hack and should be check in case of a bug
+    fn find_charset(data: &[u8], http_charset: Option<String>) -> Option<String> {
+        lazy_static! {
+            static ref CHARSET_REGEX: Regex =
+                Regex::new(r#"<meta.*charset\s*=\s*["']?([^"'\s;>]+).*>"#).unwrap();
+        }
+
+        // We don't know the real charset yet. We hope that the charset is ASCII
+        // compatible, because Rust String are in UTF-8 (also ASCII compatible).
+        let data_utf8 = unsafe { String::from_utf8_unchecked(Vec::from(data)) };
+        let captures = CHARSET_REGEX.captures_iter(&data_utf8).next();
+
+        // We use the first one, hopping we are in the <head> of the page... or if nothing is found
+        // we used the http charset (if any).
+        captures
+            .map(|first| first.get(1).unwrap().as_str().to_lowercase())
+            .or(http_charset)
+    }
+
+    /// Proceed to convert the data in utf8.
+    fn charset_convert(
+        data: &[u8],
+        charset_source: &'static Encoding,
+        charset_dest: &'static Encoding,
+    ) -> Vec<u8> {
+        let decode_result = charset_source.decode(data);
+        let decode_bytes = decode_result.0.borrow();
+
+        let encode_result = charset_dest.encode(decode_bytes);
+
+        encode_result.0.into_owned()
+    }
+
+    /// Check if the charset require conversion
+    fn needs_charset_conversion(charset: &str) -> bool {
+        !matches!(charset, "utf-8")
+    }
+
+    /// Proces an html file: add new url to the chanel and prepare for offline navigation
    fn handle_html(
        scraper: &Scraper,
-        transmitter: &Sender<(Url, i32)>,
+        transmitter: &Sender<(Url, i32, i32)>,
        url: &Url,
        depth: i32,
-        data: &str,
+        ext_depth: i32,
+        data: &[u8],
+        http_charset: Option<String>,
    ) -> Vec<u8> {
-        let dom = dom::Dom::new(data);
+        let charset_source_str = match Self::find_charset(data, http_charset) {
+            Some(s) => s,
+            None => {
+                warn!("Charset not found for {}, defaulting to UTF-8", url);
+                String::from("utf-8")
+            }
+        };
+
+        let need_charset_conversion = Self::needs_charset_conversion(&charset_source_str);
+
+        let charset_source = match encoding_rs::Encoding::for_label(charset_source_str.as_bytes()) {
+            Some(encoder) => encoder,
+            None => {
+                warn!(
+                    "Charset {} not supported for {}, defaulting to UTF-8",
+                    charset_source_str, url
+                );
+                encoding_rs::UTF_8
+            }
+        };
+        let charset_utf8 = encoding_rs::UTF_8;
+        let utf8_data = if need_charset_conversion {
+            Self::charset_convert(data, charset_source, charset_utf8)
+        } else {
+            Vec::from(data)
+        };
+
+        let dom = dom::Dom::new(&String::from_utf8_lossy(&utf8_data));
+        let source_path = match scraper.path_map.lock().unwrap().get(url.as_str()) {
+            Some(path) => path.clone(),
+            None => error!("Url {} was not found in the path map", url.as_str()),
+        };

        dom.find_urls_as_strings()
            .into_iter()
-            .filter(|candidate| Scraper::should_visit(candidate, &url))
+            .filter(|candidate| Scraper::should_visit(scraper, candidate))
            .for_each(|next_url| {
-                let next_full_url = url.join(&next_url).unwrap();
-                let path = url_helper::to_path(&next_full_url);
+                let url_to_parse = Scraper::normalize_url(next_url.clone());

-                if scraper.map_url_path(&next_full_url, path)
-                    && (scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth)
-                {
-                    Scraper::push(transmitter, next_full_url.clone(), depth + 1);
+                let next_full_url = match url.join(url_to_parse.as_str()) {
+                    Ok(url) => url,
+                    Err(e) => {
+                        warn!("Failed to parse url: {} | Error: {}", next_url, e);
+                        return;
+                    },
+                };
+
+                let path = url_helper::to_path(&next_full_url, true);
+                let path_no_fragments = url_helper::to_path(&next_full_url, false);
+
+                // We only add urls without fragments to avoid duplication
+                if scraper.map_url_path(&next_full_url, path_no_fragments.clone()) {
+                    if !Scraper::is_on_another_domain(next_url, url) {
+                        // If we are determining for a local domain
+                        if scraper.args.depth == INFINITE_DEPTH || depth < scraper.args.depth {
+                            Scraper::push(transmitter, next_full_url, depth + 1, ext_depth);
+                        }
+                    } else {
+                        // If we are determining for an external domain
+                        if scraper.args.ext_depth == INFINITE_DEPTH
+                            || ext_depth < scraper.args.ext_depth
+                        {
+                            Scraper::push(transmitter, next_full_url, depth, ext_depth + 1);
+                        }
+                    }
                }

-                scraper.fix_domtree(next_url, &next_full_url);
+                scraper.fix_domtree(next_url, &source_path, &path);
            });

-        dom.serialize().into_bytes()
+        let utf8_data = dom.serialize().into_bytes();
+
+        if need_charset_conversion {
+            Self::charset_convert(&utf8_data, charset_utf8, charset_source)
+        } else {
+            utf8_data
+        }
    }

    /// Process a single URL
-    fn handle_url(scraper: &Scraper, transmitter: &Sender<(Url, i32)>, url: Url, depth: i32) {
+    fn handle_url(
+        scraper: &Scraper,
+        transmitter: &Sender<(Url, i32, i32)>,
+        url: Url,
+        depth: i32,
+        ext_depth: i32,
+    ) {
+        let download_filter_matches = !scraper.args.exclude_download.is_match(url.as_str())
+            && scraper.args.include_download.is_match(url.as_str());
+        // download the page even if the download filter does not match,
+        // so its links can be discovered and added to the queue
        match scraper.downloader.get(&url) {
            Ok(response) => {
                let data = match response.data {
-                    response::ResponseData::Html(data) => {
-                        Scraper::handle_html(scraper, transmitter, &url, depth, &data)
-                    }
+                    response::ResponseData::Html(data) => Scraper::handle_html(
+                        scraper,
+                        transmitter,
+                        &url,
+                        depth,
+                        ext_depth,
+                        &data,
+                        response.charset,
+                    ),
                    response::ResponseData::Other(data) => data,
                };

@ -131,9 +262,7 @@ impl Scraper {
                    let path_map = scraper.path_map.lock().unwrap();
                    let path = path_map.get(url.as_str()).unwrap();

-                    if !scraper.args.exclude.is_match(url.as_str())
-                        && scraper.args.include.is_match(url.as_str())
-                    {
+                    if !scraper.args.dry_run && download_filter_matches {
                        match response.filename {
                            Some(filename) => {
                                disk::save_file(&filename, &data, &scraper.args.output);
@ -147,9 +276,10 @@ impl Scraper {
                }
            }
            Err(e) => {
-                println!("Couldn't download a page, {:?}", e);
                if !scraper.args.continue_on_error {
-                    process::exit(1);
+                    error!("Couldn't download a page, {:?}", e);
+                } else {
+                    warn!("Couldn't download a page, {:?}", e);
                }
            }
        }
@ -157,15 +287,22 @@ impl Scraper {
        scraper.visited_urls.lock().unwrap().insert(url.to_string());

        if scraper.args.verbose {
-            info!("Visited: {}", url);
+            if download_filter_matches {
+                info!("Downloaded: {}", url);
+            } else {
+                info!("Visited: {}", url);
+            }
        }
    }

    /// Run through the channel and complete it
    pub fn run(&mut self) {
        /* Push the origin URL and depth (0) through the channel */
-        self.map_url_path(&self.args.origin, url_helper::to_path(&self.args.origin));
-        Scraper::push(&self.transmitter, self.args.origin.clone(), 0);
+        self.map_url_path(
+            &self.args.origin,
+            url_helper::to_path(&self.args.origin, false),
+        );
+        Scraper::push(&self.transmitter, self.args.origin.clone(), 0, 0);

        thread::scope(|thread_scope| {
            for _ in 0..self.args.jobs {
@ -187,9 +324,9 @@ impl Scraper {
                                }
                                TryRecvError::Disconnected => panic!("{}", e),
                            },
-                            Ok((url, depth)) => {
+                            Ok((url, depth, ext_depth)) => {
                                counter = 0;
-                                Scraper::handle_url(&self_clone, &tx, url, depth);
+                                Scraper::handle_url(self_clone, &tx, url, depth, ext_depth);
                                self_clone.sleep(&mut rng);
                            }
                        }
@ -210,19 +347,22 @@ impl Scraper {
        }

        // delay_range+1 because gen_range is exclusive on the upper limit
-        let rand_delay_secs = rng.gen_range(0, random_range + 1);
+        let rand_delay_secs = rng.gen_range(0..random_range + 1);
        let delay_duration = time::Duration::from_secs(base_delay + rand_delay_secs);
        std::thread::sleep(delay_duration);
    }

-    /// If a URL should be visited, or does it belong to another domain
-    fn should_visit(url: &str, base: &Url) -> bool {
+    /// If a URL should be visited (ignores `mail:`, `javascript:` and other pseudo-links)
+    fn should_visit(scraper: &Scraper, url: &str) -> bool {
+        if scraper.args.exclude_visit.is_match(url) || !scraper.args.include_visit.is_match(url) {
+            return false;
+        }
        match Url::parse(url) {
            /* The given candidate is a valid URL, and not a relative path to
-             * the next one. Therefore, we have to check if this URL belongs
-             * to the same domain as our current URL. If the candidate has the
-             * same domain as our base, then we should visit it */
-            Ok(not_ok) => not_ok.domain() == base.domain(),
+             * the next one. Therefore, we have to check if this URL is valid.
+             * If it is, we should visit it.
+             */
+            Ok(not_ok) => not_ok.has_host() && !not_ok.cannot_be_a_base(),

            /* Since we couldn't parse this "URL", then it must be a relative
             * path or a malformed URL. If the URL is malformed, then it will
@ -230,14 +370,47 @@ impl Scraper {
            Err(_) => true,
        }
    }
+
+    /// Replaces `///` with `//`
+    /// And `//` with `https://`
+    /// Without this function, if url is `///<domain>.<extension>/`, the app crashes.
+    fn normalize_url(url: String) -> String {
+        if url.starts_with("///") {
+            return url.replacen("///", "https://", 1);
+        } else if url.starts_with("//") {
+            return url.replacen("//", "https://", 1);
+        }
+        url
+    }
+
+    /// If the URL leads to another domain
+    fn is_on_another_domain(url: &str, base: &Url) -> bool {
+        let real_url = Scraper::normalize_url(String::from(url));
+
+        match Url::parse(real_url.as_str()) {
+            /* The given candidate is a valid URL, and not a relative path to
+             * the next one. Therefore, we have to check if this URL belongs
+             * to the same domain as our current URL. If the candidate has the
+             * same domain as our base, and the depth condition is satisfied,
+             * then we should visit it,  */
+            Ok(not_ok) => not_ok.domain() != base.domain(),
+
+            /* Since we couldn't parse this "URL", then it must be a relative
+             * path or a malformed URL. If the URL is malformed, then it will
+             * be handled during the join() call in run() */
+            Err(_) => false,
+        }
+    }
 }

 #[cfg(test)]
 mod tests {
-    use super::*;
-    use regex::Regex;
    use std::path::PathBuf;

+    use regex::Regex;
+
+    use super::*;
+
    #[test]
    fn test_zero_delay_range() {
        let args = args::Args {
@ -246,13 +419,21 @@ mod tests {
            jobs: 1,
            tries: 1,
            depth: 5,
+            ext_depth: 0,
            delay: 0,
            user_agent: "suckit".to_string(),
            random_range: 0,
            verbose: true,
-            include: Regex::new("jpg").unwrap(),
-            exclude: Regex::new("png").unwrap(),
+            include_visit: Regex::new(".*").unwrap(),
+            exclude_visit: Regex::new("^$").unwrap(),
+            include_download: Regex::new("jpg").unwrap(),
+            exclude_download: Regex::new("png").unwrap(),
+            visit_filter_is_download_filter: false,
+            auth: Vec::new(),
            continue_on_error: true,
+            dry_run: false,
+            disable_certs_checks: false,
+            cookie: "".to_string(),
        };

        let _ = Scraper::new(args);
@ -266,15 +447,47 @@ mod tests {
            jobs: 1,
            tries: 1,
            depth: 5,
+            ext_depth: 0,
            delay: 2,
            user_agent: "suckit".to_string(),
            random_range: 5,
            verbose: true,
-            include: Regex::new("jpg").unwrap(),
-            exclude: Regex::new("png").unwrap(),
+            include_visit: Regex::new(".*").unwrap(),
+            exclude_visit: Regex::new("^$").unwrap(),
+            include_download: Regex::new("jpg").unwrap(),
+            exclude_download: Regex::new("png").unwrap(),
+            visit_filter_is_download_filter: false,
+            auth: Vec::new(),
            continue_on_error: true,
+            dry_run: false,
+            disable_certs_checks: false,
+            cookie: "".to_string(),
        };

        let _ = Scraper::new(args);
    }
+
+    #[test]
+    fn test_charset_parsing_double_quotes() {
+        assert_eq!(
+            Scraper::find_charset(b"<meta charset=\"UTF-8\">", None),
+            Some("utf-8".to_string())
+        );
+        assert_eq!(
+            Scraper::find_charset(b"<meta charset=\"windows-1252\">", None),
+            Some("windows-1252".to_string())
+        );
+    }
+
+    #[test]
+    fn test_charset_parsing_single_quotes() {
+        assert_eq!(
+            Scraper::find_charset(b"<meta charset=\'UTF-8\'>", None),
+            Some("utf-8".to_string())
+        );
+        assert_eq!(
+            Scraper::find_charset(b"<meta charset=\'windows-1252\'>", None),
+            Some("windows-1252".to_string())
+        );
+    }
 }
--- a/src/url_helper.rs
+++ b/src/url_helper.rs
@ -1,27 +1,61 @@
-use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
+use std::path::Path;
+
+use md5;
+use std::borrow::Cow;
 use url::Url;

 ///Max file name size supported by the file system
 const FILE_NAME_MAX_LENGTH: usize = 255;
-///Characters that need to be replaced by encode()
-const FRAGMENT: &AsciiSet = &CONTROLS.add(b'?');

-///Encode special character with '%' representation
-pub fn encode(path: &str) -> String {
-    utf8_percent_encode(path, FRAGMENT).to_string()
-}
+/// Convert an Url to the corresponding path
+pub fn to_path(url: &Url, with_fragment: bool) -> String {
+    let url_domain = url.host_str().unwrap();

-///Convert an Url to the corresponding path
-pub fn to_path(url: &Url) -> String {
-    let url = url.as_str().split("://").collect::<Vec<&str>>()[1];
-
-    let mut url = url.replace('/', "_").replace('.', "_");
-    if url.len() >= FILE_NAME_MAX_LENGTH {
-        url = url.split_at(FILE_NAME_MAX_LENGTH).0.to_string(); //Shrink too long file name
+    let mut url_path_and_query = url.path().to_string();
+    if let Some(query) = url.query() {
+        url_path_and_query.push_str("__querystring__");
+        url_path_and_query.push_str(query);
    }
-    let url = url.trim_end_matches('_'); //Remaining '/'

-    url.to_string()
+    let path = Path::new(&url_path_and_query);
+    let mut filename = path.file_name().map_or(String::from(""), |filename| {
+        filename.to_str().unwrap().to_string()
+    });
+    let mut parent = path
+        .parent()
+        .map_or("", |filename| filename.to_str().unwrap())
+        .to_string();
+
+    // Ensure the folder names are not too long
+    parent = parent
+        .split('/')
+        .map(|str| {
+            if str.len() > FILE_NAME_MAX_LENGTH {
+                Cow::Owned(format!("{:x}", md5::compute(str)))
+            } else {
+                Cow::Borrowed(str)
+            }
+        })
+        .collect::<Vec<Cow<str>>>()
+        .join("/");
+
+    if url_path_and_query.ends_with('/') {
+        filename = "index.html".to_string();
+        parent = url_path_and_query.trim_end_matches('/').to_string();
+    } else if Path::new(&filename).extension().is_none() {
+        parent = url_path_and_query.trim_end_matches('/').to_string();
+        filename = "index_no_slash.html".to_string();
+    }
+
+    if filename.len() > FILE_NAME_MAX_LENGTH {
+        let digest = md5::compute(filename);
+        filename = format!("{:x}.html", digest);
+    }
+
+    match (url.fragment(), with_fragment) {
+        (Some(fragment), true) => format!("{}{}/{}#{}", url_domain, parent, filename, fragment),
+        (_, _) => format!("{}{}/{}", url_domain, parent, filename),
+    }
 }

 #[cfg(test)]
@ -29,16 +63,79 @@ mod tests {
    use super::*;

    #[test]
-    fn url_to_path() {
-        let str = super::to_path(&Url::parse("https://lwn.net/Kernel/").unwrap());
+    fn url_to_path_domain_only() {
+        let str = super::to_path(&Url::parse("https://lwn.net/").unwrap(), false);

-        assert_eq!(str, "lwn_net_Kernel");
+        assert_eq!(str, "lwn.net/index.html");
    }

    #[test]
-    fn url_to_path_long() {
-        let str = super::to_path(&Url::parse("https://e8v0pez1lofdxoxgg5vwrnaqkjuvpowp9wtgc2eknlfpjdwmmfti8fcwyjzfdgys3nrgyqyeqjkulpyg9kfiqajza2bwxkinhhpohyrnnoy2bak374tcaxh1ycpboolmx8so9yq9kbcj5wu5cgymqndeqasdak0nvl0ijka6fkkmhhvt43l73bn38rewicd4h1ff2omhpni752jtqyzsjub5coh8dlnr3i35udmkzhxo4db3is9gnqmf3hl.comtest").unwrap());
+    fn url_to_path_domain_only_no_slash() {
+        let str = super::to_path(&Url::parse("https://lwn.net").unwrap(), false);

-        assert_eq!(str, "e8v0pez1lofdxoxgg5vwrnaqkjuvpowp9wtgc2eknlfpjdwmmfti8fcwyjzfdgys3nrgyqyeqjkulpyg9kfiqajza2bwxkinhhpohyrnnoy2bak374tcaxh1ycpboolmx8so9yq9kbcj5wu5cgymqndeqasdak0nvl0ijka6fkkmhhvt43l73bn38rewicd4h1ff2omhpni752jtqyzsjub5coh8dlnr3i35udmkzhxo4db3is9gnqmf3hl_com");
+        assert_eq!(str, "lwn.net/index.html");
+    }
+
+    #[test]
+    fn url_to_path() {
+        let str = super::to_path(
+            &Url::parse("https://lwn.net/Kernel/index.html").unwrap(),
+            false,
+        );
+
+        assert_eq!(str, "lwn.net/Kernel/index.html");
+    }
+
+    #[test]
+    fn url_to_path_index() {
+        let str = super::to_path(&Url::parse("https://lwn.net/Kernel/").unwrap(), false);
+
+        assert_eq!(str, "lwn.net/Kernel/index.html");
+    }
+
+    #[test]
+    fn url_to_path_index_no_slash() {
+        let str = super::to_path(&Url::parse("https://lwn.net/Kernel").unwrap(), false);
+
+        assert_eq!(str, "lwn.net/Kernel/index_no_slash.html");
+    }
+
+    #[test]
+    fn url_to_path_fragment() {
+        let str = super::to_path(
+            &Url::parse("https://lwn.net/Kernel/#fragment").unwrap(),
+            true,
+        );
+
+        assert_eq!(str, "lwn.net/Kernel/index.html#fragment");
+    }
+
+    #[test]
+    fn url_to_path_no_fragment() {
+        let str = super::to_path(
+            &Url::parse("https://lwn.net/Kernel/#fragment").unwrap(),
+            false,
+        );
+
+        assert_eq!(str, "lwn.net/Kernel/index.html");
+    }
+
+    #[test]
+    fn url_to_path_to_long_md5() {
+        let str = super::to_path(&Url::parse("https://lwn.net/Kernel/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.html").unwrap(), false);
+
+        assert_eq!(str, "lwn.net/Kernel/5ca82767de71fe8930587e82bb994903.html");
+    }
+
+    #[test]
+    fn url_to_path_querystrings() {
+        let str = super::to_path(
+            &Url::parse(
+                "https://google.com/foobar/platform-redirect/?next=/configuration/releases/",
+            )
+            .unwrap(),
+            false,
+        );
+        assert_eq!(str, "google.com/foobar/platform-redirect/__querystring__next=/configuration/releases/index.html");
    }
 }
--- a/tests/auth.rs
+++ b/tests/auth.rs
@ -0,0 +1,64 @@
+//! Tests for using --auth flags for suckit
+
+mod fixtures;
+
+use std::fs::read_dir;
+use std::process::Command;
+use std::process::Stdio;
+
+const PAGE: &'static str = "tests/fixtures/";
+const IP: &'static str = "0.0.0.0";
+
+// Shouldn't supply credentials to a non-matching host
+#[test]
+fn auth_different_host() {
+    let ip = fixtures::spawn_local_http_server(PAGE, true, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[
+            &url,
+            "-o",
+            output_dir,
+            "-a",
+            "username password example.com",
+        ])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+    let paths = read_dir(format!("{}/{}", output_dir, IP)).unwrap();
+
+    // Only the initial invalid response file should be present
+    assert_eq!(paths.count(), 1);
+}
+
+// Should authenticate with credentials to host (defaulting to origin host)
+#[test]
+fn auth_valid() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir, "-a", "username password"])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+
+    let paths = read_dir(format!("{}/{}", output_dir, IP)).unwrap();
+    // Should load multiple paths, not just the invalid auth response
+    assert!(paths.count() > 1);
+}
--- a/tests/charset_html_found.rs
+++ b/tests/charset_html_found.rs
@ -0,0 +1,41 @@
+//! Test for charset detection/conversion
+
+mod fixtures;
+
+use std::fs;
+use std::process::{Command, Stdio};
+use std::sync::Once;
+
+const PAGE: &'static str = "tests/fixtures";
+const PAGE_META: &'static str = "tests/fixtures/charset_test_html.html";
+const IP: &'static str = "0.0.0.0";
+
+#[test]
+fn test_html_charset_found() {
+    // Spawn a single instance of a local http server usable by all tests in this module.
+    let addr = fixtures::spawn_local_http_server(PAGE, false, None);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+    let file_dir = format!("{}/{}", output_dir, IP);
+    let url = format!("http://{}/charset_test_html.html", addr);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+    let file_path = fs::read_dir(file_dir)
+        .unwrap()
+        .next()
+        .unwrap()
+        .unwrap()
+        .path(); // There is only one file in the directory
+
+    let data_source = fs::read(PAGE_META).unwrap();
+    let data_downloaded = fs::read(file_path).unwrap();
+
+    assert!(fixtures::do_vecs_match(&data_source, &data_downloaded));
+}
--- a/tests/charset_html_not_found.rs
+++ b/tests/charset_html_not_found.rs
@ -0,0 +1,42 @@
+//! Test for charset detection/conversion
+
+mod fixtures;
+
+use std::fs;
+use std::process::{Command, Stdio};
+
+const PAGE: &'static str = "tests/fixtures/";
+const PAGE_NO_META: &'static str = "tests/fixtures/charset_test_html_no_meta.html";
+const IP: &'static str = "0.0.0.0";
+
+#[test]
+fn test_html_charset_not_found() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    // Spawn a single instance of a local http server usable by all tests in this module.
+    let file_dir = format!("{}/{}", output_dir, IP);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+
+    let file_path = fs::read_dir(file_dir)
+        .unwrap()
+        .next()
+        .unwrap()
+        .unwrap()
+        .path(); // There is only one file in the directory
+
+    let data_source = fs::read(PAGE_NO_META).unwrap();
+    let data_downloaded = fs::read(file_path).unwrap();
+
+    assert!(!fixtures::do_vecs_match(&data_source, &data_downloaded));
+}
--- a/tests/charset_http_found.rs
+++ b/tests/charset_http_found.rs
@ -0,0 +1,48 @@
+//! Test for charset detection/conversion
+
+mod fixtures;
+
+use std::fs;
+use std::process::{Command, Stdio};
+
+use lazy_static::lazy_static;
+
+const PAGE: &'static str = "tests/fixtures/";
+const PAGE_NO_META: &'static str = "tests/fixtures/charset_test_html_no_meta.html";
+const IP: &'static str = "0.0.0.0";
+
+lazy_static! {
+    static ref CHARSET_HEADER: Vec<(&'static str, &'static str)> =
+        vec![("Content-Type", "charset=windows-1252")];
+}
+
+#[test]
+fn test_http_charset_found() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, Some(&CHARSET_HEADER));
+    let url = format!("http://{}/charset_test_html_no_meta.html", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let file_dir = format!("{}/{}", output_dir, IP);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+
+    let file_path = fs::read_dir(file_dir)
+        .unwrap()
+        .next()
+        .unwrap()
+        .unwrap()
+        .path(); // There is only one file in the directory
+
+    let data_source = fs::read(PAGE_NO_META).unwrap();
+    let data_downloaded = fs::read(file_path).unwrap();
+
+    assert!(fixtures::do_vecs_match(&data_source, &data_downloaded));
+}
--- a/tests/charset_http_not_found.rs
+++ b/tests/charset_http_not_found.rs
@ -0,0 +1,42 @@
+//! Test for charset detection/conversion
+
+mod fixtures;
+
+use std::fs;
+use std::process::{Command, Stdio};
+
+const PAGE: &'static str = "tests/fixtures/";
+const PAGE_NO_META: &'static str = "tests/fixtures/charset_test_html_no_meta.html";
+const IP: &'static str = "0.0.0.0";
+
+#[test]
+fn test_http_charset_found() {
+    // Spawn a single instance of a local http server usable by all tests in this module.
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let file_dir = format!("{}/{}", output_dir, IP);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+
+    let file_path = fs::read_dir(file_dir)
+        .unwrap()
+        .next()
+        .unwrap()
+        .unwrap()
+        .path(); // There is only one file in the directory
+
+    let data_source = fs::read(PAGE_NO_META).unwrap();
+    let data_downloaded = fs::read(file_path).unwrap();
+
+    assert!(!fixtures::do_vecs_match(&data_source, &data_downloaded));
+}
--- a/tests/external.rs
+++ b/tests/external.rs
@ -0,0 +1,62 @@
+//! Tests for using --ext-depth
+
+mod fixtures;
+
+use fixtures::get_file_count_with_pattern;
+use std::fs::read_dir;
+use std::process::Command;
+use std::process::Stdio;
+use std::sync::Once;
+
+const PAGE: &'static str = "tests/fixtures/";
+const IP: &'static str = "0.0.0.0";
+
+// Test to use include flag for downloading pages only matching the given pattern.
+#[test]
+fn with_external() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let local = format!("{}/{}/", output_dir, IP);
+    let external = format!("{}/{}/", output_dir, "google.com");
+
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir, "-d", "0", "--ext-depth", "1"])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+    let path_local = read_dir(&local).unwrap();
+    let path_external = read_dir(&external).unwrap();
+
+    assert_eq!(path_local.count() + path_external.count(), 2);
+}
+
+#[test]
+fn without_external() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let external = format!("{}/{}/", output_dir, "google.com");
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir, "-d", "0", "--ext-depth", "0"])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+    let path_external = read_dir(&external);
+
+    assert!(path_external.is_err());
+}
--- a/tests/filters.rs
+++ b/tests/filters.rs
@ -8,28 +8,55 @@ use std::process::Command;
 use std::process::Stdio;
 use std::sync::Once;

-const SUCKIT: &'static str = "target/debug/suckit";
-const ADDR: &'static str = "http://0.0.0.0:8000";
+const PAGE: &'static str = "tests/fixtures/";
+const IP: &'static str = "0.0.0.0";
 static START: Once = Once::new();

 #[test]
-fn test_include_exclude() {
-    // Spawn a single instance of a local http server usable by all tests in this module.
-    START.call_once(|| {
-        fixtures::spawn_local_http_server();
-    });
+fn visit_filter_is_download_filter() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);

-    // Tests below are grouped together as they depend on the local_http_server above.
-    include_filter();
-    include_multiple_filters();
-    exclude_filter();
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let files_dir = format!("{}/{}/", output_dir, IP);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[
+            &url,
+            "-o",
+            output_dir,
+            "-v",
+            "-e",
+            "no_download_no_visit.html",
+            "--visit-filter-is-download-filter",
+        ])
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .unwrap();
+
+    let result = cmd.wait_with_output().unwrap();
+    let stdout_str = unsafe { String::from_utf8_unchecked(result.stdout) };
+    assert!(result.status.success());
+
+    let paths = read_dir(&files_dir).unwrap();
+
+    assert!(!stdout_str.contains("should_not_get_visited.html"));
 }

-// Test to use include flag for downloading pages only matching the given pattern.
-fn include_filter() {
-    let output_dir = "w2";
-    let mut cmd = Command::new(SUCKIT)
-        .args(&[ADDR, "-o", "w2", "-i", "mp[3-4]", "-j", "16"])
+// Test to use include flag for visiting pages only matching the given pattern.
+#[test]
+fn visit_include_filter() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let files_dir = format!("{}/{}/", output_dir, IP);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir, "--include-visit", "mp[3-4]"])
        .stdout(Stdio::inherit())
        .stderr(Stdio::inherit())
        .spawn()
@ -37,39 +64,131 @@ fn include_filter() {

    let status = cmd.wait().unwrap();
    assert!(status.success());
-    let paths = read_dir(output_dir).unwrap();
+    let paths = read_dir(&files_dir).unwrap();
+
+    assert_eq!(
+        paths.count() - 1, // minus one because of index.html which is downloaded unconditionally
+        get_file_count_with_pattern(".mp3", &files_dir).unwrap()
+    );
+}
+
+// Test demonstrating usage of multiple include patterns for visiting pages only matching the given pattern.
+#[test]
+fn visit_include_multiple_filters() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let files_dir = format!("{}/{}/", output_dir, IP);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir, "--include-visit", "(mp[3-4])|(txt)"])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+    let paths = read_dir(&files_dir).unwrap();
+    let mp3_count = get_file_count_with_pattern(".mp3", &files_dir).unwrap();
+    let txt_count = get_file_count_with_pattern(".txt", &files_dir).unwrap();
+    assert_eq!(
+        paths.count() - 1, // minus one because of index.html which is downloaded unconditionally
+        mp3_count + txt_count
+    );
+}
+
+// Test to use exclude flag for excluding pages matching the given pattern.
+#[test]
+fn visit_exclude_filter() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let files_dir = format!("{}/{}/", output_dir, IP);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir, "--exclude-visit", "jpe?g"])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+
+    let jpeg_count = get_file_count_with_pattern(".jpe?g", &files_dir).unwrap();
+    assert_eq!(jpeg_count, 0);
+}
+
+// Test to use include flag for downloading pages only matching the given pattern.
+#[test]
+fn download_include_filter() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let files_dir = format!("{}/{}/", output_dir, IP);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir, "-i", "mp[3-4]"])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .unwrap();
+
+    let status = cmd.wait().unwrap();
+    assert!(status.success());
+    let paths = read_dir(&files_dir).unwrap();
+
    assert_eq!(
        paths.count(),
-        get_file_count_with_pattern("*_mp3", output_dir).unwrap()
+        get_file_count_with_pattern(".mp3", &files_dir).unwrap()
    );

    std::fs::remove_dir_all(output_dir).unwrap();
 }

 // Test demonstrating usage of multiple include patterns for downloading pages only matching the given pattern.
-fn include_multiple_filters() {
-    let output_dir = "w1";
-    let mut cmd = Command::new(SUCKIT)
-        .args(&[ADDR, "-o", output_dir, "-i", "(mp[3-4])|(txt)", "-j", "16"])
+#[test]
+fn download_include_multiple_filters() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let files_dir = format!("{}/{}/", output_dir, IP);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir, "-i", "(mp[3-4])|(txt)"])
        .stdout(Stdio::inherit())
        .stderr(Stdio::inherit())
        .spawn()
        .unwrap();
    let status = cmd.wait().unwrap();
    assert!(status.success());
-    let paths = read_dir(output_dir).unwrap();
-    let mp3_count = get_file_count_with_pattern("*_mp3", output_dir).unwrap();
-    let txt_count = get_file_count_with_pattern("*_txt", output_dir).unwrap();
-    assert_eq!(paths.count(), mp3_count + txt_count);

-    std::fs::remove_dir_all(output_dir).unwrap();
+    let paths = read_dir(&files_dir).unwrap();
+    let mp3_count = get_file_count_with_pattern(".mp3", &files_dir).unwrap();
+    let txt_count = get_file_count_with_pattern(".txt", &files_dir).unwrap();
+    assert_eq!(paths.count(), mp3_count + txt_count);
 }

 // Test to use exclude flag for excluding pages matching the given pattern.
-fn exclude_filter() {
-    let output_dir = "w3";
-    let mut cmd = Command::new(SUCKIT)
-        .args(&[ADDR, "-o", output_dir, "-e", "jpe?g", "-j", "16"])
+#[test]
+fn download_exclude_filter() {
+    let ip = fixtures::spawn_local_http_server(PAGE, false, None);
+    let url = format!("http://{}", ip);
+
+    let tempdir = mktemp::Temp::new_dir().unwrap();
+    let output_dir = tempdir.to_str().unwrap();
+
+    let files_dir = format!("{}/{}/", output_dir, IP);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_suckit"))
+        .args(&[&url, "-o", output_dir, "-e", "jpe?g"])
        .stdout(Stdio::inherit())
        .stderr(Stdio::inherit())
        .spawn()
@ -77,11 +196,7 @@ fn exclude_filter() {

    let status = cmd.wait().unwrap();
    assert!(status.success());
-    let paths = read_dir(output_dir).unwrap();
-    let mp3_count = get_file_count_with_pattern("*_mp3", output_dir).unwrap();
-    let txt_count = get_file_count_with_pattern("*_txt", output_dir).unwrap();
-    let index_file = 1;
-    assert_eq!(paths.count(), mp3_count + txt_count + index_file);
-
-    std::fs::remove_dir_all(output_dir).unwrap();
+    let paths = read_dir(&files_dir).unwrap();
+    let jpeg_count = get_file_count_with_pattern(".jpe?g", &files_dir).unwrap();
+    assert_eq!(jpeg_count, 0);
 }
--- a/tests/fixtures/charset_test_html.html
+++ b/tests/fixtures/charset_test_html.html
@ -0,0 +1,13 @@
+<!-- This file is encoded in 'windows-1252' charset with no newline at the end.
+     Be careful when modifying it. To save it correctly on:
+         - vim: :write ++enc=windows-1252
+     This file should not have a newline. Don't forget to run ``truncate -s -1``
+     in case there is one.
+--><html><head>
+<meta content="text/html; charset=windows-1252" http-equiv="Content-Type">
+<title>Gamle Gjerpen - Ei Bygdebok for nett.</title>
+</head>
+
+<body>
+    <p>Gamle Valebø Redigering av sidene pågår.</p>
+</body></html>
--- a/tests/fixtures/charset_test_html_no_meta.html
+++ b/tests/fixtures/charset_test_html_no_meta.html
@ -0,0 +1,12 @@
+<!-- This file is encoded in 'windows-1252' charset with no newline at the end.
+     Be careful when modifying it. To save it correctly on:
+         - vim: :write ++enc=windows-1252
+     This file should not have a newline. Don't forget to run ``truncate -s -1``
+     in case there is one.
+--><html><head>
+<title>Gamle Gjerpen - Ei Bygdebok for nett.</title>
+</head>
+
+<body>
+    <p>Gamle Valebø Redigering av sidene pågår.</p>
+</body></html>
--- a/tests/fixtures/index.html
+++ b/tests/fixtures/index.html
@ -9,5 +9,7 @@
    <a href="jpeg.jpg" download>JPG File</a>
    <a href="mp3.mp3" download="">MP3 File</a>
    <a href="file.txt" download="">Text File</a>
+    <a href="https://google.com" download="">Google</a>
+    <a href="no_download_no_visit.html">No download</a>
 </body>
 </html>
--- a/tests/fixtures/mod.rs
+++ b/tests/fixtures/mod.rs
@ -1,56 +1,95 @@
 use std::fs::File;
-use std::process::Command;
-use std::process::Stdio;
 use std::thread;
-use tiny_http::{Response, Server};

-const PAGE: &'static str = "tests/fixtures/index.html";
+use portpicker;
+use subprocess::Exec;
+use tiny_http::{Header, Response, Server};

-pub fn spawn_local_http_server() {
-    let server = Server::http("0.0.0.0:8000").unwrap();
-    println!("Spawning http server");
+const AUTH_HEADER: &str = "Authorization";
+const AUTH_CREDENTIALS: &str = "Basic dXNlcm5hbWU6cGFzc3dvcmQ="; // base64-encoded "username:password"
+
+pub fn spawn_local_http_server(
+    page: &'static str,
+    requires_auth: bool,
+    headers: Option<&'static Vec<(&'static str, &'static str)>>,
+) -> String {
+    let port = portpicker::pick_unused_port().unwrap();
+    let addr = format!("0.0.0.0:{}", port);
+    let server = Server::http(&addr).unwrap();
    thread::spawn(move || {
        for request in server.incoming_requests() {
-            let response = Response::from_file(File::open(PAGE).unwrap());
+            // Authenticate request from headers if provided
+            let auth_header = request
+                .headers()
+                .iter()
+                .find(|h| h.field.equiv(AUTH_HEADER));
+            let valid_auth = check_auth_credentials(auth_header);
+
+            let mut response = if requires_auth && !valid_auth {
+                let mut response = Response::from_string("Invalid auth").with_status_code(401);
+                let h = Header::from_bytes("WWW-Authenticate", r#"Basic realm="Test""#).unwrap();
+                response.add_header(h);
+                response.boxed()
+            } else {
+                let file = match request.url() {
+                    "/" => format!("{}{}", page, "index.html"),
+                    other => format!("{}{}", page, other),
+                };
+                // panic!("File = {}", file);
+                Response::from_file(File::open(file).unwrap()).boxed()
+            };
+
+            match headers {
+                Some(vec) => {
+                    let mut key_vec: Vec<u8> = vec![];
+                    let mut value_vec: Vec<u8> = vec![];
+                    for (key, value) in vec {
+                        key_vec.extend_from_slice(key.as_bytes());
+                        value_vec.extend_from_slice(value.as_bytes());
+                    }
+
+                    let h = Header::from_bytes(key_vec, value_vec).unwrap();
+                    response.add_header(h);
+                }
+                _ => (),
+            }
+
            request.respond(response).unwrap();
        }
    });
+
+    return addr;
+}
+
+fn check_auth_credentials(auth_header: Option<&Header>) -> bool {
+    match auth_header {
+        None => false,
+        Some(header) => header.value.as_str() == AUTH_CREDENTIALS,
+    }
 }

 pub fn get_file_count_with_pattern(pattern: &str, dir: &str) -> Result<usize, ()> {
-    // Command being run: `ls | grep .mp3 | wc -w`
-    let mut du_output_child = Command::new("ls")
-        .args(&[dir])
-        .stdout(Stdio::piped())
-        .spawn()
-        .unwrap();
+    // Command being run: `ls | grep pattern | wc -w`

-    if let Some(du_output) = du_output_child.stdout.take() {
-        let mut sort_output_child = Command::new("egrep")
-            .arg(pattern)
-            .stdin(du_output)
-            .stdout(Stdio::piped())
-            .spawn()
-            .unwrap();
-
-        du_output_child.wait().unwrap();
-
-        if let Some(sort_output) = sort_output_child.stdout.take() {
-            let head_output_child = Command::new("wc")
-                .args(&["-w"])
-                .stdin(sort_output)
-                .stdout(Stdio::piped())
-                .spawn()
-                .unwrap();
-
-            let head_stdout = head_output_child.wait_with_output().unwrap();
-            sort_output_child.wait().unwrap();
-            return Ok(String::from_utf8(head_stdout.stdout)
-                .unwrap()
-                .trim()
-                .parse()
-                .unwrap());
-        }
+    let cmd = {
+        // Pipe ('|') is overloaded here and does a real pipe
+        Exec::shell(format!("ls {}", dir))
+            | Exec::shell(format!("grep '{}'", pattern))
+            | Exec::shell("wc -l")
+    }
+    .capture();
+
+    match cmd {
+        Ok(capture_data) => {
+            let stdout = capture_data.stdout_str();
+            let count = stdout.trim().parse::<usize>().unwrap();
+            Ok(count)
+        }
+        _ => Err(()),
    }
-    Err(())
+}
+
+pub fn do_vecs_match<T: PartialEq>(a: &Vec<T>, b: &Vec<T>) -> bool {
+    let matching = a.iter().zip(b.iter()).filter(|&(a, b)| a == b).count();
+    matching == a.len() && matching == b.len()
 }
--- a/tests/fixtures/no_download_no_visit.html
+++ b/tests/fixtures/no_download_no_visit.html
@ -0,0 +1,8 @@
+<!DOCTYPE html>
+<html lang="en">
+<head></head>
+
+<body>
+  <a href="should_not_get_visited.html">Link</a>
+</body>
+</html>
--- a/tests/fixtures/should_not_get_visited.html
+++ b/tests/fixtures/should_not_get_visited.html
--- a/tests/local_server_setup.sh
+++ b/tests/local_server_setup.sh
@ -0,0 +1,33 @@
+#!/bin/sh
+
+set -e
+
+# Create the local's server directory if necessary
+if [[ ! -d local_server ]]; then
+    mkdir local_server
+fi
+
+cd local_server
+
+# Clone repository if necessary
+if [[ ! -d linux ]]; then
+    # Clone a big repository
+    git clone https://github.com/torvalds/linux
+
+    cd linux
+
+    # Checkout the 5.9 release
+    git checkout v5.9
+
+    # No need for the .git directory. It creates a ton of files that take too long
+    # to scrape
+    rm -rf .git
+
+    # Get back to local_server
+    cd ..
+fi
+
+printf "\nWEBSERVER UP\n"
+
+# Start up the local python server
+python3 -m http.server
--- a/tests/speed.csv
+++ b/tests/speed.csv
@ -1,16 +1 @@
 1, 1, 1
-92.01026630401611, 45.85903334617615, 24.938209056854248
-92.68794660568237, 45.51665663719177, 36.877896785736084
-94.26381759643554, 48.01623530387879, 39.034280157089235
-94.51657667160035, 47.43879041671753, 25.296046495437622
-94.91624474525452, 47.318924236297605, 38.720614719390866
-94.61654992103577, 57.59067335128784, 24.995196104049683
-95.28456358909607, 46.86330976486206, 24.708560609817503
-77.68792352676391, 39.08626494407654, 19.765260219573975
-67.09948434829712, 34.11661868095398, 17.574066400527954
-68.2605634689331, 33.728125953674315, 17.433270835876463
-70.19220662117004, 41.964057207107544, 17.660782480239867
-68.06500535011291, 34.10616636276245, 28.643333148956298
-67.92092838287354, 33.87600998878479, 17.644397974014282
-67.94186737537385, 33.967066860198976, 17.586638736724854
-68.20248901844025, 37.70992383956909, 22.916122150421142
--- a/tests/speed_regression.py
+++ b/tests/speed_regression.py
@ -1,7 +1,7 @@
 #!/usr/bin/python3

 # Number of tests for each bench-set
-TEST_RETRIES = 10
+TEST_RETRIES = 20

 # File to store the results
 FILENAME = "speed.csv"
@ -9,8 +9,8 @@ FILENAME = "speed.csv"
 # Path to the suckit binary
 SUCKIT = "suckit"

-# URL to download
-URL = "http://books.toscrape.com"
+# URL to download: localhost
+URL = "http://0.0.0.0:8000"

 # Path to store the downloaded data
 PATH = "/tmp/suckit_speed"
@ -23,6 +23,18 @@ import subprocess
 import time
 from termcolor import colored

+def start_webserver():
+    print("Launching webserver")
+
+    webserver_pid = subprocess.Popen(["./local_server_setup.sh"], stdout = subprocess.PIPE)
+
+    while webserver_pid.stdout.readline() != b"WEBSERVER UP\n":
+        print("Waiting on webserver...", end = "\r")
+
+    print("Webserver launched")
+
+    return webserver_pid
+
 def parse_args():
    global FILENAME
    global SUCKIT
@ -120,4 +132,14 @@ def main():
    shutil.rmtree(PATH)

 if __name__ == "__main__":
-    main()
+    webserver_pid = None
+
+    try:
+        webserver_pid = start_webserver()
+
+        main()
+
+        # Terminate the webserver
+        webserver_pid.kill()
+    except: # Kill the webserver if an exception occurs
+        webserver_pid.kill()