scraping/skallwar.suckit
1
0
Fork 0
mirror of https://github.com/Skallwar/suckit.git synced 2026-09-10 07:16:19 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

test: Remove unit test that need to be functional tests

Browse source
This commit is contained in:
Esteban Blanc 2020-04-29 12:07:30 +02:00
parent 4115ba870d
commit 93c5241bf0
6 changed files with 32 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/args.rs
View file

@ -1,13 +1,13 @@
use std::path::PathBuf;
use reqwest::Url;
use structopt::StructOpt;
use url::Url;
use super::url;
use super::url_helper;
#[derive(Debug, StructOpt)]
pub struct Args {
#[structopt(name = "url", parse(try_from_str = url::str_to_url))]
#[structopt(name = "url", parse(try_from_str = url_helper::str_to_url))]
pub origin: Url,
#[structopt(short, long, parse(from_os_str))]

21
src/disk.rs
View file

@ -2,8 +2,6 @@ use std::fs;
use std::io::Write;
use std::path::PathBuf;
use reqwest::Url;
//TODO: Recover insted of panic
pub fn save_file(file_name: &String, content: &[u8], path: &Option<PathBuf>) {
let path = match path {
@ -38,22 +36,3 @@ pub fn symlink(source: &String, destination: &String, path: &Option<PathBuf>) {
std::os::unix::fs::symlink(source, destination).unwrap();
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn url_to_path() {
let str = super::url_to_path(&Url::parse("https://lwn.net/Kernel/").unwrap());
assert_eq!(str, "lwn_net_Kernel");
}
#[test]
fn url_to_path_long() {
let str = super::url_to_path(&Url::parse("https://e8v0pez1lofdxoxgg5vwrnaqkjuvpowp9wtgc2eknlfpjdwmmfti8fcwyjzfdgys3nrgyqyeqjkulpyg9kfiqajza2bwxkinhhpohyrnnoy2bak374tcaxh1ycpboolmx8so9yq9kbcj5wu5cgymqndeqasdak0nvl0ijka6fkkmhhvt43l73bn38rewicd4h1ff2omhpni752jtqyzsjub5coh8dlnr3i35udmkzhxo4db3is9gnqmf3hl.comtest").unwrap());
assert_eq!(str, "e8v0pez1lofdxoxgg5vwrnaqkjuvpowp9wtgc2eknlfpjdwmmfti8fcwyjzfdgys3nrgyqyeqjkulpyg9kfiqajza2bwxkinhhpohyrnnoy2bak374tcaxh1ycpboolmx8so9yq9kbcj5wu5cgymqndeqasdak0nvl0ijka6fkkmhhvt43l73bn38rewicd4h1ff2omhpni752jtqyzsjub5coh8dlnr3i35udmkzhxo4db3is9gnqmf3hl_com");
}
}

57
src/downloader.rs
View file

@ -1,4 +1,4 @@
use reqwest::Url;
use url::Url;
/// Wrapper around a reqwest client, used to get the content of web pages
pub struct Downloader {
@ -116,59 +116,4 @@ mod tests {
_ => {}
}
}
#[test]
fn test_url_content() {
let url: Url = Url::parse("https://example.com").unwrap();
match Downloader::new(1).get(&url) {
Err(e) => assert!(false, "Fail to download example.com: {:?}", e),
Ok(content) => assert_eq!(content,
"<!doctype html>
<html>
<head>
<title>Example Domain</title>
<meta charset=\"utf-8\" />
<meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\" />
<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />
<style type=\"text/css\">
body {
background-color: #f0f0f2;
margin: 0;
padding: 0;
font-family: -apple-system, system-ui, BlinkMacSystemFont, \"Segoe UI\", \"Open Sans\", \"Helvetica Neue\", Helvetica, Arial, sans-serif;
}
div {
width: 600px;
margin: 5em auto;
padding: 2em;
background-color: #fdfdff;
border-radius: 0.5em;
box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
}
a:link, a:visited {
color: #38488f;
text-decoration: none;
}
@media (max-width: 700px) {
div {
margin: 0 auto;
width: auto;
}
}
</style>
</head>
<body>
<div>
<h1>Example Domain</h1>
<p>This domain is for use in illustrative examples in documents. You may use this
domain in literature without prior coordination or asking for permission.</p>
<p><a href=\"https://www.iana.org/domains/example\">More information...</a></p>
</div>
</body>
</html>\n"),
}
}
}

2
src/main.rs
View file

@ -4,7 +4,7 @@ mod dom;
mod downloader;
mod logger;
mod scraper;
mod url;
mod url_helper;
use scraper::Scraper;

127
src/scraper.rs
View file

@ -1,19 +1,18 @@
use crossbeam::channel::{Receiver, Sender, TryRecvError};
use crossbeam::thread;
use reqwest::Url;
use url::Url;
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Mutex;
use std::time;
#[cfg(not(test))] //For the "mock" at the end of file
use super::downloader;
use super::args;
use super::disk;
use super::dom;
use super::url;
use super::url_helper;
use crate::info;
@ -74,7 +73,7 @@ impl Scraper {
/// Fix the URLs contained in the DOM-tree so they point to each other
fn fix_domtree(&self, old_url_str: &mut String, new_url: &Url) {
let path_map = self.path_map.lock().unwrap();
let new_url_str = url::str_percent_encode(path_map.get(new_url.as_str()).unwrap());
let new_url_str = url_helper::str_percent_encode(path_map.get(new_url.as_str()).unwrap());
old_url_str.clear();
old_url_str.push_str(&new_url_str);
@ -94,7 +93,7 @@ impl Scraper {
.filter(|candidate| Scraper::should_visit(candidate, &url))
.for_each(|next_url| {
let next_full_url = url.join(&next_url).unwrap();
match scraper.map_url(&next_full_url, url::url_to_path(&next_full_url)) {
match scraper.map_url(&next_full_url, url_helper::url_to_path(&next_full_url)) {
true => {
if depth < scraper.args.depth {
Scraper::push(transmitter, next_full_url.clone(), depth + 1);
@ -151,7 +150,10 @@ impl Scraper {
/// Run through the channel and complete it
pub fn run(&mut self) {
/* Push the origin URL and depth (0) through the channel */
self.map_url(&self.args.origin, url::url_to_path(&self.args.origin));
self.map_url(
&self.args.origin,
url_helper::url_to_path(&self.args.origin),
);
Scraper::push(&self.transmitter, self.args.origin.clone(), 0);
thread::scope(|thread_scope| {
@ -219,117 +221,4 @@ mod tests {
let _ = Scraper::new(args);
}
#[test]
fn run() {
let args = args::Args {
origin: Url::parse("https://fake_start.net/").unwrap(),
output: Some(PathBuf::from("/tmp")),
jobs: 1,
tries: 1,
depth: 5,
verbose: true,
};
let mut s = Scraper::new(args);
s.run();
let visited_urls = s.visited_urls.lock().unwrap();
assert!(!visited_urls.contains("https://example.net"));
assert!(!visited_urls.contains("https://no-no-no.com"));
assert!(visited_urls.contains("https://fake_start.net/a_file"));
assert!(visited_urls.contains("https://fake_start.net/dir/nested/file"));
assert!(visited_urls.contains("https://fake_start.net/an_answer_file"));
}
#[test]
fn depth() {
let args = args::Args {
origin: Url::parse("https://fake_start.net/").unwrap(),
output: Some(PathBuf::from("/tmp")),
jobs: 1,
tries: 1,
depth: 0,
verbose: true,
};
let mut s = Scraper::new(args);
s.run();
let visited_urls = s.visited_urls.lock().unwrap();
assert!(!visited_urls.contains("https://example.net"));
assert!(!visited_urls.contains("https://no-no-no.com"));
assert!(!visited_urls.contains("https://fake_start.net/a_file"));
assert!(!visited_urls.contains("https://fake_start.net/an_answer_file"));
assert!(!visited_urls.contains("https://fake_start.net/dir/nested/file"));
}
#[test]
fn depth_tricky() {
let args = args::Args {
origin: Url::parse("https://fake_start.net/").unwrap(),
output: Some(PathBuf::from("/tmp")),
jobs: 1,
tries: 1,
depth: 1,
verbose: true,
};
let mut s = Scraper::new(args);
s.run();
let visited_urls = s.visited_urls.lock().unwrap();
assert!(visited_urls.contains("https://fake_start.net/a_file"));
assert!(visited_urls.contains("https://fake_start.net/dir/nested/file"));
assert!(!visited_urls.contains("https://fake_start.net/an_answer_file"));
}
}
#[cfg(test)]
mod downloader {
static TEST_BEG: &str = "<!DOCTYPE html>
<html>
<body>
<p>Absolute<a href=\"https://no-no-no.com\"></a></p>
<p>Relative<a href=\"a_file\"></a></p>
<p>Relative nested<a href=\"dir/nested/file\"></a></p>
</body>
</html>
";
static TEST_ANS: &str = "<!DOCTYPE html>
<html>
<body>
<p>Relative<a href=\"an_answer_file\"></a></p>
</body>
</html>
";
pub struct Downloader {}
impl Downloader {
pub fn new(_tries: usize) -> Downloader {
Downloader {}
}
pub fn get(&self, url: &reqwest::Url) -> Result<String, reqwest::Error> {
let mut res = String::from("");
match url.as_str() == "https://fake_start.net/" {
true => res = String::from(TEST_BEG),
false => {}
}
match url.as_str() == "https://fake_start.net/a_file" {
true => res = String::from(TEST_ANS),
false => {}
}
Ok(res)
}
}
}

19
src/url.rs → src/url_helper.rs
View file

@ -27,3 +27,22 @@ pub fn url_to_path(url: &Url) -> String {
return url.to_string();
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn url_to_path() {
let str = super::url_to_path(&Url::parse("https://lwn.net/Kernel/").unwrap());
assert_eq!(str, "lwn_net_Kernel");
}
#[test]
fn url_to_path_long() {
let str = super::url_to_path(&Url::parse("https://e8v0pez1lofdxoxgg5vwrnaqkjuvpowp9wtgc2eknlfpjdwmmfti8fcwyjzfdgys3nrgyqyeqjkulpyg9kfiqajza2bwxkinhhpohyrnnoy2bak374tcaxh1ycpboolmx8so9yq9kbcj5wu5cgymqndeqasdak0nvl0ijka6fkkmhhvt43l73bn38rewicd4h1ff2omhpni752jtqyzsjub5coh8dlnr3i35udmkzhxo4db3is9gnqmf3hl.comtest").unwrap());
assert_eq!(str, "e8v0pez1lofdxoxgg5vwrnaqkjuvpowp9wtgc2eknlfpjdwmmfti8fcwyjzfdgys3nrgyqyeqjkulpyg9kfiqajza2bwxkinhhpohyrnnoy2bak374tcaxh1ycpboolmx8so9yq9kbcj5wu5cgymqndeqasdak0nvl0ijka6fkkmhhvt43l73bn38rewicd4h1ff2omhpni752jtqyzsjub5coh8dlnr3i35udmkzhxo4db3is9gnqmf3hl_com");
}
}