Merge branch 'master' into add-logo-readme

This commit is contained in:
Esteban Blanc 2020-04-29 22:50:33 +02:00 committed by GitHub
commit 1264745379
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 51 additions and 56 deletions

View file

@ -3,18 +3,17 @@ use std::io::Write;
use std::path::PathBuf;
//TODO: Recover insted of panic
pub fn save_file(file_name: &String, content: &[u8], path: &Option<PathBuf>) {
pub fn save_file(file_name: &str, content: &[u8], path: &Option<PathBuf>) {
let path = match path {
Some(path) => path.join(file_name),
None => PathBuf::from(file_name),
};
match path.parent() {
Some(parent) => match fs::create_dir_all(parent) {
if let Some(parent) = path.parent() {
match fs::create_dir_all(parent) {
Err(err) => panic!("Couldn't create folder {}: {}", parent.display(), err),
Ok(()) => (),
},
None => (),
}
}
let mut file = match fs::File::create(&path) {
@ -22,13 +21,12 @@ pub fn save_file(file_name: &String, content: &[u8], path: &Option<PathBuf>) {
Ok(file) => file,
};
match file.write_all(content) {
Err(err) => panic!("Couldn't write to {}: {}", path.display(), err),
Ok(_) => (),
};
if let Err(err) = file.write_all(content) {
panic!("Couldn't write to {}: {}", path.display(), err);
}
}
pub fn symlink(source: &String, destination: &String, path: &Option<PathBuf>) {
pub fn symlink(source: &str, destination: &str, path: &Option<PathBuf>) {
let destination = match path {
Some(path) => path.join(destination),
None => PathBuf::from(destination),

View file

@ -17,12 +17,11 @@ impl Dom {
pub fn serialize(&self) -> String {
let mut vec: Vec<u8> = Vec::new();
match self.tree.serialize(&mut vec) {
Err(err) => panic!("Couldn't serialize domtree: {}", err),
Ok(_) => (),
if let Err(err) = self.tree.serialize(&mut vec) {
panic!("Couldn't serialize domtree: {}", err)
}
return String::from_utf8(vec).unwrap();
String::from_utf8(vec).unwrap()
}
pub fn find_urls_as_strings(&self) -> Vec<&mut String> {
@ -39,14 +38,15 @@ impl Dom {
//TODO: Prettify this, we may need more than src and href in the futur
match unsafe { (*attributes).get_mut("src") } {
Some(url) => vec.push(url),
None => match unsafe { (*attributes).get_mut("href") } {
Some(url) => vec.push(url),
None => (),
},
None => {
if let Some(url) = unsafe { (*attributes).get_mut("href") } {
vec.push(url)
}
}
}
}
return vec;
vec
}
}

View file

@ -24,7 +24,7 @@ impl Downloader {
.cookie_store(true)
.build()
.unwrap(),
tries: tries,
tries,
}
}
@ -32,9 +32,9 @@ impl Downloader {
content_type.contains("text/html")
}
fn get_filename(content_disposition: &String) -> String {
let content_disposition = content_disposition.clone();
let index = content_disposition.find("=").unwrap() + 1;
fn get_filename(content_disposition: &str) -> String {
let content_disposition = content_disposition.to_string();
let index = content_disposition.find('=').unwrap() + 1;
content_disposition[index..].to_string()
}
@ -63,17 +63,17 @@ impl Downloader {
}
};
let data = match Downloader::is_html(&data_type) {
true => ResponseData::Html(data.text().unwrap()),
false => {
let mut raw_data: Vec<u8> = Vec::new();
data.copy_to(&mut raw_data).unwrap();
ResponseData::Other(raw_data)
}
let data = if Downloader::is_html(&data_type) {
ResponseData::Html(data.text().unwrap())
} else {
let mut raw_data: Vec<u8> = Vec::new();
data.copy_to(&mut raw_data).unwrap();
ResponseData::Other(raw_data)
};
return Ok(Response::new(data, filename));
}
Err(e) => {
println!("Downloader.get() has encountered an error: {}", e);
error = Some(e);
@ -81,16 +81,13 @@ impl Downloader {
};
}
return Err(error.unwrap());
Err(error.unwrap())
}
}
impl Response {
pub fn new(data: ResponseData, filename: Option<String>) -> Response {
Response {
data: data,
filename: filename,
}
Response { data, filename }
}
pub fn get_data(&self) -> &ResponseData {

View file

@ -1,10 +1,9 @@
use colored::*;
use chrono::Local;
use colored::*;
/// Write a message using the following format
/// <time>: [<header>] <message>
pub struct Logger {
}
pub struct Logger {}
// Allow the dead code in case we don't use all the macros. Using it here means
// it doesn't affect the other modules in the crate
@ -16,7 +15,7 @@ impl Logger {
}
/// Write a log message to stdout
fn write_log(header: ColoredString, message: &String) {
fn write_log(header: ColoredString, message: &str) {
// Sadly we can't use a static format litteral so we have to retype
// this for every function...
println!("{}: [{}] {}", Logger::get_timestamp(), header, message);
@ -34,7 +33,12 @@ impl Logger {
/// Display an ERROR message
pub fn error(message: String) {
eprintln!("{}: [{}] {}", Logger::get_timestamp(), "ERROR".red(), message);
eprintln!(
"{}: [{}] {}",
Logger::get_timestamp(),
"ERROR".red(),
message
);
}
}

View file

@ -41,7 +41,7 @@ impl Scraper {
Scraper {
downloader: downloader::Downloader::new(args.tries),
args: args,
args,
transmitter: tx,
receiver: rx,
visited_urls: Mutex::new(HashSet::new()),
@ -53,12 +53,11 @@ impl Scraper {
fn map_url(&self, url: &Url, path: String) -> bool {
let mut path_map = self.path_map.lock().unwrap();
match path_map.contains_key(url.as_str()) {
false => {
path_map.insert(url.to_string(), path);
true
}
true => false,
if !path_map.contains_key(url.as_str()) {
path_map.insert(url.to_string(), path);
true
} else {
false
}
}
@ -93,14 +92,11 @@ impl Scraper {
.filter(|candidate| Scraper::should_visit(candidate, &url))
.for_each(|next_url| {
let next_full_url = url.join(&next_url).unwrap();
match scraper.map_url(&next_full_url, url_helper::url_to_path(&next_full_url)) {
true => {
if depth < scraper.args.depth {
Scraper::push(transmitter, next_full_url.clone(), depth + 1);
}
}
false => (),
};
if scraper.map_url(&next_full_url, url_helper::url_to_path(&next_full_url))
&& depth < scraper.args.depth
{
Scraper::push(transmitter, next_full_url.clone(), depth + 1);
}
scraper.fix_domtree(next_url, &next_full_url);
});

View file

@ -25,7 +25,7 @@ pub fn url_to_path(url: &Url) -> String {
}
let url = url.trim_end_matches('_'); //Remaining '/'
return url.to_string();
url.to_string()
}
#[cfg(test)]