mirror of
https://github.com/Skallwar/suckit.git
synced 2026-09-10 07:16:19 -04:00
Merge branch 'master' into add-logo-readme
This commit is contained in:
commit
1264745379
18
src/disk.rs
18
src/disk.rs
|
|
@ -3,18 +3,17 @@ use std::io::Write;
|
|||
use std::path::PathBuf;
|
||||
|
||||
//TODO: Recover insted of panic
|
||||
pub fn save_file(file_name: &String, content: &[u8], path: &Option<PathBuf>) {
|
||||
pub fn save_file(file_name: &str, content: &[u8], path: &Option<PathBuf>) {
|
||||
let path = match path {
|
||||
Some(path) => path.join(file_name),
|
||||
None => PathBuf::from(file_name),
|
||||
};
|
||||
|
||||
match path.parent() {
|
||||
Some(parent) => match fs::create_dir_all(parent) {
|
||||
if let Some(parent) = path.parent() {
|
||||
match fs::create_dir_all(parent) {
|
||||
Err(err) => panic!("Couldn't create folder {}: {}", parent.display(), err),
|
||||
Ok(()) => (),
|
||||
},
|
||||
None => (),
|
||||
}
|
||||
}
|
||||
|
||||
let mut file = match fs::File::create(&path) {
|
||||
|
|
@ -22,13 +21,12 @@ pub fn save_file(file_name: &String, content: &[u8], path: &Option<PathBuf>) {
|
|||
Ok(file) => file,
|
||||
};
|
||||
|
||||
match file.write_all(content) {
|
||||
Err(err) => panic!("Couldn't write to {}: {}", path.display(), err),
|
||||
Ok(_) => (),
|
||||
};
|
||||
if let Err(err) = file.write_all(content) {
|
||||
panic!("Couldn't write to {}: {}", path.display(), err);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn symlink(source: &String, destination: &String, path: &Option<PathBuf>) {
|
||||
pub fn symlink(source: &str, destination: &str, path: &Option<PathBuf>) {
|
||||
let destination = match path {
|
||||
Some(path) => path.join(destination),
|
||||
None => PathBuf::from(destination),
|
||||
|
|
|
|||
18
src/dom.rs
18
src/dom.rs
|
|
@ -17,12 +17,11 @@ impl Dom {
|
|||
pub fn serialize(&self) -> String {
|
||||
let mut vec: Vec<u8> = Vec::new();
|
||||
|
||||
match self.tree.serialize(&mut vec) {
|
||||
Err(err) => panic!("Couldn't serialize domtree: {}", err),
|
||||
Ok(_) => (),
|
||||
if let Err(err) = self.tree.serialize(&mut vec) {
|
||||
panic!("Couldn't serialize domtree: {}", err)
|
||||
}
|
||||
|
||||
return String::from_utf8(vec).unwrap();
|
||||
String::from_utf8(vec).unwrap()
|
||||
}
|
||||
|
||||
pub fn find_urls_as_strings(&self) -> Vec<&mut String> {
|
||||
|
|
@ -39,14 +38,15 @@ impl Dom {
|
|||
//TODO: Prettify this, we may need more than src and href in the futur
|
||||
match unsafe { (*attributes).get_mut("src") } {
|
||||
Some(url) => vec.push(url),
|
||||
None => match unsafe { (*attributes).get_mut("href") } {
|
||||
Some(url) => vec.push(url),
|
||||
None => (),
|
||||
},
|
||||
None => {
|
||||
if let Some(url) = unsafe { (*attributes).get_mut("href") } {
|
||||
vec.push(url)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return vec;
|
||||
vec
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ impl Downloader {
|
|||
.cookie_store(true)
|
||||
.build()
|
||||
.unwrap(),
|
||||
tries: tries,
|
||||
tries,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -32,9 +32,9 @@ impl Downloader {
|
|||
content_type.contains("text/html")
|
||||
}
|
||||
|
||||
fn get_filename(content_disposition: &String) -> String {
|
||||
let content_disposition = content_disposition.clone();
|
||||
let index = content_disposition.find("=").unwrap() + 1;
|
||||
fn get_filename(content_disposition: &str) -> String {
|
||||
let content_disposition = content_disposition.to_string();
|
||||
let index = content_disposition.find('=').unwrap() + 1;
|
||||
|
||||
content_disposition[index..].to_string()
|
||||
}
|
||||
|
|
@ -63,17 +63,17 @@ impl Downloader {
|
|||
}
|
||||
};
|
||||
|
||||
let data = match Downloader::is_html(&data_type) {
|
||||
true => ResponseData::Html(data.text().unwrap()),
|
||||
false => {
|
||||
let mut raw_data: Vec<u8> = Vec::new();
|
||||
data.copy_to(&mut raw_data).unwrap();
|
||||
ResponseData::Other(raw_data)
|
||||
}
|
||||
let data = if Downloader::is_html(&data_type) {
|
||||
ResponseData::Html(data.text().unwrap())
|
||||
} else {
|
||||
let mut raw_data: Vec<u8> = Vec::new();
|
||||
data.copy_to(&mut raw_data).unwrap();
|
||||
ResponseData::Other(raw_data)
|
||||
};
|
||||
|
||||
return Ok(Response::new(data, filename));
|
||||
}
|
||||
|
||||
Err(e) => {
|
||||
println!("Downloader.get() has encountered an error: {}", e);
|
||||
error = Some(e);
|
||||
|
|
@ -81,16 +81,13 @@ impl Downloader {
|
|||
};
|
||||
}
|
||||
|
||||
return Err(error.unwrap());
|
||||
Err(error.unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
impl Response {
|
||||
pub fn new(data: ResponseData, filename: Option<String>) -> Response {
|
||||
Response {
|
||||
data: data,
|
||||
filename: filename,
|
||||
}
|
||||
Response { data, filename }
|
||||
}
|
||||
|
||||
pub fn get_data(&self) -> &ResponseData {
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
use colored::*;
|
||||
use chrono::Local;
|
||||
use colored::*;
|
||||
|
||||
/// Write a message using the following format
|
||||
/// <time>: [<header>] <message>
|
||||
pub struct Logger {
|
||||
}
|
||||
pub struct Logger {}
|
||||
|
||||
// Allow the dead code in case we don't use all the macros. Using it here means
|
||||
// it doesn't affect the other modules in the crate
|
||||
|
|
@ -16,7 +15,7 @@ impl Logger {
|
|||
}
|
||||
|
||||
/// Write a log message to stdout
|
||||
fn write_log(header: ColoredString, message: &String) {
|
||||
fn write_log(header: ColoredString, message: &str) {
|
||||
// Sadly we can't use a static format litteral so we have to retype
|
||||
// this for every function...
|
||||
println!("{}: [{}] {}", Logger::get_timestamp(), header, message);
|
||||
|
|
@ -34,7 +33,12 @@ impl Logger {
|
|||
|
||||
/// Display an ERROR message
|
||||
pub fn error(message: String) {
|
||||
eprintln!("{}: [{}] {}", Logger::get_timestamp(), "ERROR".red(), message);
|
||||
eprintln!(
|
||||
"{}: [{}] {}",
|
||||
Logger::get_timestamp(),
|
||||
"ERROR".red(),
|
||||
message
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ impl Scraper {
|
|||
|
||||
Scraper {
|
||||
downloader: downloader::Downloader::new(args.tries),
|
||||
args: args,
|
||||
args,
|
||||
transmitter: tx,
|
||||
receiver: rx,
|
||||
visited_urls: Mutex::new(HashSet::new()),
|
||||
|
|
@ -53,12 +53,11 @@ impl Scraper {
|
|||
fn map_url(&self, url: &Url, path: String) -> bool {
|
||||
let mut path_map = self.path_map.lock().unwrap();
|
||||
|
||||
match path_map.contains_key(url.as_str()) {
|
||||
false => {
|
||||
path_map.insert(url.to_string(), path);
|
||||
true
|
||||
}
|
||||
true => false,
|
||||
if !path_map.contains_key(url.as_str()) {
|
||||
path_map.insert(url.to_string(), path);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -93,14 +92,11 @@ impl Scraper {
|
|||
.filter(|candidate| Scraper::should_visit(candidate, &url))
|
||||
.for_each(|next_url| {
|
||||
let next_full_url = url.join(&next_url).unwrap();
|
||||
match scraper.map_url(&next_full_url, url_helper::url_to_path(&next_full_url)) {
|
||||
true => {
|
||||
if depth < scraper.args.depth {
|
||||
Scraper::push(transmitter, next_full_url.clone(), depth + 1);
|
||||
}
|
||||
}
|
||||
false => (),
|
||||
};
|
||||
if scraper.map_url(&next_full_url, url_helper::url_to_path(&next_full_url))
|
||||
&& depth < scraper.args.depth
|
||||
{
|
||||
Scraper::push(transmitter, next_full_url.clone(), depth + 1);
|
||||
}
|
||||
|
||||
scraper.fix_domtree(next_url, &next_full_url);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ pub fn url_to_path(url: &Url) -> String {
|
|||
}
|
||||
let url = url.trim_end_matches('_'); //Remaining '/'
|
||||
|
||||
return url.to_string();
|
||||
url.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
Loading…
Reference in a new issue