utils-system/darakian.ddh
1
0
Fork 0
mirror of https://github.com/darakian/ddh.git synced 2026-09-10 07:06:21 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #55 from darakian/ignore-dirs

Browse source 
Add directory exclusion logic
This commit is contained in:
Jon 2022-04-28 16:13:07 -07:00 committed by GitHub
parent e4c574ffd9 4d71a84a05
commit 73da6f76a6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
src/lib.rs
View file

@ -20,23 +20,23 @@ enum ChannelPackage {
/// Constructs a list of unique files from a list of directories.
///
/// # Examples
/// ```no_run
/// let search_dirs = vec!["/home/jon", "/home/doe"];
/// let ignore_dirs = vec!["/home/jon/.git", "/home/doe/scratch"];
/// let min_size = 1024;
/// let (files, errors) = ddh::deduplicate_dirs(search_dirs, ignore_dirs, min_size);
/// ```
/// let directories = vec!["/home/jon", "/home/doe"];
/// let (files, errors) = ddh::deduplicate_dirs(directories);
/// ```
pub fn deduplicate_dirs<P: AsRef<Path> + Sync>(
search_dirs: Vec<P>,
) -> (Vec<Fileinfo>, Vec<(PathBuf, std::io::Error)>) {
deduplicate_dirs_with_min(search_dirs, 0)
}
pub fn deduplicate_dirs_with_min<P: AsRef<Path> + Sync>(
search_dirs: Vec<P>, min_size: u64) -> (Vec<Fileinfo>, Vec<(PathBuf, std::io::Error)>) {
pub fn deduplicate_dirs<P: AsRef<Path> + Sync>(
search_dirs: Vec<P>,
ignore_dirs: Vec<P>,
min_size: u64) -> (Vec<Fileinfo>, Vec<(PathBuf, std::io::Error)>) {
let (sender, receiver) = channel();
let ignore_paths = ignore_dirs.iter().map(|x| x.as_ref().canonicalize().unwrap()).collect();
search_dirs
.par_iter()
.for_each_with(sender, |s, search_dir| {
traverse_and_spawn(search_dir.as_ref(), s.clone(), min_size);
traverse_and_spawn(search_dir.as_ref(), &ignore_paths, s.clone(), min_size);
});
let mut files_of_lengths: IntMap<u64, Vec<Fileinfo>> = IntMap::default();
let mut errors = Vec::new();
@ -61,7 +61,10 @@ pub fn deduplicate_dirs_with_min<P: AsRef<Path> + Sync>(
(complete_files, errors)
}
fn traverse_and_spawn(current_path: impl AsRef<Path>, sender: Sender<ChannelPackage>, min_size: u64) {
fn traverse_and_spawn(current_path: impl AsRef<Path>, ignore_dirs: &Vec<PathBuf>, sender: Sender<ChannelPackage>, min_size: u64) {
if current_path.as_ref().canonicalize().is_ok() && ignore_dirs.iter().any(|x| current_path.as_ref().canonicalize().unwrap().starts_with(x)){
return;
}
let current_path_metadata = match fs::symlink_metadata(&current_path) {
Err(e) => {
sender
@ -104,10 +107,10 @@ fn traverse_and_spawn(current_path: impl AsRef<Path>, sender: Sender<ChannelPack
.is_file()
});
files.par_iter().for_each_with(sender.clone(), |sender, x| {
traverse_and_spawn(&x.path(), sender.clone(), min_size)
traverse_and_spawn(&x.path(), ignore_dirs, sender.clone(), min_size)
});
dirs.into_par_iter().for_each_with(sender, |sender, x| {
traverse_and_spawn(x.path().as_path(), sender.clone(), min_size);
traverse_and_spawn(x.path().as_path(), ignore_dirs, sender.clone(), min_size);
})
}
Err(e) => {

11
src/main.rs
View file

@ -34,6 +34,14 @@ fn main() {
.min_values(1)
.required(true)
.takes_value(true))
.arg(Arg::new("ignore")
.short('i')
.long("ignore")
.value_name("Ignore")
.help("Directories to ignore")
.min_values(1)
.required(false)
.takes_value(true))
.arg(Arg::new("Blocksize")
.short('b')
.long("blocksize")
@ -74,10 +82,11 @@ fn main() {
//let (sender, receiver) = channel();
let search_dirs: Vec<_> = arguments.values_of("directories").unwrap().collect();
let ignore_dirs: Vec<_> = arguments.values_of("ignore").unwrap().collect();
let min_size: u64 = arguments.value_of("Minimum").unwrap().parse::<u64>().unwrap_or(0);
let (complete_files, read_errors): (Vec<Fileinfo>, Vec<(_, _)>) =
ddh::deduplicate_dirs_with_min(search_dirs, min_size);
ddh::deduplicate_dirs(search_dirs, ignore_dirs, min_size);
let (shared_files, unique_files): (Vec<&Fileinfo>, Vec<&Fileinfo>) = complete_files
.par_iter()
.partition(|&x| x.get_paths().len() > 1);