utils-system/darakian.ddh
1
0
Fork 0
mirror of https://github.com/darakian/ddh.git synced 2026-09-10 07:06:21 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Wire up cli option

Browse source
This commit is contained in:
Jon Moroney 2025-08-27 15:12:04 -07:00
parent 4493e48786
commit eb4bcfb262
No known key found for this signature in database
GPG key ID: 0DE20E0350BECFF2
3 changed files with 13 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/fileinfo.rs
View file

@ -4,6 +4,7 @@ use std::cmp::Ordering;
use std::fs::{self, Metadata}; use std::fs::{self, Metadata};
use std::io::Read; use std::io::Read;
use std::path::PathBuf; use std::path::PathBuf;
use clap::ValueEnum;
const BLOCK_SIZE: usize = 4096; const BLOCK_SIZE: usize = 4096;
@ -13,6 +14,7 @@ pub enum HashMode {
Partial, Partial,
} }
#[derive(Debug, Copy, Clone, ValueEnum)]
pub enum HashAlgoEnum { pub enum HashAlgoEnum {
Sha256, Sha256,
Blake3, Blake3,

11
src/lib.rs
View file

@ -31,7 +31,8 @@ enum ChannelPackage {
pub fn deduplicate_dirs<P: AsRef<Path> + Sync>( pub fn deduplicate_dirs<P: AsRef<Path> + Sync>(
search_dirs: Vec<P>, search_dirs: Vec<P>,
ignore_dirs: Vec<P>, ignore_dirs: Vec<P>,
min_size: u64) -> (Vec<Fileinfo>, Vec<(PathBuf, std::io::Error)>) { min_size: u64,
hash_algo: HashAlgoEnum) -> (Vec<Fileinfo>, Vec<(PathBuf, std::io::Error)>) {
let (sender, receiver) = channel(); let (sender, receiver) = channel();
let ignore_paths = ignore_dirs.iter().map(|x| x.as_ref().canonicalize().unwrap()).collect(); let ignore_paths = ignore_dirs.iter().map(|x| x.as_ref().canonicalize().unwrap()).collect();
search_dirs search_dirs
@ -56,7 +57,7 @@ pub fn deduplicate_dirs<P: AsRef<Path> + Sync>(
}); });
let complete_files: Vec<Fileinfo> = files_of_lengths let complete_files: Vec<Fileinfo> = files_of_lengths
.into_par_iter() .into_par_iter()
.map(|x| differentiate_and_consolidate(x.0, x.1)) .map(|x| differentiate_and_consolidate(x.0, x.1, hash_algo))
.flatten() .flatten()
.collect(); .collect();
(complete_files, errors) (complete_files, errors)
@ -124,7 +125,7 @@ fn traverse_and_spawn(current_path: impl AsRef<Path>, ignore_dirs: &Vec<PathBuf>
} }
} }
fn differentiate_and_consolidate(file_length: u64, mut files: Vec<Fileinfo>) -> Vec<Fileinfo> { fn differentiate_and_consolidate(file_length: u64, mut files: Vec<Fileinfo>, hash_algo: HashAlgoEnum) -> Vec<Fileinfo> {
if file_length == 0 || files.is_empty() { if file_length == 0 || files.is_empty() {
return files; return files;
} }
@ -132,7 +133,7 @@ fn differentiate_and_consolidate(file_length: u64, mut files: Vec<Fileinfo>) ->
1 => return files, 1 => return files,
n if n > 1 => { n if n > 1 => {
files.par_iter_mut().for_each(|file_ref| { files.par_iter_mut().for_each(|file_ref| {
let hash = file_ref.generate_hash(HashMode::Partial, HashAlgoEnum::Blake3); let hash = file_ref.generate_hash(HashMode::Partial, hash_algo);
file_ref.set_partial_hash(hash); file_ref.set_partial_hash(hash);
}); });
if file_length <= 4096 { if file_length <= 4096 {
@ -159,7 +160,7 @@ fn differentiate_and_consolidate(file_length: u64, mut files: Vec<Fileinfo>) ->
.collect(); .collect();
files.par_iter_mut().for_each(|x| { files.par_iter_mut().for_each(|x| {
if dedupe_hashes.contains(&x.get_partial_hash()) { if dedupe_hashes.contains(&x.get_partial_hash()) {
let hash = x.generate_hash(HashMode::Full, HashAlgoEnum::Blake3); let hash = x.generate_hash(HashMode::Full, hash_algo);
x.set_full_hash(hash); x.set_full_hash(hash);
} }
}); });

7
src/main.rs
View file

@ -1,5 +1,5 @@
use clap::{Parser, ValueEnum}; use clap::{Parser, ValueEnum};
use ddh::fileinfo::Fileinfo; use ddh::fileinfo::{Fileinfo, HashAlgoEnum};
use rayon::prelude::*; use rayon::prelude::*;
use std::fs::{self}; use std::fs::{self};
use std::io::prelude::*; use std::io::prelude::*;
@ -24,6 +24,9 @@ struct Args {
/// Set output format /// Set output format
#[arg(short('f'), long("format"), ignore_case(true), value_enum, num_args(0..=1), default_value_t = PrintFmt::Standard)] #[arg(short('f'), long("format"), ignore_case(true), value_enum, num_args(0..=1), default_value_t = PrintFmt::Standard)]
fmt: PrintFmt, fmt: PrintFmt,
/// Set output format
#[arg(short('a'), long("hashalgo"), ignore_case(true), value_enum, num_args(0..=1), default_value_t = HashAlgoEnum::Blake3)]
hash: HashAlgoEnum,
/// Directories to ignore (comma separated list) /// Directories to ignore (comma separated list)
#[arg(short, long("ignore"), value_delimiter(','))] #[arg(short, long("ignore"), value_delimiter(','))]
ignore_dirs: Vec<String>, ignore_dirs: Vec<String>,
@ -64,7 +67,7 @@ fn main() {
let arguments = Args::parse(); let arguments = Args::parse();
let (complete_files, read_errors): (Vec<Fileinfo>, Vec<(_, _)>) = let (complete_files, read_errors): (Vec<Fileinfo>, Vec<(_, _)>) =
ddh::deduplicate_dirs(arguments.directories, arguments.ignore_dirs, arguments.min_size); ddh::deduplicate_dirs(arguments.directories, arguments.ignore_dirs, arguments.min_size, arguments.hash);
let (shared_files, unique_files): (Vec<&Fileinfo>, Vec<&Fileinfo>) = complete_files let (shared_files, unique_files): (Vec<&Fileinfo>, Vec<&Fileinfo>) = complete_files
.par_iter() .par_iter()
.partition(|&x| x.get_paths().len() > 1); .partition(|&x| x.get_paths().len() > 1);