utils-system/darakian.ddh
1
0
Fork 0
mirror of https://github.com/darakian/ddh.git synced 2026-09-10 07:06:21 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Prehash on 4096 bytes (one filesystem block)

Browse source
This commit is contained in:
Jon Moroney 2018-03-23 17:26:28 +01:00
parent 07f58ae645
commit b5cc0601f3
1 changed files with 4 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/main.rs
View file

@ -212,7 +212,7 @@ fn differentiate_and_consolidate(file_length: u64, mut files: Vec<Fileinfo>) ->
match fs::File::open(x.file_paths.iter().next().expect("Error opening file for hashing")) {
Ok(f) => {
let mut buffer_reader = BufReader::new(f);
let mut hash_buffer = [0;131072]; //read 128KB
let mut hash_buffer = [0;4096]; //read 128KB
match buffer_reader.read(&mut hash_buffer) {
Ok(n) if n>0 => hasher.write(&hash_buffer[0..n]),
Ok(n) if n==0 => { //No more data in the file
@ -220,15 +220,14 @@ fn differentiate_and_consolidate(file_length: u64, mut files: Vec<Fileinfo>) ->
Err(e) => println!("{:?} reading {:?}", e, x.file_paths.iter().next().expect("Error opening file for hashing")),
_ => println!("Should not be here"),
}
x.file_hash=hasher.finish();
}
Err(e) => {println!("Error:{} when opening {:?}. Skipping.", e, x.file_paths.iter().next().expect("Error opening file for hashing"))}
}
});
//Find unique elements and extend hash for similar-ish files
//Find unique elements and extend hash for similar-ish files
files.par_sort_unstable_by(|a, b| b.file_hash.cmp(&a.file_hash)); //O(nlog(n))
if file_length>131072 /*128KB*/ { //only hash again if we are not done hashing
if file_length>4096 /*128KB*/ { //only hash again if we are not done hashing
files.dedup_by(|a, b| if a==b{ //O(n)
a.hashed=true;
b.hashed=true;
@ -236,7 +235,7 @@ fn differentiate_and_consolidate(file_length: u64, mut files: Vec<Fileinfo>) ->
}else{false});
files.par_iter_mut().filter(|x| x.hashed==true).for_each(|y| {
y.hashed=false;
hash_and_update(y, 131072); //Skip 128KB
hash_and_update(y, 4096); //Skip 128KB
});
}
},