<commit_before>extern crate clap;
extern crate walkdir;
extern crate twox_hash;

use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};

use clap::{App, Arg};
use walkdir::WalkDir;
use twox_hash::XxHash;
use std::hash::Hasher;

#[derive(Default)]
struct Stats {
    file_count: u64,
    total_size: u64,
    duplicate_count: u64,
}


fn collect_files(path: &str, stats: &mut Stats) -> HashMap<u64, Vec<PathBuf>> {

    let mut files = HashMap::new();

    for entry in WalkDir::new(path) {
        let entry = entry.unwrap();

        if !entry.file_type().is_file() {
            continue;
        }

        let metadata = entry.metadata().unwrap();

        if metadata.len() == 0 {
            continue;
        }

        stats.file_count += 1;
        stats.total_size += metadata.len();

        let path = entry.path().to_path_buf();

        files.entry(metadata.len()).or_insert(Vec::new()).push(path);
    }

    files
}



fn main() {
    let matches = App::new("dupr")
        .version("0.1.0")
        .author("<NAME>")
        .about("Duplicate file finder")
        .arg(
            Arg::with_name("DIR")
                .help("Directory to process")
                .required(true),
        )
        .arg(
            Arg::with_name("summary")
                .short("s")
                .long("summary")
                .help("Print out summary information after duplicates")
        )
        .get_matches();

    let mut stats: Stats = Default::default();

    let files = collect_files(matches.value_of("DIR").unwrap(), &mut stats);

    for f_paths in files.values().filter(|x| x.len() > 1) {

        let mut hashes = HashMap::new();
        for f_path in f_paths {
            hashes
                .entry(hash_file(&f_path))
                .or_insert(Vec::new())
                .push(f_path);
        }

        for (hash, paths) in hashes.iter().filter(|&(_, v)| v.len() > 1) {
            stats.duplicate_count += paths.len() as u64;
            for p in paths {
                println!("{:016x} - {}", hash, p.display());
            }
        }
    }

    if matches.is_present("summary")  {
        println!(
            "Processed {} files with a total size of {} bytes. {} duplicates found.",
            stats.file_count,
            stats.total_size,
            stats.duplicate_count
        );
    }
}

fn hash_file<P: AsRef<Path>>(path: P) -> u64 {
    let mut file = File::open(path).unwrap();<commit_msg>Tidy up. Factor out arg parsing.<commit_after>extern crate clap;
extern crate walkdir;
extern crate twox_hash;

use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};

use clap::{App, Arg, ArgMatches};
use walkdir::WalkDir;
use twox_hash::XxHash;
use std::hash::Hasher;

#[derive(Default)]
struct Stats {
    file_count: u64,
    total_size: u64,
    duplicate_count: u64,
}

fn parse_args<'a>() -> ArgMatches<'a> {
    App::new("dupr")
        .version("0.1.0")
        .author("<NAME>")
        .about("Duplicate file finder")
        .arg(
            Arg::with_name("DIR")
                .help("Directory to process")
                .required(true),
        )
        .arg(Arg::with_name("summary").short("s").long("summary").help(
            "Print out summary information after duplicates",
        ))
        .get_matches()
}


fn collect_files(path: &str, stats: &mut Stats) -> HashMap<u64, Vec<PathBuf>> {

    let mut files = HashMap::new();

    for entry in WalkDir::new(path) {
        let entry = entry.unwrap();

        if !entry.file_type().is_file() {
            continue;
        }

        let metadata = entry.metadata().unwrap();

        if metadata.len() == 0 {
            continue;
        }

        stats.file_count += 1;
        stats.total_size += metadata.len();

        let path = entry.path().to_path_buf();

        files.entry(metadata.len()).or_insert(Vec::new()).push(path);
    }

    files
}


fn main() {
    let matches = parse_args();

    let mut stats: Stats = Default::default();

    let files = collect_files(matches.value_of("DIR").unwrap(), &mut stats);

    for f_paths in files.values().filter(|x| x.len() > 1) {

        let mut hashes = HashMap::new();
        for f_path in f_paths {
            hashes
                .entry(hash_file(&f_path))
                .or_insert(Vec::new())
                .push(f_path);
        }

        for (hash, paths) in hashes.iter().filter(|&(_, v)| v.len() > 1) {
            stats.duplicate_count += paths.len() as u64;
            for p in paths {
                println!("{:016x} - {}", hash, p.display());
            }
        }
    }

    if matches.is_present("summary") {
        println!(
            "Processed {} files with a total size of {} bytes. {} duplicates found.",
            stats.file_count,
            stats.total_size,
            stats.duplicate_count
        );
    }
}

fn hash_file<P: AsRef<Path>>(path: P) -> u64 {
    let mut file = File::open(path).unwrap();
{'fraction_non_alphanumeric': 0.15629349771326306, 'fraction_numerical': 0.00874925432491549, 'mean_word_length': 2.031946955997589, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1587697', 'n_tokens_mistral': 1642, 'n_tokens_neox': 1567, 'n_words': 352}