<commit_before>use std::cmp::Ordering;
use std::fs;
use std::path::Path;

pub mod tree;
pub mod print;
use self::tree::{DirectoryNode, FileNode, FSNode};

pub fn read_recursive(path: &String, ignore_dotfiles: bool) -> FSNode {
    let path = fs::canonicalize(Path::new(&path)).unwrap();
    let mut node = DirectoryNode::new(path.file_name().unwrap().to_str().unwrap().to_string());

    for entry in fs::read_dir(path).unwrap() {
        let entry = entry.unwrap();
        let meta = entry.metadata().unwrap();
        let name = entry.file_name().to_str().unwrap().to_string();

        if ignore_dotfiles && name.starts_with(".") {
            continue;
        }

        if meta.is_file() {
            node.children.push(FSNode::File(FileNode {
                name: name,
                size: meta.len(),
            }));
            node.size += meta.len();
        } else if meta.is_dir() {
            let path = entry.path().to_str().unwrap().to_string();
            let dir = read_recursive(&path, ignore_dotfiles);
            node.size += dir.size();
            node.children.push(dir);<commit_msg>fix clippy warning<commit_after>use std::cmp::Ordering;
use std::fs;
use std::path::Path;

pub mod tree;
pub mod print;
use self::tree::{DirectoryNode, FileNode, FSNode};

pub fn read_recursive(path: &String, ignore_dotfiles: bool) -> FSNode {
    let path = fs::canonicalize(Path::new(&path)).unwrap();
    let mut node = DirectoryNode::new(path.file_name().unwrap().to_str().unwrap().to_string());

    for entry in fs::read_dir(path).unwrap() {
        let entry = entry.unwrap();
        let meta = entry.metadata().unwrap();
        let name = entry.file_name().to_str().unwrap().to_string();

        if ignore_dotfiles && name.starts_with('.') {
            continue;
        }

        if meta.is_file() {
            node.children.push(FSNode::File(FileNode {
                name: name,
                size: meta.len(),
            }));
            node.size += meta.len();
        } else if meta.is_dir() {
            let path = entry.path().to_str().unwrap().to_string();
            let dir = read_recursive(&path, ignore_dotfiles);
            node.size += dir.size();
            node.children.push(dir);
{'fraction_non_alphanumeric': 0.16361198033080018, 'fraction_numerical': 0.0, 'mean_word_length': 2.469767441860465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3701853', 'n_tokens_mistral': 686, 'n_tokens_neox': 684, 'n_words': 139}