<commit_before>package de.dergrafiker.filenamecleaner;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.util.Set;

import static java.nio.file.FileVisitResult.CONTINUE;

@Component
public class FileVisitor extends SimpleFileVisitor<Path> {
    private static final Logger LOGGER = LoggerFactory.getLogger(FileVisitor.class);
    private final RemovedCharsUtil removedCharsUtil;
    private final FilenameChecker filenameChecker;
    private final FilenameCleaner filenameCleaner;

    public FileVisitor(RemovedCharsUtil removedCharsUtil, FilenameChecker filenameChecker,
                       FilenameCleaner filenameCleaner) {
        this.removedCharsUtil = removedCharsUtil;
        this.filenameChecker = filenameChecker;
        this.filenameCleaner = filenameCleaner;
    }

    @Override
    public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
        if (exc != null) {
            throw exc;
        }

        final File file = dir.toFile();
        String oldName = file.getName();
        if (filenameChecker.isInvalid(oldName, file.isDirectory())) {
            String cleaned = filenameCleaner.clean(oldName, file.isDirectory());

            Set<Character> removedChars = removedCharsUtil.getRemovedChars(oldName, cleaned);

            if (LOGGER.isInfoEnabled() && !removedChars.isEmpty()) {
                LOGGER.info("REMOVEDCHARS {} has removed >> {} << {}",
                            oldName,
                            StringUtils.join(removedChars),
                            cleaned);
            }

            LOGGER.info("RENAME '{}' => '{}' [{}]",
                        oldName,
                        cleaned,
                        file.getParentFile().getAbsolutePath());
        }
        LOGGER.trace("WALKED dir {}", dir);
        return CONTINUE;
    }


}<commit_msg>use nio<commit_after>package de.dergrafiker.filenamecleaner;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.util.Set;

import static java.nio.file.FileVisitResult.CONTINUE;

@Component
public class FileVisitor extends SimpleFileVisitor<Path> {
    private static final Logger LOGGER = LoggerFactory.getLogger(FileVisitor.class);
    private final RemovedCharsUtil removedCharsUtil;
    private final FilenameChecker filenameChecker;
    private final FilenameCleaner filenameCleaner;

    public FileVisitor(RemovedCharsUtil removedCharsUtil, FilenameChecker filenameChecker,
                       FilenameCleaner filenameCleaner) {
        this.removedCharsUtil = removedCharsUtil;
        this.filenameChecker = filenameChecker;
        this.filenameCleaner = filenameCleaner;
    }

    @Override
    public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
        if (exc != null) {
            throw exc;
        }

        String oldName = dir.getFileName().toString();
        boolean isDirectory = Files.isDirectory(dir);

        if (filenameChecker.isInvalid(oldName, isDirectory)) {
            String cleaned = filenameCleaner.clean(oldName, isDirectory);

            Set<Character> removedChars = removedCharsUtil.getRemovedChars(oldName, cleaned);

            if (LOGGER.isInfoEnabled() && !removedChars.isEmpty()) {
                LOGGER.info("REMOVEDCHARS {} has removed >> {} << {}",
                        oldName,
                        StringUtils.join(removedChars),
                        cleaned);
            }

            LOGGER.info("RENAME '{}' => '{}' [{}]",
                    oldName,
                    cleaned,
                    dir.getParent().toAbsolutePath()
            );
        }
        LOGGER.trace("WALKED dir {}", dir);
        return CONTINUE;
    }


}
{'fraction_non_alphanumeric': 0.09422850412249706, 'fraction_numerical': 0.0014134275618374558, 'mean_word_length': 2.5770850884582983, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '19672', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1121, 'n_words': 224}