<commit_before>/*******************************************************************************
*  Copyright 2013 <NAME>, <NAME>                                   *
*                                                                             *
*  Licensed under the Apache License, Version 2.0 (the "License");            *
*  you may not use this file except in compliance with the License.           *
*  You may obtain a copy of the License at                                    *
*                                                                             *
*      http://www.apache.org/licenses/LICENSE-2.0                             *
*                                                                             *
*  Unless required by applicable law or agreed to in writing, software        *
*  distributed under the License is distributed on an "AS IS" BASIS,          *
*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   *
*  See the License for the specific language governing permissions and        *
*  limitations under the License.                                             *
*******************************************************************************/

package net.snakedoc.superd;

import net.snakedoc.jutils.Config;
import net.snakedoc.jutils.io.Hasher;
import net.snakedoc.jutils.io.HasherException;
import org.apache.log4j.Logger;

import java.io.File;
import java.io.IOException;

public class Walker {
    private static int BUFFER;
    private static final Logger log = Logger.getLogger(DedupeR.class);
    private static Hasher hasher = new Hasher();
    private static DedupeSQL sql = new DedupeSQL();
    private static Config config = new Config("props/superD.properties");

    public Walker(int buffer){
        this.BUFFER = buffer;
    }

    public static void walk(File path){
        File[] contents = path.listFiles();

        for (File curFile : contents){
            try{
                if (isValidDirectory(curFile)){
                    walk(curFile);
                } else if (isValidFile(curFile)){
                    hashFile(curFile);
                }
            } catch (Exception e) {
                log.warn("Failed to access file!", e);
                log.debug("path: " + curFile.getPath() +
                        "  |  pathcalled: " + path.getPath());
            }
        }
    }

    public static boolean isValidDirectory(File curFile){
        if (curFile.isDirectory() && curFile!=null && !curFile.isHidden()){
            return true;
        }else{
            return false;
        }
    }

    public static boolean isValidFile(File curFile){
        if (!curFile.isDirectory() && curFile!=null && !curFile.isHidden()){
            return true;
        }else{
            return false;
        }
    }

    public static void hashFile(File curFile){
        try{
        String hashAlgo = new String(config.getConfig("HASH_ALGO"));
        String file = "";
        String hash = "";
        try {
            file = curFile.getPath();
            log.debug("File: " + file);
            hash = hasher.getHash(curFile.getPath(), hashAlgo, BUFFER);
        } catch (IOException | HasherException e1) {
            log.error("Failed to access and/or hash file!", e1);
        }
        sql.writeRecord(file, hash);
        log.debug("\n\t Hash: " + hash);
        }catch( Exception e){
            log.warn(e.getStackTrace());
        }
    }
}
<commit_msg>Cleaned up code<commit_after>/*******************************************************************************
*  Copyright 2013 <NAME>, <NAME>                                   *
*                                                                             *
*  Licensed under the Apache License, Version 2.0 (the "License");            *
*  you may not use this file except in compliance with the License.           *
*  You may obtain a copy of the License at                                    *
*                                                                             *
*      http://www.apache.org/licenses/LICENSE-2.0                             *
*                                                                             *
*  Unless required by applicable law or agreed to in writing, software        *
*  distributed under the License is distributed on an "AS IS" BASIS,          *
*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   *
*  See the License for the specific language governing permissions and        *
*  limitations under the License.                                             *
*******************************************************************************/

package net.snakedoc.superd;

import net.snakedoc.jutils.Config;
import net.snakedoc.jutils.io.Hasher;
import net.snakedoc.jutils.io.HasherException;
import org.apache.log4j.Logger;

import java.io.File;
import java.io.IOException;

public class Walker {
    private static int BUFFER;
    private static final Logger log = Logger.getLogger(DedupeR.class);
    private static final Hasher hasher = new Hasher();
    private static final DedupeSQL sql = new DedupeSQL();
    private static final Config config = new Config("props/superD.properties");

    public Walker(int buffer){
        BUFFER = buffer;
    }

    public static void walk(File path){
        File[] contents = path.listFiles();

        for (File curFile : contents){
            try{
                if (isValidDirectory(curFile)){
                    walk(curFile);
                } else if (isValidFile(curFile)){
                    hashFile(curFile);
                }
            } catch (Exception e) {
                log.warn("Failed to access file!", e);
                log.debug("path: " + curFile.getPath() +
                        "  |  pathcalled: " + path.getPath());
            }
        }
    }

    private static boolean isValidDirectory(File curFile){
        return curFile.isDirectory() && curFile != null && !curFile.isHidden();
    }

    private static boolean isValidFile(File curFile){
        return !curFile.isDirectory() && curFile != null && !curFile.isHidden();
    }

    private static void hashFile(File curFile){
        try{
        String hashAlgo = config.getConfig("HASH_ALGO");
        String file = "";
        String hash = "";
        try {
            file = curFile.getPath();
            log.debug("File: " + file);
            hash = hasher.getHash(curFile.getPath(), hashAlgo, BUFFER);
        } catch (IOException | HasherException e1) {
            log.error("Failed to access and/or hash file!", e1);
        }
        sql.writeRecord(file, hash);
        log.debug("\n\t Hash: " + hash);
        }catch( Exception e){
            log.warn(e.getStackTrace());
        }
    }
}

{'fraction_non_alphanumeric': 0.14010017678255746, 'fraction_numerical': 0.003241013553329405, 'mean_word_length': 1.6334367726920094, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7040903', 'n_tokens_mistral': 1660, 'n_tokens_neox': 1612, 'n_words': 468}