<commit_before># -*- coding: utf-8 -*-
"""
Application file operations
"""
import os
import shutil
from logger import log
from settings import SETTINGS

__author__ = '<NAME>'


class FSTools():
    """
    File System Tools Class
    create_directory: Creates directory in given path
    control_directory: Checks directory existence in given path
    safe_make_directory: Cehcks directory existence before make
    user_path: Returns current user home directory path
    target_dir_path: Returns given target directory full path under current user
    """
    def __init__(self, directory=None):
        if directory is None:
            raise Exception("No directory name or path given.")
        self.directory = directory

    @property
    def user_path(self):
        return os.path.expanduser(u"~")

    def target_dir_path(self):
        return os.path.join(self.user_path, self.directory)

    def make_directory(self):
        created = False
        try:
            os.makedirs(self.target_dir_path())
            created = True
        except Exception as e:
            log.exception(e.message)
        finally:
            return created

    def check_directory(self):
        log.debug("Checking existence of {directory}".format(directory=self.target_dir_path()))
        return os.path.exists(self.target_dir_path())

    def safe_make_directory(self):
        if not self.check_directory():
            if not self.make_directory():
                raise Exception("Unable to create directory: {directory}".format(directory=self.directory))
            else:
                log.info("Directory created: {directory}".format(directory=self.directory))
        else:
            log.warning("Directory exsists: {directory}".format(directory=self.directory))


class GetFileList():
    """
    Returns files list in given target directory
    """
    def __init__(self):
        self.fstools = FSTools(SETTINGS["target_directory"])
        log.debug(self.fstools.check_directory())
        self.target_dir = self.fstools.target_dir_path()
        log.info("Getting file list in {target}".format(target=self.target_dir))
        self.file_list = os.listdir(self.target_dir)
        self.file_list = [os.path.join(self.target_dir, f) for f in self.file_list]
        self.exclude_directories()
        self.exclude_files()

    def exclude_directories(self):
        excluded_dirs = list()
        for f in self.file_list:
            if os.path.isdir(f):
                excluded_dirs.append(f)
        log.debug("Directories are removed from list: {}".format(repr(excluded_dirs)))
        self.file_list = set(self.file_list).symmetric_difference(excluded_dirs)

    def exclude_files(self):
        excluded_files = list()
        for x in SETTINGS["excluded_files"]:
            for f in self.file_list:
                if f.endswith(x):
                    excluded_files.append(f)<commit_msg>Automattically create given directory feature added.<commit_after># -*- coding: utf-8 -*-
"""
Application file operations
"""
import os
import shutil
from logger import log
from settings import SETTINGS

__author__ = '<NAME>'


class FSTools():
    """
    File System Tools Class
    create_directory: Creates directory in given path
    control_directory: Checks directory existence in given path
    safe_make_directory: Cehcks directory existence before make
    user_path: Returns current user home directory path
    target_dir_path: Returns given target directory full path under current user
    """
    def __init__(self, directory=None):
        if directory is None:
            raise Exception("No directory name or path given.")
        self.directory = directory

    @property
    def user_path(self):
        return os.path.expanduser(u"~")

    def target_dir_path(self):
        return os.path.join(self.user_path, self.directory)

    def make_directory(self):
        created = False
        try:
            os.makedirs(self.target_dir_path())
            created = True
        except Exception as e:
            log.exception(e.message)
        finally:
            return created

    def check_directory(self):
        log.debug("Checking existence of {directory}".format(directory=self.target_dir_path()))
        return os.path.exists(self.target_dir_path())

    def safe_make_directory(self):
        if not self.check_directory():
            log.warning("Directory does not exists: {directory}".format(directory=self.directory))
            if not self.make_directory():
                raise Exception("Unable to create directory: {directory}".format(directory=self.directory))
            else:
                log.info("Directory created: {directory}".format(directory=self.directory))
        else:
            log.warning("Directory exists: {directory}".format(directory=self.directory))


class GetFileList():
    """
    Returns files list in given target directory
    """
    def __init__(self):
        self.fstools = FSTools(SETTINGS["target_directory"])
        self.fstools.safe_make_directory()
        self.target_dir = self.fstools.target_dir_path()
        log.info("Getting file list in {target}".format(target=self.target_dir))
        self.file_list = os.listdir(self.target_dir)
        self.file_list = [os.path.join(self.target_dir, f) for f in self.file_list]
        self.exclude_directories()
        self.exclude_files()

    def exclude_directories(self):
        excluded_dirs = list()
        for f in self.file_list:
            if os.path.isdir(f):
                excluded_dirs.append(f)
        log.debug("Directories are removed from list: {}".format(repr(excluded_dirs)))
        self.file_list = set(self.file_list).symmetric_difference(excluded_dirs)

    def exclude_files(self):
        excluded_files = list()
        for x in SETTINGS["excluded_files"]:
            for f in self.file_list:
                if f.endswith(x):
                    excluded_files.append(f)
{'fraction_non_alphanumeric': 0.09892509237487404, 'fraction_numerical': 0.0003359086328518643, 'mean_word_length': 2.703358208955224, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1616322', 'n_tokens_mistral': 1667, 'n_tokens_neox': 1637, 'n_words': 409}