<commit_before>import os
import csv
import glob
import logging


logger = logging.getLogger(__name__)


def extract(func):
    """
    Decorator function. Open and extract data from CSV files.  Return list of dictionaries.

    :param func: Wrapped function with *args and **kwargs arguments.
    """
    def _wrapper(*args):
        out = []
        instance, prefix = args
        for fname in glob.glob(os.path.join(getattr(instance, 'directory'), prefix)):
            with open(fname) as g:
                out.extend(func(instance, data=csv.DictReader(g)))
        return out
    return _wrapper


class BaseCSV(object):
    def __init__(self, settings):
        self.directory = settings.CSV_DATA_DIR

    @staticmethod
    def column(field, **kwargs):
        try:
            value = kwargs[field].strip()
            return value if value != "" else None
        except (AttributeError, KeyError, TypeError) as ex:
            raise ex

    def column_unicode(self, field, **kwargs):
        try:
            return self.column(field, **kwargs).decode('utf-8')
        except (KeyError, AttributeError):
            return None

    def column_int(self, field, **kwargs):
        try:
            return int(self.column(field, **kwargs))
        except (KeyError, TypeError):
            return None

    def column_bool(self, field, **kwargs):
        try:
            return bool(self.column_int(field, **kwargs))
        except (KeyError, TypeError):
            return None<commit_msg>Add optional directory override variable to BaseCSV init<commit_after>import os
import csv
import glob
import logging


logger = logging.getLogger(__name__)


def extract(func):
    """
    Decorator function. Open and extract data from CSV files.  Return list of dictionaries.

    :param func: Wrapped function with *args and **kwargs arguments.
    """
    def _wrapper(*args):
        out = []
        instance, prefix = args
        for fname in glob.glob(os.path.join(getattr(instance, 'directory'), prefix)):
            with open(fname) as g:
                out.extend(func(instance, data=csv.DictReader(g)))
        return out
    return _wrapper


class BaseCSV(object):
    def __init__(self, settings, dir_override=None):
        self.directory = dir_override or settings.CSV_DATA_DIR

    @staticmethod
    def column(field, **kwargs):
        try:
            value = kwargs[field].strip()
            return value if value != "" else None
        except (AttributeError, KeyError, TypeError) as ex:
            raise ex

    def column_unicode(self, field, **kwargs):
        try:
            return self.column(field, **kwargs).decode('utf-8')
        except (KeyError, AttributeError):
            return None

    def column_int(self, field, **kwargs):
        try:
            return int(self.column(field, **kwargs))
        except (KeyError, TypeError):
            return None

    def column_bool(self, field, **kwargs):
        try:
            return bool(self.column_int(field, **kwargs))
        except (KeyError, TypeError):
            return None
{'fraction_non_alphanumeric': 0.1033810143042913, 'fraction_numerical': 0.0006501950585175553, 'mean_word_length': 2.377607025246981, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2831715', 'n_tokens_mistral': 852, 'n_tokens_neox': 827, 'n_words': 255}