<commit_before>import os
import sys

from copy import \
        deepcopy
from os.path import \
        join as pjoin, dirname as pdirname
from ConfigParser \
        import ConfigParser

import numscons
from numscons.core.utils import DefaultDict

_CONFDIR = pjoin(pdirname(numscons.__file__), 'configurations')

def _get_win32_config_files():
    # We import platform here as we only need it for windows and platform
    # import is relatively slow
    import platform

    files = [pjoin(_CONFDIR, 'win32', 'perflib.cfg')]
    if platform.machine() == 'AMD64':
        files.append(pjoin(_CONFDIR, 'win64', 'perflib.cfg'))
    return files

def get_config_files(env):
    """Return the list of configuration files to consider for perflib
    configuration."""
    files = [pjoin(_CONFDIR, 'perflib.cfg')]
    if sys.platform == 'win32':
        files.extend(_get_win32_config_files())
    if env:
        files.append(pjoin(str(env.fs.Top), 'numscons.cfg'))
    return files

class ConfigDict(DefaultDict):
    __keys = ['libraries', 'blas_libraries', 'lapack_libraries', 'cblas_libraries',
            'cflags', 'ldflags', 'include_dirs', 'library_dirs', 'frameworks']
    @classmethod
    def fromcallable(cls, avkeys, default=None):
        raise UnimplementedError()

    def __init__(self):
        DefaultDict.__init__(self, self.__keys)
        for k in self.keys():
            self[k] = []

class BuildDict(DefaultDict):
    __keys = ['CFLAGS', 'LIBS', 'LINKFLAGS', 'CPPPATH', 'LIBPATH', 'FRAMEWORKS']
    __convert_dict = {'cflags': 'CFLAGS',
            'libraries': 'LIBS',
            'ldflags': 'LINKFLAGS',
            'include_dirs': 'CPPPATH',
            'library_dirs': 'LIBPATH',
            'frameworks': 'FRAMEWORKS'
            }
    @classmethod
    def fromcallable(cls, avkeys, default=None):
        raise UnimplementedError()

    @classmethod
    def from_config_dict(cls, config_dict):
        ret = cls()
        for k, v in config_dict.items():
            ret[cls.__convert_dict[k]] = v
        return ret

    def __init__(self):
        DefaultDict.__init__(self, self.__keys)
        for k in self.keys():
            self[k] = []

def _read_section(section, env):
    parser = ConfigParser()
    files = get_config_files(env)
    r = parser.read(files)
    if len(r) < 1:<commit_msg>BUG: BuildDict did not work with simple checker.<commit_after>import os
import sys

from copy import \
        deepcopy
from os.path import \
        join as pjoin, dirname as pdirname
from ConfigParser \
        import ConfigParser

import numscons
from numscons.core.utils import DefaultDict

_CONFDIR = pjoin(pdirname(numscons.__file__), 'configurations')

def _get_win32_config_files():
    # We import platform here as we only need it for windows and platform
    # import is relatively slow
    import platform

    files = [pjoin(_CONFDIR, 'win32', 'perflib.cfg')]
    if platform.machine() == 'AMD64':
        files.append(pjoin(_CONFDIR, 'win64', 'perflib.cfg'))
    return files

def get_config_files(env):
    """Return the list of configuration files to consider for perflib
    configuration."""
    files = [pjoin(_CONFDIR, 'perflib.cfg')]
    if sys.platform == 'win32':
        files.extend(_get_win32_config_files())
    if env:
        files.append(pjoin(str(env.fs.Top), 'numscons.cfg'))
    return files

class ConfigDict(DefaultDict):
    __keys = ['libraries', 'blas_libraries', 'lapack_libraries', 'cblas_libraries',
            'cflags', 'ldflags', 'include_dirs', 'library_dirs', 'frameworks']
    @classmethod
    def fromcallable(cls, avkeys, default=None):
        raise UnimplementedError()

    def __init__(self):
        DefaultDict.__init__(self, self.__keys)
        for k in self.keys():
            self[k] = []

class BuildDict(DefaultDict):
    __keys = ['CFLAGS', 'LIBS', 'LINKFLAGS', 'CPPPATH', 'LIBPATH', 'FRAMEWORKS']
    __convert_dict = {'cflags': 'CFLAGS',
            'libraries': 'LIBS',
            'ldflags': 'LINKFLAGS',
            'include_dirs': 'CPPPATH',
            'library_dirs': 'LIBPATH',
            'frameworks': 'FRAMEWORKS'
            }
    @classmethod
    def fromcallable(cls, avkeys, default=None):
        raise UnimplementedError()

    @classmethod
    def from_config_dict(cls, config_dict):
        ret = cls()
        for k in cls.__convert_dict.keys():
            ret[cls.__convert_dict[k]] = config_dict[k]
        return ret

    def __init__(self):
        DefaultDict.__init__(self, self.__keys)
        for k in self.keys():
            self[k] = []

def _read_section(section, env):
    parser = ConfigParser()
    files = get_config_files(env)
    r = parser.read(files)
    if len(r) < 1:
{'fraction_non_alphanumeric': 0.12097976570820021, 'fraction_numerical': 0.005537806176783813, 'mean_word_length': 2.9297071129707115, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2459680', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1472, 'n_words': 369}