<commit_before># :coding: utf-8

import os
import re

#: Regular Expression pattern for single line comments
ONE_LINE_COMMENT_PATTERN = re.compile(r"//.*?\n")

#: Regular Expression pattern for multi-line comments
MULTI_LINES_COMMENT_PATTERN = re.compile(r"/\*(.|\n)*?\*/")

#: Regular Expression pattern for classes
CLASS_PATTERN = re.compile(
    r"(?P<export>export +)?(?P<default>default +)?"
    r"(class +(?P<class_name>\w+)|const +(?P<variable_name>\w+) *= *class +\w+)"
    r"( +extends +(?P<mother_class>\w+))? *{"
)

#: Regular Expression pattern for function expressions
FUNCTION_PATTERN = re.compile(
    r"(?P<export>export +)?(?P<default>default +)?"
    r"function (?P<function_name>\w+) *\((?P<arguments>.*)\) *{"
)

#: Regular Expression pattern for arrow functions
FUNCTION_ARROW_PATTERN = re.compile(
    r"(?P<export>export +)?(?P<default>default +)?"
    r"const (?P<function_name>\w+) *= *\((?P<arguments>.*)\) *=> *{"
)

#: Regular Expression pattern for variables
VARIABLES_PATTERN = re.compile(
    r"(?P<export>export +)?(?P<default>default +)?"
    r"(const|let|var) (?P<variable_name>\w+) *= *(?P<variable_value>[^;]+);"
)


def get_environment(path):
    """Return javascript environment dictionary from *path*.

    Analyse the content of *path* recursively.

    Raises :exc:`OSError` if the directory is incorrect.

    """
    if not os.path.isdir(path) or not os.access(path, os.R_OK):
        raise OSError(
            "The javascript package directory is incorrect: {0}".format(path)
        )

    environment = dict(
        modules={},
        classes={},
        functions={},
        variables={},
        files={}
    )

    repository_name = os.path.basename(path)

    extensions = [".js", ".jsx"]

    for root, dirs, files in os.walk(path):
        root_folders = (
            [repository_name] + root.split(path)[-1].split(os.sep)[1:]
        )

        files[:] = [
            f for f in files
            if os.path.splitext(f)[1] in extensions
            and not f.startswith(".")
        ]

        dirs[:] = [
            d for d in dirs
            if not d.startswith(".")
        ]

        for _file in files:
            file_id = "/".join(root_folders + [_file])
            file_path = os.path.join(root, _file)

            module_id, environment = get_module_environment(
                file_id, files, environment
            )

            environment = get_file_environment(
                file_path, file_id, module_id, environment
            )

    return environment


def get_module_environment(file_id, files, environment=None):
    """Return module ID and updated environment from *file_id*.

    *file_id* is in the form of::

        relative/path/to/file.js

    *files* is the list of files at the same level of the *file_id* analysed.

    Update the *environment* is available and return it as-is if the file
    is not readable.

    """
    if environment is None:
        environment = {}

    hierarchy = file_id.split("/")
    file_name = hierarchy.pop()

    if file_name == "index.js":
        module_id = ".".join(hierarchy)
        module_name = guess_module_name(
            hierarchy[-1],
            hierarchy_folders=hierarchy[:-1],
            module_names=environment["modules"].keys()<commit_msg>Fill the environment key is unavailable<commit_after># :coding: utf-8

import os
import re

#: Regular Expression pattern for single line comments
ONE_LINE_COMMENT_PATTERN = re.compile(r"//.*?\n")

#: Regular Expression pattern for multi-line comments
MULTI_LINES_COMMENT_PATTERN = re.compile(r"/\*(.|\n)*?\*/")

#: Regular Expression pattern for classes
CLASS_PATTERN = re.compile(
    r"(?P<export>export +)?(?P<default>default +)?"
    r"(class +(?P<class_name>\w+)|const +(?P<variable_name>\w+) *= *class +\w+)"
    r"( +extends +(?P<mother_class>\w+))? *{"
)

#: Regular Expression pattern for function expressions
FUNCTION_PATTERN = re.compile(
    r"(?P<export>export +)?(?P<default>default +)?"
    r"function (?P<function_name>\w+) *\((?P<arguments>.*)\) *{"
)

#: Regular Expression pattern for arrow functions
FUNCTION_ARROW_PATTERN = re.compile(
    r"(?P<export>export +)?(?P<default>default +)?"
    r"const (?P<function_name>\w+) *= *\((?P<arguments>.*)\) *=> *{"
)

#: Regular Expression pattern for variables
VARIABLES_PATTERN = re.compile(
    r"(?P<export>export +)?(?P<default>default +)?"
    r"(const|let|var) (?P<variable_name>\w+) *= *(?P<variable_value>[^;]+);"
)


def get_environment(path):
    """Return javascript environment dictionary from *path*.

    Analyse the content of *path* recursively.

    Raises :exc:`OSError` if the directory is incorrect.

    """
    if not os.path.isdir(path) or not os.access(path, os.R_OK):
        raise OSError(
            "The javascript package directory is incorrect: {0}".format(path)
        )

    environment = dict(
        modules={},
        classes={},
        functions={},
        variables={},
        files={}
    )

    repository_name = os.path.basename(path)

    extensions = [".js", ".jsx"]

    for root, dirs, files in os.walk(path):
        root_folders = (
            [repository_name] + root.split(path)[-1].split(os.sep)[1:]
        )

        files[:] = [
            f for f in files
            if os.path.splitext(f)[1] in extensions
            and not f.startswith(".")
        ]

        dirs[:] = [
            d for d in dirs
            if not d.startswith(".")
        ]

        for _file in files:
            file_id = "/".join(root_folders + [_file])
            file_path = os.path.join(root, _file)

            module_id, environment = get_module_environment(
                file_id, files, environment
            )

            environment = get_file_environment(
                file_path, file_id, module_id, environment
            )

    return environment


def get_module_environment(file_id, files, environment=None):
    """Return module ID and updated environment from *file_id*.

    *file_id* is in the form of::

        relative/path/to/file.js

    *files* is the list of files at the same level of the *file_id* analysed.

    Update the *environment* is available and return it as-is if the file
    is not readable.

    """
    if environment is None:
        environment = {
            "modules": {}
        }

    hierarchy = file_id.split("/")
    file_name = hierarchy.pop()

    if file_name == "index.js":
        module_id = ".".join(hierarchy)
        module_name = guess_module_name(
            hierarchy[-1],
            hierarchy_folders=hierarchy[:-1],
            module_names=environment["modules"].keys()
{'fraction_non_alphanumeric': 0.14895535848489402, 'fraction_numerical': 0.0021043138433789266, 'mean_word_length': 2.6340797378481704, 'pattern_counts': {'":': 3, '<': 37, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1053939', 'n_tokens_mistral': 2198, 'n_tokens_neox': 2066, 'n_words': 505}