<commit_before>import os
import runpy
from itertools import chain

from setuptools import find_packages, setup


BASE_DIR = os.path.dirname(os.path.abspath(__file__))
OPTIONAL_DEPENDENCIES = ("autograd", "tensorflow", "theano")


def parse_requirements_file(filename):
    with open(filename) as f:
        return f.read().splitlines()


if __name__ == "__main__":
    requirements = parse_requirements_file("requirements/base.txt")

    install_requires = []
    optional_dependencies = {}
    for requirement in requirements:
        # We manually separate out hard from optional dependencies.
        if any((d in requirement for d in OPTIONAL_DEPENDENCIES)):
            package = requirement.split(">")[0].split("=")[0]
            optional_dependencies[package] = [requirement]
        else:
            install_requires.append(requirement)

    dev_requirements = parse_requirements_file("requirements/dev.txt")
    extras_require = {
        "test": dev_requirements,
        **optional_dependencies
    }<commit_msg>setup.py: use regex to extract package name<commit_after>import os
import re
import runpy
from itertools import chain

from setuptools import find_packages, setup


BASE_DIR = os.path.dirname(os.path.abspath(__file__))
OPTIONAL_DEPENDENCIES = ("autograd", "tensorflow", "theano")


def parse_requirements_file(filename):
    with open(filename) as f:
        return [line.strip() for line in f.read().splitlines()]


if __name__ == "__main__":
    requirements = parse_requirements_file("requirements/base.txt")

    install_requires = []
    optional_dependencies = {}
    for requirement in requirements:
        # We manually separate hard from optional dependencies.
        if any(requirement.startswith(optional_dependency)
               for optional_dependency in OPTIONAL_DEPENDENCIES):
            package = re.match(r"([A-Za-z0-9\-_]+).*", requirement).group(1)
            optional_dependencies[package] = [requirement]
        else:
            install_requires.append(requirement)

    dev_requirements = parse_requirements_file("requirements/dev.txt")
    extras_require = {
        "test": dev_requirements,
        **optional_dependencies
    }
{'fraction_non_alphanumeric': 0.10109789569990851, 'fraction_numerical': 0.0022872827081427266, 'mean_word_length': 3.623678646934461, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5333978', 'n_tokens_mistral': 642, 'n_tokens_neox': 627, 'n_words': 142}