<commit_before>
import os.path
import pprint
import re
import setuptools
import sys
try:
    import ConfigParser
except ImportError:
    import configparser as ConfigParser


# The path to the classifiers and the config
classifiers_path = 'setup.classifiers.txt'
config_path = 'setup.config.ini'


# Setup-object
class Setup(object):

    def __init__(self, base_path, classifiers_path, config_path):
        """
        Construct setup
        :param base_path:           The base path
        :param classifiers_path:    Path to the classifiers
        :param config_path:         Path to the config
        """

        self.base_path = base_path

        self.classifiers = self._load_classifiers(classifiers_path)
        self.config = self._load_config(config_path)

    def _load_classifiers(self, classifiers_path):
        """
        Load classifiers
        :param classifiers_path:    Path to the classifiers
        :return:                    list
        """

        classifiers = []
        with open(os.path.join(self.base_path, classifiers_path)) as classifiers_file:
            for classifier_line in classifiers_file:
                classifier_match = re.search('^([^#]*)', classifier_line)
                if not classifier_match:
                    continue
                classifier = classifier_match.group(1).strip()
                if not classifier:
                    continue
                classifiers.append(classifier)

        return classifiers

    def _load_config(self, config_path):
        """
        Load the config
        :param config_path: Path to the config
        :return:            Config-dict
        """

        config_parser = ConfigParser.ConfigParser()<commit_msg>Fetch setup.classifiers.txt dynamically.<commit_after>
import os.path
import pprint
import re
import setuptools
import sys
try:
    import ConfigParser
except ImportError:
    import configparser as ConfigParser
try:
    from urllib2 import urlopen, URLError
except ImportError:
    from urllib.request import urlopen
    from urllib.error import URLError


# The path to the classifiers and the config
classifiers_path = 'setup.classifiers.txt'
config_path = 'setup.config.ini'


# Setup-object
class Setup(object):

    def __init__(self, base_path, classifiers_path, config_path):
        """
        Construct setup
        :param base_path:           The base path
        :param classifiers_path:    Path to the classifiers
        :param config_path:         Path to the config
        """

        self.base_path = base_path

        self.classifiers = self._load_classifiers(classifiers_path)
        self.config = self._load_config(config_path)

    def _load_classifiers(self, classifiers_path):
        """
        Load classifiers
        :param classifiers_path:    Path to the classifiers
        :return:                    list
        """

        classifiers_path = os.path.join(self.base_path, classifiers_path)

        # Get classifiers if not already fetched
        if not os.path.isfile(classifiers_path):
            try:
                url = 'https://pypi.python.org/pypi?%3Aaction=list_classifiers'
                response = urlopen(url)
                classifiers_data = response.read().decode('utf-8')
            except URLError:
                self.exit('Could not fetch classifiers from \'%s\'' % url)

            with open(classifiers_path, 'w') as classifiers_file:
                classifiers_file.write('# See https://pypi.python.org/pypi?%3Aaction=list_classifiers\n')
                classifiers_file.write(classifiers_data)

        # Read classifiers
        classifiers = []
        with open(classifiers_path, 'r') as classifiers_file:
            for classifier_line in classifiers_file:
                classifier_match = re.search('^([^#]*)', classifier_line)
                if not classifier_match:
                    continue
                classifier = classifier_match.group(1).strip()
                if not classifier:
                    continue
                classifiers.append(classifier)

        if not classifiers:
            self.exit('No classifiers were found in \'%s\'\nDelete the file and try again.' % classifiers_path)

        return classifiers

    def _load_config(self, config_path):
        """
        Load the config
        :param config_path: Path to the config
        :return:            Config-dict
        """

        config_parser = ConfigParser.ConfigParser()
{'fraction_non_alphanumeric': 0.07723214285714286, 'fraction_numerical': 0.0013392857142857143, 'mean_word_length': 2.1335664335664335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '544532', 'n_tokens_mistral': 1241, 'n_tokens_neox': 1138, 'n_words': 311}