<commit_before>import re
import os
import socket
from pyparsing import Forward, Keyword, QuotedString, Word, Literal, Suppress, Regex, Optional, SkipTo, ZeroOrMore, \
    Group, lineno, col, TokenConverter, replaceWith, alphanums
from pyparsing import ParserElement
from pyhocon.config_tree import ConfigTree, ConfigSubstitution, ConfigList, ConfigValues, ConfigUnquotedString, \
    ConfigInclude, NoneValue
from pyhocon.exceptions import ConfigSubstitutionException, ConfigMissingException
import logging

use_urllib2 = False
try:
    # For Python 3.0 and later
    from urllib.request import urlopen
    from urllib.error import HTTPError
except ImportError:
    # Fall back to Python 2's urllib2
    from urllib2 import urlopen, HTTPError

    use_urllib2 = True
try:
    basestring
except NameError:
    basestring = str

logger = logging.getLogger(__name__)


class ConfigFactory(object):
    @staticmethod
    def parse_file(filename, required=True, resolve=True):
        """Parse file

        :param filename: filename
        :type filename: basestring
        :param required: If true, raises an exception if can't load file
        :type required: boolean
        :param resolve: If true, resolve substitutions
        :type resolve: boolean
        :return: Config object
        :type return: Config
        """
        try:
            with open(filename, 'r') as fd:
                content = fd.read()
                return ConfigFactory.parse_string(content, os.path.dirname(filename), resolve)
        except IOError as e:
            if required:
                raise e
            logger.warn('Cannot include file %s. File does not exist or cannot be read.', filename)
            return []

    @staticmethod
    def parse_URL(url, timeout=None, resolve=True):
        """Parse URL

        :param url: url to parse
        :type url: basestring
        :param resolve: If true, resolve substitutions
        :type resolve: boolean
        :return: Config object
        :type return: Config
        """
        socket_timeout = socket._GLOBAL_DEFAULT_TIMEOUT if timeout is None else timeout
        try:
            fd = urlopen(url, timeout=socket_timeout)
            content = fd.read() if use_urllib2 else fd.read().decode('utf-8')
            return ConfigFactory.parse_string(content, os.path.dirname(url), resolve)
        except HTTPError:
            logger.warn('Cannot include url %s. Resource is inaccessible.', url)
        finally:
            fd.close()

    @staticmethod
    def parse_string(content, basedir=None, resolve=True):
        """Parse URL

        :param content: content to parse
        :type content: basestring
        :param resolve: If true, resolve substitutions
        :type resolve: boolean
        :return: Config object
        :type return: Config
        """
        return ConfigParser().parse(content, basedir, resolve)

    @classmethod
    def from_dict(cls, dictionary):
        """Convert dictionary (and ordered dictionary) into a ConfigTree
        :param dictionary: dictionary to convert
        :type dictionary: dict
        :return: Config object
        :type return: Config
        """

        def create_tree(value):
            if isinstance(value, dict):
                res = ConfigTree()
                for key, child_value in value.items():
                    res.put(key, create_tree(child_value))
                return res
            if isinstance(value, list):<commit_msg>handle unreachable URL include<commit_after>import re
import os
import socket
import contextlib
from pyparsing import Forward, Keyword, QuotedString, Word, Literal, Suppress, Regex, Optional, SkipTo, ZeroOrMore, \
    Group, lineno, col, TokenConverter, replaceWith, alphanums
from pyparsing import ParserElement
from pyhocon.config_tree import ConfigTree, ConfigSubstitution, ConfigList, ConfigValues, ConfigUnquotedString, \
    ConfigInclude, NoneValue
from pyhocon.exceptions import ConfigSubstitutionException, ConfigMissingException
import logging

use_urllib2 = False
try:
    # For Python 3.0 and later
    from urllib.request import urlopen
    from urllib.error import HTTPError
except ImportError:
    # Fall back to Python 2's urllib2
    from urllib2 import urlopen, HTTPError

    use_urllib2 = True
try:
    basestring
except NameError:
    basestring = str

logger = logging.getLogger(__name__)


class ConfigFactory(object):
    @staticmethod
    def parse_file(filename, required=True, resolve=True):
        """Parse file

        :param filename: filename
        :type filename: basestring
        :param required: If true, raises an exception if can't load file
        :type required: boolean
        :param resolve: If true, resolve substitutions
        :type resolve: boolean
        :return: Config object
        :type return: Config
        """
        try:
            with open(filename, 'r') as fd:
                content = fd.read()
                return ConfigFactory.parse_string(content, os.path.dirname(filename), resolve)
        except IOError as e:
            if required:
                raise e
            logger.warn('Cannot include file %s. File does not exist or cannot be read.', filename)
            return []

    @staticmethod
    def parse_URL(url, timeout=None, resolve=True):
        """Parse URL

        :param url: url to parse
        :type url: basestring
        :param resolve: If true, resolve substitutions
        :type resolve: boolean
        :return: Config object
        :type return: Config
        """
        socket_timeout = socket._GLOBAL_DEFAULT_TIMEOUT if timeout is None else timeout

        try:
            with contextlib.closing(urlopen(url, timeout=socket_timeout)) as fd:
                content = fd.read() if use_urllib2 else fd.read().decode('utf-8')
                return ConfigFactory.parse_string(content, os.path.dirname(url), resolve)
        except HTTPError:
            logger.warn('Cannot include url %s. Resource is inaccessible.', url)
            return []

    @staticmethod
    def parse_string(content, basedir=None, resolve=True):
        """Parse URL

        :param content: content to parse
        :type content: basestring
        :param resolve: If true, resolve substitutions
        :type resolve: boolean
        :return: Config object
        :type return: Config
        """
        return ConfigParser().parse(content, basedir, resolve)

    @classmethod
    def from_dict(cls, dictionary):
        """Convert dictionary (and ordered dictionary) into a ConfigTree
        :param dictionary: dictionary to convert
        :type dictionary: dict
        :return: Config object
        :type return: Config
        """

        def create_tree(value):
            if isinstance(value, dict):
                res = ConfigTree()
                for key, child_value in value.items():
                    res.put(key, create_tree(child_value))
                return res
            if isinstance(value, list):
{'fraction_non_alphanumeric': 0.0765050765050765, 'fraction_numerical': 0.002574002574002574, 'mean_word_length': 2.4658077304261643, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4004936', 'n_tokens_mistral': 1843, 'n_tokens_neox': 1791, 'n_words': 619}