<commit_before># -*- coding: utf-8 -*-
"Main module defining filesystem and file classes"
from __future__ import absolute_import

import ctypes
import logging
import os
import sys
import re
import warnings
from collections import deque
from .lib import _lib

PY3 = sys.version_info.major > 2

from .compatibility import FileNotFoundError, urlparse, ConnectionError
from .utils import read_block, seek_delimiter


logger = logging.getLogger(__name__)


def hdfs_conf():
    """ Load HDFS config from default locations. """
    confd = os.environ.get('HADOOP_CONF_DIR', os.environ.get('HADOOP_INSTALL',
                           '') + '/hadoop/conf')
    files = 'core-site.xml', 'hdfs-site.xml'
    conf = {}
    for afile in files:
        try:
            conf.update(conf_to_dict(os.sep.join([confd, afile])))
        except FileNotFoundError:
            pass
    if 'fs.defaultFS' in conf:
        u = urlparse(conf['fs.defaultFS'])  # pragma: no cover
        if u.hostname != None:
            conf['host'] = u.hostname  # pragma: no cover
        if u.port != None:
            conf['port'] = u.port  # pragma: no cover
    return conf


def conf_to_dict(fname):
    """ Read a hdfs-site.xml style conf file, produces dictionary """
    name_match = re.compile("<name>(.*?)</name>")
    val_match = re.compile("<value>(.*?)</value>")
    conf = {}
    for line in open(fname):
        name = name_match.search(line)
        if name:
            key = name.groups()[0]
        val = val_match.search(line)
        if val:
            val = val.groups()[0]
            conf[key] = val
    return conf
<commit_msg>added optional parameter for default conf directory<commit_after># -*- coding: utf-8 -*-
"Main module defining filesystem and file classes"
from __future__ import absolute_import

import ctypes
import logging
import os
import sys
import re
import warnings
from collections import deque
from .lib import _lib

PY3 = sys.version_info.major > 2

from .compatibility import FileNotFoundError, urlparse, ConnectionError
from .utils import read_block, seek_delimiter


logger = logging.getLogger(__name__)


def hdfs_conf(conf_dir='/hadoop/conf'):
    """ Load HDFS config from default locations. """
    confd = os.environ.get('HADOOP_CONF_DIR', os.environ.get('HADOOP_INSTALL',
                           '') + conf_dir)
    files = 'core-site.xml', 'hdfs-site.xml'
    conf = {}
    for afile in files:
        try:
            conf.update(conf_to_dict(os.sep.join([confd, afile])))
        except FileNotFoundError:
            pass
    if 'fs.defaultFS' in conf:
        u = urlparse(conf['fs.defaultFS'])  # pragma: no cover
        if u.hostname != None:
            conf['host'] = u.hostname  # pragma: no cover
        if u.port != None:
            conf['port'] = u.port  # pragma: no cover
    return conf


def conf_to_dict(fname):
    """ Read a hdfs-site.xml style conf file, produces dictionary """
    name_match = re.compile("<name>(.*?)</name>")
    val_match = re.compile("<value>(.*?)</value>")
    conf = {}
    for line in open(fname):
        name = name_match.search(line)
        if name:
            key = name.groups()[0]
        val = val_match.search(line)
        if val:
            val = val.groups()[0]
            conf[key] = val
    return conf

{'fraction_non_alphanumeric': 0.1223808077740662, 'fraction_numerical': 0.0030367446097783178, 'mean_word_length': 2.5533980582524274, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4281733', 'n_tokens_mistral': 1015, 'n_tokens_neox': 1014, 'n_words': 282}