<commit_before># BEGIN_COPYRIGHT
#
# Copyright 2009-2014 CRS4.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# END_COPYRIGHT

# DEV NOTE: this module is used by the setup script, so it MUST be
# importable even if Pydoop has not been installed (yet).

"""
Tools for retrieving information on the available Hadoop
installation(s): version, classpath, etc.

Known issues: from-tarball CDH installation is not supported.
"""

import os
import glob
import re
import platform
import subprocess as sp
import xml.dom.minidom as dom
from xml.parsers.expat import ExpatError

try:
    from pydoop.config import DEFAULT_HADOOP_HOME
except ImportError:  # should only happen at compile time
    DEFAULT_HADOOP_HOME = None
SYSTEM = platform.system().lower()


def first_dir_in_glob(pattern):
    for path in sorted(glob.glob(pattern)):
        if os.path.isdir(path):
            return path


CDH_HADOOP_HOME_PKG = '/usr/lib/hadoop'  # Cloudera bin packages
CDH_HADOOP_HOME_PARCEL = first_dir_in_glob(
    '/opt/cloudera/parcels/CDH-*/lib/hadoop'  # Cloudera Manager
)


class HadoopVersionError(Exception):

    def __init__(self, version_str):
        super(HadoopVersionError, self).__init__(
            'unrecognized version string format: %r' % (version_str,)
        )


class HadoopXMLError(Exception):
    pass


def get_arch():
    #if SYSTEM == 'darwin':
    #  return "", ""
    bits, _ = platform.architecture()
    if bits == "64bit":
        return "amd64", "64"
    return "i386", "32"


def _cdh_hadoop_home():
    if os.path.isdir(CDH_HADOOP_HOME_PKG):
        return CDH_HADOOP_HOME_PKG
    elif os.path.isdir(CDH_HADOOP_HOME_PARCEL or ''):
        return CDH_HADOOP_HOME_PARCEL
    raise RuntimeError("unsupported CDH deployment")


def _jars_from_dirs(dirs):
    jars = []
    for d in dirs:
        jars.extend(glob.glob(os.path.join(d, '*.jar')))
    return jars


def _hadoop1_jars(hadoop_home):
    return _jars_from_dirs([hadoop_home, os.path.join(hadoop_home, 'lib')])


def _hadoop2_jars(hadoop_home):
    jar_root = os.path.join(hadoop_home, 'share', 'hadoop')
    return _jars_from_dirs([os.path.join(jar_root, d) for d in (
        'hdfs',
        'common',
        os.path.join('common', 'lib'),
        'mapreduce',
        'yarn',  # hadoop >= 2.2.0
    )])


def _cdh_jars(hadoop_home, is_yarn):

    hadoop_hdfs = hadoop_home + "-hdfs"
    hadoop_yarn = hadoop_home + "-yarn"
    hadoop_mapred_v1 = hadoop_home + "-0.20-mapreduce"
    hadoop_mapred_v2 = hadoop_home + "-mapreduce"

    if not is_yarn:
        dirs = [hadoop_home, os.path.join(hadoop_home, "lib"),
                hadoop_hdfs, os.path.join(hadoop_hdfs, "lib"),
                hadoop_yarn, os.path.join(hadoop_yarn, "lib"),
                hadoop_mapred_v1, os.path.join(hadoop_mapred_v1, "lib")]
    else:
        dirs = [hadoop_home, os.path.join(hadoop_home, "lib"),
                hadoop_hdfs, os.path.join(hadoop_hdfs, "lib"),
                hadoop_yarn, os.path.join(hadoop_yarn, "lib"),
                hadoop_mapred_v1, os.path.join(hadoop_mapred_v2, "lib")]

    jars = _jars_from_dirs(dirs)
    return jars


class HadoopVersion(object):
    """
    Stores Hadoop version information.

    Hadoop version strings are in the <MAIN>-<REST> format, where <MAIN>
    is in the typical dot-separated integers format, while <REST> is
    subject to a higher degree of variation.  Examples: '0.20.2',
    '0.20.203.0', '0.20.2-cdh3u4', '1.0.4-SNAPSHOT', '2.0.0-mr1-cdh4.1.0'.<commit_msg>Fixed hadoop classpath for mrv2<commit_after># BEGIN_COPYRIGHT
#
# Copyright 2009-2014 CRS4.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# END_COPYRIGHT

# DEV NOTE: this module is used by the setup script, so it MUST be
# importable even if Pydoop has not been installed (yet).

"""
Tools for retrieving information on the available Hadoop
installation(s): version, classpath, etc.

Known issues: from-tarball CDH installation is not supported.
"""

import os
import glob
import re
import platform
import subprocess as sp
import xml.dom.minidom as dom
from xml.parsers.expat import ExpatError

try:
    from pydoop.config import DEFAULT_HADOOP_HOME
except ImportError:  # should only happen at compile time
    DEFAULT_HADOOP_HOME = None
SYSTEM = platform.system().lower()


def first_dir_in_glob(pattern):
    for path in sorted(glob.glob(pattern)):
        if os.path.isdir(path):
            return path


CDH_HADOOP_HOME_PKG = '/usr/lib/hadoop'  # Cloudera bin packages
CDH_HADOOP_HOME_PARCEL = first_dir_in_glob(
    '/opt/cloudera/parcels/CDH-*/lib/hadoop'  # Cloudera Manager
)


class HadoopVersionError(Exception):

    def __init__(self, version_str):
        super(HadoopVersionError, self).__init__(
            'unrecognized version string format: %r' % (version_str,)
        )


class HadoopXMLError(Exception):
    pass


def get_arch():
    #if SYSTEM == 'darwin':
    #  return "", ""
    bits, _ = platform.architecture()
    if bits == "64bit":
        return "amd64", "64"
    return "i386", "32"


def _cdh_hadoop_home():
    if os.path.isdir(CDH_HADOOP_HOME_PKG):
        return CDH_HADOOP_HOME_PKG
    elif os.path.isdir(CDH_HADOOP_HOME_PARCEL or ''):
        return CDH_HADOOP_HOME_PARCEL
    raise RuntimeError("unsupported CDH deployment")


def _jars_from_dirs(dirs):
    jars = []
    for d in dirs:
        jars.extend(glob.glob(os.path.join(d, '*.jar')))
    return jars


def _hadoop1_jars(hadoop_home):
    return _jars_from_dirs([hadoop_home, os.path.join(hadoop_home, 'lib')])


def _hadoop2_jars(hadoop_home):
    jar_root = os.path.join(hadoop_home, 'share', 'hadoop')
    return _jars_from_dirs([os.path.join(jar_root, d) for d in (
        'hdfs',
        'common',
        os.path.join('common', 'lib'),
        'mapreduce',
        'yarn',  # hadoop >= 2.2.0
    )])


def _cdh_jars(hadoop_home, is_yarn):

    hadoop_hdfs = hadoop_home + "-hdfs"
    hadoop_yarn = hadoop_home + "-yarn"
    hadoop_mapred_v1 = hadoop_home + "-0.20-mapreduce"
    hadoop_mapred_v2 = hadoop_home + "-mapreduce"

    if not is_yarn:
        dirs = [hadoop_home, os.path.join(hadoop_home, "lib"),
                hadoop_hdfs, os.path.join(hadoop_hdfs, "lib"),
                hadoop_yarn, os.path.join(hadoop_yarn, "lib"),
                hadoop_mapred_v1, os.path.join(hadoop_mapred_v1, "lib")]
    else:
        dirs = [hadoop_home, os.path.join(hadoop_home, "lib"),
                hadoop_hdfs, os.path.join(hadoop_hdfs, "lib"),
                hadoop_yarn, os.path.join(hadoop_yarn, "lib"),
                hadoop_mapred_v2, os.path.join(hadoop_mapred_v2, "lib")]

    jars = _jars_from_dirs(dirs)
    return jars


class HadoopVersion(object):
    """
    Stores Hadoop version information.

    Hadoop version strings are in the <MAIN>-<REST> format, where <MAIN>
    is in the typical dot-separated integers format, while <REST> is
    subject to a higher degree of variation.  Examples: '0.20.2',
    '0.20.203.0', '0.20.2-cdh3u4', '1.0.4-SNAPSHOT', '2.0.0-mr1-cdh4.1.0'.
{'fraction_non_alphanumeric': 0.1116954670673679, 'fraction_numerical': 0.016403706486351116, 'mean_word_length': 3.291778613648576, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5794530', 'n_tokens_mistral': 2897, 'n_tokens_neox': 2738, 'n_words': 717}