<commit_before># Copyright 2014-2015 Canonical Limited.
#
# This file is part of jujubigdata.
#
# jujubigdata is free software: you can redistribute it and/or modify
# it under the terms of the Apache License version 2.0.
#
# jujubigdata is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# Apache License for more details.

from subprocess import check_output
import time

from path import Path

import jujuresources

from charmhelpers.core import host
from charmhelpers.core import hookenv
from charmhelpers.core import unitdata
from charmhelpers.core.charmframework import helpers

from jujubigdata import utils


class HadoopBase(object):
    def __init__(self, dist_config):
        self.dist_config = dist_config
        self.charm_config = hookenv.config()
        self.cpu_arch = host.cpu_arch()

        # dist_config will have simple validation done on primary keys in the
        # dist.yaml, but we need to ensure deeper values are present.
        required_dirs = ['hadoop', 'hadoop_conf', 'hdfs_log_dir',
                         'yarn_log_dir']
        missing_dirs = set(required_dirs) - set(self.dist_config.dirs.keys())
        if missing_dirs:
            raise ValueError('dirs option in {} is missing required entr{}: {}'.format(
                self.dist_config.yaml_file,
                'ies' if len(missing_dirs) > 1 else 'y',
                ', '.join(missing_dirs)))

        self.client_spec = {
            'hadoop': self.dist_config.hadoop_version,
        }
        self.verify_conditional_resources = utils.verify_resources('hadoop-%s' % self.cpu_arch)

    def spec(self):
        """
        Generate the full spec for keeping charms in sync.

        NB: This has to be a callback instead of a plain property because it is
        passed to the relations during construction of the Manager but needs to
        properly reflect the Java version in the same hook invocation that installs
        Java.
        """
        java_version = unitdata.kv().get('java.version')
        if java_version:
            return {
                'vendor': self.dist_config.vendor,
                'hadoop': self.dist_config.hadoop_version,
                'java': java_version,
                'arch': self.cpu_arch,
            }
        else:
            return None

    def is_installed(self):
        return unitdata.kv().get('hadoop.base.installed')

    def install(self, force=False):
        if not force and self.is_installed():
            return
        self.configure_hosts_file()
        self.dist_config.add_users()
        self.dist_config.add_dirs()
        self.dist_config.add_packages()
        self.install_base_packages()
        self.setup_hadoop_config()
        self.configure_hadoop()
        unitdata.kv().set('hadoop.base.installed', True)
        unitdata.kv().flush(True)

    def configure_hosts_file(self):
        """
        Add the unit's private-address to /etc/hosts to ensure that Java
        can resolve the hostname of the server to its real IP address.
        We derive our hostname from the unit_id, replacing / with -.
        """
        local_ip = utils.resolve_private_address(hookenv.unit_get('private-address'))
        hostname = hookenv.local_unit().replace('/', '-')
        utils.update_etc_hosts({local_ip: hostname})

    def install_base_packages(self):
        with utils.disable_firewall():
            self.install_java()
            self.install_hadoop()

    def install_java(self):
        """
        Run the java-installer resource to install Java and determine
        the JAVA_HOME and Java version.

        The java-installer must be idempotent and its only output (on stdout)
        should be two lines: the JAVA_HOME path, and the Java version, respectively.

        If there is an error installing Java, the installer should exit
        with a non-zero exit code.
        """
        env = utils.read_etc_env()
        java_installer = Path(jujuresources.resource_path('java-installer'))
        java_installer.chmod(0o755)
        output = check_output([java_installer], env=env)<commit_msg>Restore /etc/hostname update accidentally removed in previous commit<commit_after># Copyright 2014-2015 Canonical Limited.
#
# This file is part of jujubigdata.
#
# jujubigdata is free software: you can redistribute it and/or modify
# it under the terms of the Apache License version 2.0.
#
# jujubigdata is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# Apache License for more details.

from subprocess import check_call, check_output
import time

from path import Path

import jujuresources

from charmhelpers.core import host
from charmhelpers.core import hookenv
from charmhelpers.core import unitdata
from charmhelpers.core.charmframework import helpers

from jujubigdata import utils


class HadoopBase(object):
    def __init__(self, dist_config):
        self.dist_config = dist_config
        self.charm_config = hookenv.config()
        self.cpu_arch = host.cpu_arch()

        # dist_config will have simple validation done on primary keys in the
        # dist.yaml, but we need to ensure deeper values are present.
        required_dirs = ['hadoop', 'hadoop_conf', 'hdfs_log_dir',
                         'yarn_log_dir']
        missing_dirs = set(required_dirs) - set(self.dist_config.dirs.keys())
        if missing_dirs:
            raise ValueError('dirs option in {} is missing required entr{}: {}'.format(
                self.dist_config.yaml_file,
                'ies' if len(missing_dirs) > 1 else 'y',
                ', '.join(missing_dirs)))

        self.client_spec = {
            'hadoop': self.dist_config.hadoop_version,
        }
        self.verify_conditional_resources = utils.verify_resources('hadoop-%s' % self.cpu_arch)

    def spec(self):
        """
        Generate the full spec for keeping charms in sync.

        NB: This has to be a callback instead of a plain property because it is
        passed to the relations during construction of the Manager but needs to
        properly reflect the Java version in the same hook invocation that installs
        Java.
        """
        java_version = unitdata.kv().get('java.version')
        if java_version:
            return {
                'vendor': self.dist_config.vendor,
                'hadoop': self.dist_config.hadoop_version,
                'java': java_version,
                'arch': self.cpu_arch,
            }
        else:
            return None

    def is_installed(self):
        return unitdata.kv().get('hadoop.base.installed')

    def install(self, force=False):
        if not force and self.is_installed():
            return
        self.configure_hosts_file()
        self.dist_config.add_users()
        self.dist_config.add_dirs()
        self.dist_config.add_packages()
        self.install_base_packages()
        self.setup_hadoop_config()
        self.configure_hadoop()
        unitdata.kv().set('hadoop.base.installed', True)
        unitdata.kv().flush(True)

    def configure_hosts_file(self):
        """
        Add the unit's private-address to /etc/hosts to ensure that Java
        can resolve the hostname of the server to its real IP address.
        We derive our hostname from the unit_id, replacing / with -.
        """
        local_ip = utils.resolve_private_address(hookenv.unit_get('private-address'))
        hostname = hookenv.local_unit().replace('/', '-')
        utils.update_etc_hosts({local_ip: hostname})

        # update name of host to more semantically meaningful value
        etc_hostname = Path('/etc/hostname')
        etc_hostname.write_text(hostname)
        check_call(['hostname', '-F', etc_hostname])

    def install_base_packages(self):
        with utils.disable_firewall():
            self.install_java()
            self.install_hadoop()

    def install_java(self):
        """
        Run the java-installer resource to install Java and determine
        the JAVA_HOME and Java version.

        The java-installer must be idempotent and its only output (on stdout)
        should be two lines: the JAVA_HOME path, and the Java version, respectively.

        If there is an error installing Java, the installer should exit
        with a non-zero exit code.
        """
        env = utils.read_etc_env()
        java_installer = Path(jujuresources.resource_path('java-installer'))
        java_installer.chmod(0o755)
        output = check_output([java_installer], env=env)
{'fraction_non_alphanumeric': 0.082548794489093, 'fraction_numerical': 0.003444316877152698, 'mean_word_length': 2.531009323064451, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1842132', 'n_tokens_mistral': 2428, 'n_tokens_neox': 2355, 'n_words': 744}