<commit_before>from cStringIO import StringIO
import contextlib
import logging
from teuthology import misc as teuthology
from teuthology import contextutil
from ..orchestra import run

log = logging.getLogger(__name__)

HADOOP_2x_URL = "http://apache.osuosl.org/hadoop/common/hadoop-2.5.2/hadoop-2.5.2.tar.gz"

def dict_to_hadoop_conf(items):
    out = "<configuration>\n"
    for key, value in items.iteritems():
        out += "  <property>\n"
        out += "    <name>" + key + "</name>\n"
        out += "    <value>" + value + "</value>\n"
        out += "  </property>\n"
    out += "</configuration>\n"
    return out

def get_slaves_data(ctx):
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/slaves".format(tdir=tempdir)
    nodes = ctx.cluster.only(teuthology.is_type('hadoop.slave'))
    hosts = [s.ssh.get_transport().getpeername()[0] for s in nodes.remotes]
    data = '\n'.join(hosts)
    return path, data

def get_masters_data(ctx):
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/masters".format(tdir=tempdir)
    nodes = ctx.cluster.only(teuthology.is_type('hadoop.master'))
    hosts = [s.ssh.get_transport().getpeername()[0] for s in nodes.remotes]
    data = '\n'.join(hosts)
    return path, data

def get_core_site_data(ctx, config):
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/core-site.xml".format(tdir=tempdir)
    nodes = ctx.cluster.only(teuthology.is_type('hadoop.master'))
    host = [s.ssh.get_transport().getpeername()[0] for s in nodes.remotes][0]

    conf = {}
    if config.get('hdfs', False):
        conf.update({
            'fs.defaultFS': 'hdfs://{namenode}:9000',
            'hadoop.tmp.dir': '{tdir}/hadoop_tmp',
        })
    else:
        conf.update({
            'fs.default.name': 'ceph://{namenode}:6789/',
            'fs.defaultFS': 'ceph://{namenode}:6789/',
            'ceph.conf.file': '/etc/ceph/ceph.conf',
            'ceph.mon.address': '{namenode}:6789',
            'ceph.auth.id': 'admin',
            #'ceph.data.pools': 'cephfs_data',
            'fs.AbstractFileSystem.ceph.impl': 'org.apache.hadoop.fs.ceph.CephFs',
            'fs.ceph.impl': 'org.apache.hadoop.fs.ceph.CephFileSystem',
        })

    data_tmpl = dict_to_hadoop_conf(conf)
    return path, data_tmpl.format(tdir=tempdir, namenode=host)

def get_mapred_site_data(ctx):
    data_tmpl = """
<configuration>
     <property>
         <name>mapred.job.tracker</name>
         <value>{namenode}:9001</value>
     </property>
     <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>
"""
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/mapred-site.xml".format(tdir=tempdir)
    nodes = ctx.cluster.only(teuthology.is_type('hadoop.master'))
    hosts = [s.ssh.get_transport().getpeername()[0] for s in nodes.remotes]
    assert len(hosts) == 1
    host = hosts[0]
    return path, data_tmpl.format(namenode=host)

def get_yarn_site_data(ctx):
    data_tmpl = """
<configuration>
<property>
 <name>yarn.resourcemanager.resourcetracker.address</name>
 <value>{namenode}:8025</value>  
</property>
<property>
 <name>yarn.resourcemanager.scheduler.address</name>
 <value>{namenode}:8030</value>  
</property>
<property>
 <name>yarn.resourcemanager.address</name>
 <value>{namenode}:8050</value>  
</property>
<property>
 <name>yarn.resourcemanager.admin.address</name>
 <value>{namenode}:8041</value>  
</property>
<property>
  <name>yarn.resourcemanager.hostname</name>
  <value>{namenode}</value>
</property>
<property>
  <name>yarn.nodemanager.aux-services</name>
  <value>mapreduce_shuffle</value>
</property>
</configuration>
"""
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/yarn-site.xml".format(tdir=tempdir)
    nodes = ctx.cluster.only(teuthology.is_type('hadoop.master'))
    hosts = [s.ssh.get_transport().getpeername()[0] for s in nodes.remotes]
    assert len(hosts) == 1
    host = hosts[0]
    return path, data_tmpl.format(namenode=host)

def get_hdfs_site_data(ctx):
    data = """
<configuration>
     <property>
         <name>dfs.replication</name>
         <value>1</value>
     </property>
</configuration>
"""
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/hdfs-site.xml".format(tdir=tempdir)
    return path, data

def configure(ctx, config, hadoops):
    tempdir = teuthology.get_testdir(ctx)

    log.info("Writing Hadoop slaves file...")
    for remote in hadoops.remotes:
        path, data = get_slaves_data(ctx)
        teuthology.write_file(remote, path, StringIO(data))
<commit_msg>hadoop: use dict-to-conf converter<commit_after>from cStringIO import StringIO
import contextlib
import logging
from teuthology import misc as teuthology
from teuthology import contextutil
from ..orchestra import run

log = logging.getLogger(__name__)

HADOOP_2x_URL = "http://apache.osuosl.org/hadoop/common/hadoop-2.5.2/hadoop-2.5.2.tar.gz"

def dict_to_hadoop_conf(items):
    out = "<configuration>\n"
    for key, value in items.iteritems():
        out += "  <property>\n"
        out += "    <name>" + key + "</name>\n"
        out += "    <value>" + value + "</value>\n"
        out += "  </property>\n"
    out += "</configuration>\n"
    return out

def get_slaves_data(ctx):
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/slaves".format(tdir=tempdir)
    nodes = ctx.cluster.only(teuthology.is_type('hadoop.slave'))
    hosts = [s.ssh.get_transport().getpeername()[0] for s in nodes.remotes]
    data = '\n'.join(hosts)
    return path, data

def get_masters_data(ctx):
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/masters".format(tdir=tempdir)
    nodes = ctx.cluster.only(teuthology.is_type('hadoop.master'))
    hosts = [s.ssh.get_transport().getpeername()[0] for s in nodes.remotes]
    data = '\n'.join(hosts)
    return path, data

def get_core_site_data(ctx, config):
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/core-site.xml".format(tdir=tempdir)
    nodes = ctx.cluster.only(teuthology.is_type('hadoop.master'))
    host = [s.ssh.get_transport().getpeername()[0] for s in nodes.remotes][0]

    conf = {}
    if config.get('hdfs', False):
        conf.update({
            'fs.defaultFS': 'hdfs://{namenode}:9000',
            'hadoop.tmp.dir': '{tdir}/hadoop_tmp',
        })
    else:
        conf.update({
            'fs.default.name': 'ceph://{namenode}:6789/',
            'fs.defaultFS': 'ceph://{namenode}:6789/',
            'ceph.conf.file': '/etc/ceph/ceph.conf',
            'ceph.mon.address': '{namenode}:6789',
            'ceph.auth.id': 'admin',
            #'ceph.data.pools': 'cephfs_data',
            'fs.AbstractFileSystem.ceph.impl': 'org.apache.hadoop.fs.ceph.CephFs',
            'fs.ceph.impl': 'org.apache.hadoop.fs.ceph.CephFileSystem',
        })

    data_tmpl = dict_to_hadoop_conf(conf)
    return path, data_tmpl.format(tdir=tempdir, namenode=host)

def get_mapred_site_data(ctx):
    data_tmpl = """
<configuration>
     <property>
         <name>mapred.job.tracker</name>
         <value>{namenode}:9001</value>
     </property>
     <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>
"""
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/mapred-site.xml".format(tdir=tempdir)
    nodes = ctx.cluster.only(teuthology.is_type('hadoop.master'))
    hosts = [s.ssh.get_transport().getpeername()[0] for s in nodes.remotes]
    assert len(hosts) == 1
    host = hosts[0]
    return path, data_tmpl.format(namenode=host)

def get_yarn_site_data(ctx):
    conf = {}
    conf.update({
        'yarn.resourcemanager.resourcetracker.address': '{namenode}:8025',
        'yarn.resourcemanager.scheduler.address': '{namenode}:8030',
        'yarn.resourcemanager.address': '{namenode}:8050',
        'yarn.resourcemanager.admin.address': '{namenode}:8041',
        'yarn.resourcemanager.hostname': '{namenode}',
        'yarn.nodemanager.aux-services': 'mapreduce_shuffle',
    })
    data_tmpl = dict_to_hadoop_conf(conf)

    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/yarn-site.xml".format(tdir=tempdir)
    nodes = ctx.cluster.only(teuthology.is_type('hadoop.master'))
    hosts = [s.ssh.get_transport().getpeername()[0] for s in nodes.remotes]
    assert len(hosts) == 1
    host = hosts[0]
    return path, data_tmpl.format(namenode=host)

def get_hdfs_site_data(ctx):
    data = """
<configuration>
     <property>
         <name>dfs.replication</name>
         <value>1</value>
     </property>
</configuration>
"""
    tempdir = teuthology.get_testdir(ctx)
    path = "{tdir}/hadoop/etc/hadoop/hdfs-site.xml".format(tdir=tempdir)
    return path, data

def configure(ctx, config, hadoops):
    tempdir = teuthology.get_testdir(ctx)

    log.info("Writing Hadoop slaves file...")
    for remote in hadoops.remotes:
        path, data = get_slaves_data(ctx)
        teuthology.write_file(remote, path, StringIO(data))

{'fraction_non_alphanumeric': 0.1631498803567544, 'fraction_numerical': 0.011746791385686317, 'mean_word_length': 3.31892907468295, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5254953', 'n_tokens_mistral': 3332, 'n_tokens_neox': 3254, 'n_words': 502}