<commit_before>import sys
import argparse
import subprocess

queue = 'staff'
__version__ = '2.5.0'

big_description = """Hadoop and related software cluster tester. Given the
fast-paced nature of Hadoop, Spark, Hive, and other technologies, running a
test suite after an upgrade can ensure that all expected functionality
continues to work.  This script will generate and upload data to
$HOME/trozamon_testing in HDFS."""

def hdfs_mkdir(dir):
    if subprocess.call('hdfs dfs -test -d ' + dir, shell=True) != 0:
        if subprocess.call('hdfs dfs -mkdir ' + dir, shell=True) != 0:
            print('Failed to create ' + dir + ' in HDFS')
            return 1
    return 0

def hdfs_put(local, remote):
    if subprocess.call('hdfs dfs -test -f ' + remote, shell=True) != 0:
        if subprocess.call('hdfs dfs -put ' + local + ' ' + remote,
                shell=True) != 0:
            print('Failed to upload ' + local + ' to ' + remote)
            return 1
    return 0

def hdfs_rmdir(dir):
    if subprocess.call('hdfs dfs -test -d ' + dir, shell=True) == 0:
        return subprocess.call('hdfs dfs -rm -r ' + dir, shell=True)
    return 0

def test_hadoop_java():
    cmd = ' '.join([
        'yarn jar hadoop/target/hadoop-examples-hadoop-' + __version__ + '.jar',
        'com.alectenharmsel.research.LineCount',
        '-Dmapreduce.job.queuename=' + queue,
        'trozamon_testing/input',
        'trozamon_testing/output/hadoop_java']
        )

    if hdfs_rmdir('trozamon_testing/output/hadoop_java') != 0:
        return 1

    print('Testing Hadoop Java by running:')
    print(cmd)

    return subprocess.call(cmd, shell=True)

def test_hadoop_streaming():
    print("Testing Hadoop Streaming")
    return 0

def test_pig():
    print("Testing Pig")
    return 0

def test_spark():
    print("Testing Spark")
    return 0

def test_pyspark():
    print("Testing PySpark")
    return 0

def test_hive():
    print("Testing Hive")
    return 0

def run():
    parser = argparse.ArgumentParser(description=big_description)

    parser.add_argument('--all',
            action='store_true',
            help='Run all codes that would be run by using all possible flags')
    parser.add_argument('--hadoop-java',
            action='store_true',
            help='Run a selection of Java codes written for Hadoop')
    parser.add_argument('--hadoop-streaming',
            action='store_true',
            help='Run a selection of Python codes written for Hadoop')
    parser.add_argument('--pig',
            action='store_true',
            help='Run a selection of Pig scripts')
    parser.add_argument('--spark',
            action='store_true',
            help='Run a selection of Scala codes written for Spark')
    parser.add_argument('--pyspark',
            action='store_true',
            help='Run a selection of Python codes written for Spark')
    parser.add_argument('--hive',
            action='store_true',
            help='Run a selection of Hive queries')

    parsed_args = parser.parse_args()

    tests = list()
    if parsed_args.all:
        parsed_args.hadoop_java = True
        parsed_args.hadoop_streaming = True
        parsed_args.pig = True
        parsed_args.hive = True
        parsed_args.spark = True
        parsed_args.pyspark = True

    if parsed_args.hadoop_java:
        tests.append(test_hadoop_java)
    if parsed_args.hadoop_streaming:
        tests.append(test_hadoop_streaming)
    if parsed_args.pig:
        tests.append(test_pig)
    if parsed_args.hive:
        tests.append(test_hive)
    if parsed_args.pyspark:
        tests.append(test_pyspark)
    if parsed_args.spark:
        tests.append(test_spark)

    print("Running \"mvn package -DskipTests\"...")
    if subprocess.call("mvn package -DskipTests", shell=True) != 0:
        print("Maven build failed. Are you running this in the root " +
                "directory of the repo?")
        return 1

    print("Uploading pom.xml to trozamon_testing/input/pom.xml...")
    err = hdfs_mkdir('trozamon_testing')
    if err != 0:
        return err
    err = hdfs_mkdir('trozamon_testing/input')
    if err != 0:
        return err
    err = hdfs_mkdir('trozamon_testing/output')
    if err != 0:
        return err
    err = hdfs_put('pom.xml', 'trozamon_testing/input/pom.xml')
    if err != 0:
        return err

    print("Starting to run tests...")
    for test in tests:
        res = test()
        if res != 0:
            print(str(test) + ': FAILURE')
            return res

    return 0

if __name__ == "__main__":
    sys.exit(run())<commit_msg>Add Hadoop Streaming test to cluster_test<commit_after>import sys
import argparse
import subprocess

queue = 'staff'
__version__ = '2.5.0'

big_description = """Hadoop and related software cluster tester. Given the
fast-paced nature of Hadoop, Spark, Hive, and other technologies, running a
test suite after an upgrade can ensure that all expected functionality
continues to work.  This script will generate and upload data to
$HOME/trozamon_testing in HDFS."""

def hdfs_mkdir(dir):
    if subprocess.call('hdfs dfs -test -d ' + dir, shell=True) != 0:
        if subprocess.call('hdfs dfs -mkdir ' + dir, shell=True) != 0:
            print('Failed to create ' + dir + ' in HDFS')
            return 1
    return 0

def hdfs_put(local, remote):
    if subprocess.call('hdfs dfs -test -f ' + remote, shell=True) != 0:
        if subprocess.call('hdfs dfs -put ' + local + ' ' + remote,
                shell=True) != 0:
            print('Failed to upload ' + local + ' to ' + remote)
            return 1
    return 0

def hdfs_rmdir(dir):
    if subprocess.call('hdfs dfs -test -d ' + dir, shell=True) == 0:
        return subprocess.call('hdfs dfs -rm -r ' + dir, shell=True)
    return 0

def test_hadoop_java():
    cmd = ' '.join([
        'yarn jar hadoop/target/hadoop-examples-hadoop-' + __version__ + '.jar',
        'com.alectenharmsel.research.LineCount',
        '-Dmapreduce.job.queuename=' + queue,
        'trozamon_testing/input',
        'trozamon_testing/output/hadoop_java'
        ])

    if hdfs_rmdir('trozamon_testing/output/hadoop_java') != 0:
        return 1

    print('Testing Hadoop Java by running:')
    print(cmd)

    return subprocess.call(cmd, shell=True)

def test_hadoop_streaming():
    cmd = ' '.join([
        'yarn jar /usr/lib/hadoop-mapreduce/hadoop-streaming.jar',
        '-Dmapreduce.job.queuename=' + queue,
        '-input trozamon_testing/input',
        '-output trozamon_testing/output/hadoop_streaming',
        '-mapper python/srctok-map.py -reducer python/sum.py',
        '-file python/srctok-map.py -file python/sum.py'
        ])

    if hdfs_rmdir('trozamon_testing/output/hadoop_streaming') != 0:
        return 1

    print('Testing Hadoop Streaming by running:')
    print(cmd)

    return subprocess.call(cmd, shell=True)

def test_pig():
    print("Testing Pig")
    return 0

def test_spark():
    print("Testing Spark")
    return 0

def test_pyspark():
    print("Testing PySpark")
    return 0

def test_hive():
    print("Testing Hive")
    return 0

def run():
    parser = argparse.ArgumentParser(description=big_description)

    parser.add_argument('--all',
            action='store_true',
            help='Run all codes that would be run by using all possible flags')
    parser.add_argument('--hadoop-java',
            action='store_true',
            help='Run a selection of Java codes written for Hadoop')
    parser.add_argument('--hadoop-streaming',
            action='store_true',
            help='Run a selection of Python codes written for Hadoop')
    parser.add_argument('--pig',
            action='store_true',
            help='Run a selection of Pig scripts')
    parser.add_argument('--spark',
            action='store_true',
            help='Run a selection of Scala codes written for Spark')
    parser.add_argument('--pyspark',
            action='store_true',
            help='Run a selection of Python codes written for Spark')
    parser.add_argument('--hive',
            action='store_true',
            help='Run a selection of Hive queries')

    parsed_args = parser.parse_args()

    tests = list()
    if parsed_args.all:
        parsed_args.hadoop_java = True
        parsed_args.hadoop_streaming = True
        parsed_args.pig = True
        parsed_args.hive = True
        parsed_args.spark = True
        parsed_args.pyspark = True

    if parsed_args.hadoop_java:
        tests.append(test_hadoop_java)
    if parsed_args.hadoop_streaming:
        tests.append(test_hadoop_streaming)
    if parsed_args.pig:
        tests.append(test_pig)
    if parsed_args.hive:
        tests.append(test_hive)
    if parsed_args.pyspark:
        tests.append(test_pyspark)
    if parsed_args.spark:
        tests.append(test_spark)

    print("Running \"mvn package -DskipTests\"...")
    if subprocess.call("mvn package -DskipTests", shell=True) != 0:
        print("Maven build failed. Are you running this in the root " +
                "directory of the repo?")
        return 1

    print("Uploading pom.xml to trozamon_testing/input/pom.xml...")
    err = hdfs_mkdir('trozamon_testing')
    if err != 0:
        return err
    err = hdfs_mkdir('trozamon_testing/input')
    if err != 0:
        return err
    err = hdfs_mkdir('trozamon_testing/output')
    if err != 0:
        return err
    err = hdfs_put('pom.xml', 'trozamon_testing/input/pom.xml')
    if err != 0:
        return err

    print("Starting to run tests...")
    for test in tests:
        res = test()
        if res != 0:
            print(test.__name__ + ': FAILURE')
            return res

    return 0

if __name__ == "__main__":
    sys.exit(run())
{'fraction_non_alphanumeric': 0.11353352529823119, 'fraction_numerical': 0.005861785273549979, 'mean_word_length': 2.5057678442682048, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1178887', 'n_tokens_mistral': 3139, 'n_tokens_neox': 3005, 'n_words': 802}