<commit_before># Copyright 2019 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Executes the 3 phases of Teasort phases on a Apache Hadoop MapReduce cluster.

TeraSort is a popular benchmark that measures the amount of time to sort a
configured amount of randomly distributed data on a given cluster. It is
commonly used to measure MapReduce performance of an Apache Hadoop cluster.
The following report compares performance of a YARN-scheduled TeraSort job on

A full TeraSort benchmark run consists of the following three steps:

* Generating the input data via TeraGen.
* Running the actual TeraSort on the input data.
* Validating the sorted output data via TeraValidate.

The benchmark reports the detailed latency of executing each phase.
"""
from __future__ import division

import logging

from perfkitbenchmarker import configs
from perfkitbenchmarker import dpb_service
from perfkitbenchmarker import errors
from perfkitbenchmarker import flags
from perfkitbenchmarker import sample
from perfkitbenchmarker.dpb_service import BaseDpbService

BENCHMARK_NAME = 'dpb_terasort_benchmark'

BENCHMARK_CONFIG = """
dpb_terasort_benchmark:
  description: Run terasort on dataproc and emr
  dpb_service:
    service_type: unmanaged_dpb_svc_yarn_cluster
    worker_group:
      vm_spec:
        GCP:
          machine_type: n1-standard-4
        AWS:
          machine_type: m5.xlarge
      disk_spec:
        GCP:
          disk_size: 500
          disk_type: pd-standard
          mount_point: /scratch_ts
        AWS:
          disk_size: 500
          disk_type: st1
          mount_point: /scratch_ts
    worker_count: 2
"""

_FS_TYPE_EPHEMERAL = 'ephemeral'
_FS_TYPE_PERSISTENT = 'persistent'

flags.DEFINE_enum('dpb_terasort_storage_type', _FS_TYPE_PERSISTENT,
                  [_FS_TYPE_EPHEMERAL, _FS_TYPE_PERSISTENT],
                  'The type of storage for executing the Terasort benchmark')
flags.DEFINE_integer('dpb_terasort_num_records', 10000,
                     'Number of 100-byte rows to generate.')
flags.DEFINE_bool(
    'dpb_terasort_pre_cleanup', False,
    'Cleanup the terasort directories on the specified filesystem.')

FLAGS = flags.FLAGS

SUPPORTED_DPB_BACKENDS = [dpb_service.DATAPROC, dpb_service.EMR,
                          dpb_service.UNMANAGED_DPB_SVC_YARN_CLUSTER]<commit_msg>Add support for running hadoop on Azure VMs<commit_after># Copyright 2019 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Executes the 3 phases of Teasort phases on a Apache Hadoop MapReduce cluster.

TeraSort is a popular benchmark that measures the amount of time to sort a
configured amount of randomly distributed data on a given cluster. It is
commonly used to measure MapReduce performance of an Apache Hadoop cluster.
The following report compares performance of a YARN-scheduled TeraSort job on

A full TeraSort benchmark run consists of the following three steps:

* Generating the input data via TeraGen.
* Running the actual TeraSort on the input data.
* Validating the sorted output data via TeraValidate.

The benchmark reports the detailed latency of executing each phase.
"""
from __future__ import division

import logging

from perfkitbenchmarker import configs
from perfkitbenchmarker import dpb_service
from perfkitbenchmarker import errors
from perfkitbenchmarker import flags
from perfkitbenchmarker import sample
from perfkitbenchmarker.dpb_service import BaseDpbService

BENCHMARK_NAME = 'dpb_terasort_benchmark'

BENCHMARK_CONFIG = """
dpb_terasort_benchmark:
  description: Run terasort on dataproc and emr
  dpb_service:
    service_type: unmanaged_dpb_svc_yarn_cluster
    worker_group:
      vm_spec:
        GCP:
          machine_type: n1-standard-4
        AWS:
          machine_type: m5.xlarge
        Azure:
          machine_type: Standard_F4s_v2
      disk_spec:
        GCP:
          disk_size: 500
          disk_type: pd-standard
          mount_point: /scratch_ts
        AWS:
          disk_size: 500
          disk_type: st1
          mount_point: /scratch_ts
        Azure:
          disk_size: 500
          disk_type: Standard_LRS
          mount_point: /scratch_ts
    worker_count: 2
"""

_FS_TYPE_EPHEMERAL = 'ephemeral'
_FS_TYPE_PERSISTENT = 'persistent'

flags.DEFINE_enum('dpb_terasort_storage_type', _FS_TYPE_PERSISTENT,
                  [_FS_TYPE_EPHEMERAL, _FS_TYPE_PERSISTENT],
                  'The type of storage for executing the Terasort benchmark')
flags.DEFINE_integer('dpb_terasort_num_records', 10000,
                     'Number of 100-byte rows to generate.')
flags.DEFINE_bool(
    'dpb_terasort_pre_cleanup', False,
    'Cleanup the terasort directories on the specified filesystem.')

FLAGS = flags.FLAGS

SUPPORTED_DPB_BACKENDS = [dpb_service.DATAPROC, dpb_service.EMR,
                          dpb_service.UNMANAGED_DPB_SVC_YARN_CLUSTER]
{'fraction_non_alphanumeric': 0.050775978407557355, 'fraction_numerical': 0.010290148448043185, 'mean_word_length': 3.599689681923972, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3107622', 'n_tokens_mistral': 1798, 'n_tokens_neox': 1710, 'n_words': 616}