<commit_before>import logging
import os
import re
import uuid
from urlparse import urljoin
from StringIO import StringIO

import yaml
from fabric import api as fab
from fabric.state import env
from fabric.tasks import execute

from util import download_file, download_file_contents, digest_file

logging.basicConfig()
logger = logging.getLogger('dse')
logger.setLevel(logging.INFO)


# I don't like global ...
global config
global dse_builds, dse_cache, dse_tarball

DSE_NODE_TYPE_TO_STARTUP_PARAM = {'spark': '-k',
                                  'hadoop': '-t',
                                  'cassandra': '',
                                  'search': '-s',
                                  'spark-hadoop': '-k -t',
                                  'search-analytics': '-s -k'}

# this is used with pgrep / pkill
DSE_PROCESS_CHECK_STR = '.*java.*dse.*cassandra|.*java.*dse.*spark|.*java.*dse.*executor'

name = 'dse'


def setup(cfg):
    "Local setup for dse"

    global config, dse_builds, dse_cache, dse_tarball

    config = cfg

    if 'dse_url' not in config:
        raise ValueError("dse_url is missing from cluster_config.json.")

    # Create dse_cache_dir
    dse_builds = os.path.expanduser("~/fab/dse_builds")
    dse_cache = os.path.join(dse_builds, '_cache')
    if not os.path.exists(dse_cache):
        os.makedirs(dse_cache)

    download_tarball = True

    revision = config['revision']
    logger.info('Using DSE revision: {rev}'.format(rev=revision))
    if revision.startswith('bdp/'):
        download_tarball = False
        oauth_token = config.get('dse_source_build_oauth_token', None)
        if not config.get('dse_source_build_oauth_token', None):
            raise ValueError('dse_source_build_oauth_token for checking out a DSE branch is missing from cluster_config.json.')

        if not config.get('dse_source_build_artifactory_username', None):
            raise ValueError('dse_source_build_artifactory_username for building a DSE branch is missing from cluster_config.json.')

        if not config.get('dse_source_build_artifactory_password', None):
            raise ValueError('dse_source_build_artifactory_password for building a DSE branch is missing from cluster_config.json.')

        if not config.get('dse_source_build_artifactory_url', None):
            raise ValueError('dse_source_build_artifactory_url for building a DSE branch is missing from cluster_config.json.')

        _checkout_dse_branch_and_build_tarball_from_source(branch=revision[4:])

    if download_tarball:
        dse_tarball = "dse-{revision}-bin.tar.gz".format(revision=revision)
        download_binaries()

<commit_msg>CSTAR-632: Allow for use of existing DSE tarball rather than downloading new binary<commit_after>import logging
import os
import re
import uuid
from urlparse import urljoin
from StringIO import StringIO
import shutil
import yaml
from fabric import api as fab
from fabric.state import env
from fabric.tasks import execute

from util import download_file, download_file_contents, digest_file

logging.basicConfig()
logger = logging.getLogger('dse')
logger.setLevel(logging.INFO)


# I don't like global ...
global config
global dse_builds, dse_cache, dse_tarball

DSE_NODE_TYPE_TO_STARTUP_PARAM = {'spark': '-k',
                                  'hadoop': '-t',
                                  'cassandra': '',
                                  'search': '-s',
                                  'spark-hadoop': '-k -t',
                                  'search-analytics': '-s -k'}

# this is used with pgrep / pkill
DSE_PROCESS_CHECK_STR = '.*java.*dse.*cassandra|.*java.*dse.*spark|.*java.*dse.*executor'

name = 'dse'


def setup(cfg):
    "Local setup for dse"

    global config, dse_builds, dse_cache, dse_tarball

    config = cfg

    if 'dse_url' not in config:
        raise ValueError("dse_url is missing from cluster_config.json.")

    # Create dse_cache_dir
    dse_builds = os.path.expanduser("~/fab/dse_builds")
    dse_cache = os.path.join(dse_builds, '_cache')
    if not os.path.exists(dse_cache):
        os.makedirs(dse_cache)

    download_tarball = True
    # use this config setting to supply a full path to a dse tarball that already exists on the
    # cstar_perf tool host. See https://datastax.jira.com/browse/CSTAR-632 for details
    use_existing_tarball = config.get('use_existing_tarball', '')

    revision = config['revision']
    logger.info('Using DSE revision: {rev}'.format(rev=revision))
    if revision.startswith('bdp/'):
        download_tarball = False
        oauth_token = config.get('dse_source_build_oauth_token', None)
        if not config.get('dse_source_build_oauth_token', None):
            raise ValueError('dse_source_build_oauth_token for checking out a DSE branch is missing from cluster_config.json.')

        if not config.get('dse_source_build_artifactory_username', None):
            raise ValueError('dse_source_build_artifactory_username for building a DSE branch is missing from cluster_config.json.')

        if not config.get('dse_source_build_artifactory_password', None):
            raise ValueError('dse_source_build_artifactory_password for building a DSE branch is missing from cluster_config.json.')

        if not config.get('dse_source_build_artifactory_url', None):
            raise ValueError('dse_source_build_artifactory_url for building a DSE branch is missing from cluster_config.json.')

        _checkout_dse_branch_and_build_tarball_from_source(branch=revision[4:])

    if use_existing_tarball:
        if not os.path.exists(use_existing_tarball):
            raise IOError('Could not find {existing_tarball}, '
                          'please make sure it exists'.format(existing_tarball=use_existing_tarball))
        dse_tarball = os.path.basename(use_existing_tarball)
        if re.match('.*?-bin\.tar\.gz$', dse_tarball) is None:
            raise UserWarning('The supplied existing tarball must be of the form *-bin.tar.gz')
        shutil.copy(use_existing_tarball, dse_cache)
        download_tarball = False

    if download_tarball:
        dse_tarball = "dse-{revision}-bin.tar.gz".format(revision=revision)
        download_binaries()


{'fraction_non_alphanumeric': 0.09404742700435555, 'fraction_numerical': 0.0012905307307630263, 'mean_word_length': 3.073587385019711, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5694271', 'n_tokens_mistral': 1945, 'n_tokens_neox': 1895, 'n_words': 464}