<commit_before>#!/usr/bin/env python
from __future__ import print_function
__metaclass__ = type


from argparse import ArgumentParser
import errno
import logging
import os
import random
import re
import string
import subprocess
import sys
import yaml

from jujuconfig import (
    get_environments_path,
    get_jenv_path,
    get_juju_home,
)
from jujupy import (
    Environment,
)
from utility import (
    PortTimeoutError,
    scoped_environ,
    temp_dir,
    until_timeout,
    wait_for_port,
)


def prepare_environment(environment, already_bootstrapped, machines):
    """Prepare an environment for deployment.

    As well as bootstrapping, this ensures the correct agent version is in
    use.

    :param environment: The name of the environment to use.
    :param already_bootstrapped: If true, the environment is already
        bootstrapped.
    """
    if sys.platform == 'win32':
        # Ensure OpenSSH is never in the path for win tests.
        sys.path = [p for p in sys.path if 'OpenSSH' not in p]
    env = Environment.from_config(environment)
    if not already_bootstrapped:
        env.bootstrap()
    agent_version = env.get_matching_agent_version()
    for ignored in until_timeout(30):
        agent_versions = env.get_status().get_agent_versions()
        if 'unknown' not in agent_versions and len(agent_versions) == 1:
            break
    if agent_versions.keys() != [agent_version]:
        print("Current versions: %s" % ', '.join(agent_versions.keys()))
        env.juju('upgrade-juju', '--version', agent_version)
    env.wait_for_version(env.get_matching_agent_version())
    for machine in machines:
        env.juju('add-machine', 'ssh:' + machine)
    return env


def destroy_environment(environment):
    if environment.config['type'] != 'manual':
        environment.destroy_environment()
    else:
        destroy_job_instances(os.environ['JOB_NAME'])


def destroy_job_instances(job_name):
    instances = list(get_job_instances(job_name))
    if len(instances) == 0:
        return
    subprocess.check_call(['euca-terminate-instances'] + instances)


def run_instances(count):
    environ = dict(os.environ)
    environ.update({
        'INSTANCE_TYPE': 'm1.large',
        'AMI_IMAGE': 'ami-bd6d40d4',
    })
    command = ['ec2-run-instance-get-id', '-g', 'manual-juju-test']
    machine_ids = [subprocess.check_output(command, env=environ).strip()
                   for x in range(count)]
    subprocess.call(['ec2-tag-job-instances'] + machine_ids)
    machine_names = [
        subprocess.check_output(['ec2-get-name', instance]).strip()
        for instance in machine_ids]
    return machine_names


def deploy_dummy_stack(env, charm_prefix):
    """"Deploy a dummy stack in the specified environment.
    """
    allowed_chars = string.ascii_uppercase + string.digits
    token=''.join(random.choice(allowed_chars) for n in range(20))
    env.deploy(charm_prefix + 'dummy-source')
    env.juju('set', 'dummy-source', 'token=%s' % token)
    env.deploy(charm_prefix + 'dummy-sink')
    env.juju('add-relation', 'dummy-source', 'dummy-sink')
    env.juju('expose', 'dummy-sink')
    env.wait_for_started()
    # Wait up to 30 seconds for token to be created.
    logging.info('Retrieving token.')
    get_token="""
        for x in $(seq 30); do
          if [ -f /var/run/dummy-sink/token ]; then
            if [ "$(cat /var/run/dummy-sink/token)" != "" ]; then
              break
            fi
          fi
          sleep 1
        done
        cat /var/run/dummy-sink/token
    """
    result = env.client.get_juju_output(env, 'ssh', 'dummy-sink/0', get_token)
    result = re.match(r'([^\n\r]*)\r?\n?', result).group(1)
    if result != token:
        raise ValueError('Token is %r' % result)


def scp_logs(log_names, directory):
    subprocess.check_call(['timeout', '5m', 'scp'] + log_names + [directory])


def dump_logs(env, host, directory):
    log_names = []
    if env.local:
        local = os.path.join(get_juju_home(), 'local')
        log_names = [os.path.join(local, 'cloud-init-output.log')]
        log_dir = os.path.join(local, 'log')
        log_names.extend(os.path.join(log_dir, l) for l
                         in os.listdir(log_dir) if l.endswith('.log'))
        scp_logs(log_names, directory)
    else:
        log_names = [
            'ubuntu@%s:/var/log/%s' % (host, n)
            for n in ['juju/all-machines.log', 'cloud-init-output.log']]
        try:
            wait_for_port(host, 22, timeout=60)
        except PortTimeoutError:<commit_msg>SCP logs with the ignore-host-verification options.<commit_after>#!/usr/bin/env python
from __future__ import print_function
__metaclass__ = type


from argparse import ArgumentParser
import errno
import logging
import os
import random
import re
import string
import subprocess
import sys
import yaml

from jujuconfig import (
    get_environments_path,
    get_jenv_path,
    get_juju_home,
)
from jujupy import (
    Environment,
)
from utility import (
    PortTimeoutError,
    scoped_environ,
    temp_dir,
    until_timeout,
    wait_for_port,
)


def prepare_environment(environment, already_bootstrapped, machines):
    """Prepare an environment for deployment.

    As well as bootstrapping, this ensures the correct agent version is in
    use.

    :param environment: The name of the environment to use.
    :param already_bootstrapped: If true, the environment is already
        bootstrapped.
    """
    if sys.platform == 'win32':
        # Ensure OpenSSH is never in the path for win tests.
        sys.path = [p for p in sys.path if 'OpenSSH' not in p]
    env = Environment.from_config(environment)
    if not already_bootstrapped:
        env.bootstrap()
    agent_version = env.get_matching_agent_version()
    for ignored in until_timeout(30):
        agent_versions = env.get_status().get_agent_versions()
        if 'unknown' not in agent_versions and len(agent_versions) == 1:
            break
    if agent_versions.keys() != [agent_version]:
        print("Current versions: %s" % ', '.join(agent_versions.keys()))
        env.juju('upgrade-juju', '--version', agent_version)
    env.wait_for_version(env.get_matching_agent_version())
    for machine in machines:
        env.juju('add-machine', 'ssh:' + machine)
    return env


def destroy_environment(environment):
    if environment.config['type'] != 'manual':
        environment.destroy_environment()
    else:
        destroy_job_instances(os.environ['JOB_NAME'])


def destroy_job_instances(job_name):
    instances = list(get_job_instances(job_name))
    if len(instances) == 0:
        return
    subprocess.check_call(['euca-terminate-instances'] + instances)


def run_instances(count):
    environ = dict(os.environ)
    environ.update({
        'INSTANCE_TYPE': 'm1.large',
        'AMI_IMAGE': 'ami-bd6d40d4',
    })
    command = ['ec2-run-instance-get-id', '-g', 'manual-juju-test']
    machine_ids = [subprocess.check_output(command, env=environ).strip()
                   for x in range(count)]
    subprocess.call(['ec2-tag-job-instances'] + machine_ids)
    machine_names = [
        subprocess.check_output(['ec2-get-name', instance]).strip()
        for instance in machine_ids]
    return machine_names


def deploy_dummy_stack(env, charm_prefix):
    """"Deploy a dummy stack in the specified environment.
    """
    allowed_chars = string.ascii_uppercase + string.digits
    token=''.join(random.choice(allowed_chars) for n in range(20))
    env.deploy(charm_prefix + 'dummy-source')
    env.juju('set', 'dummy-source', 'token=%s' % token)
    env.deploy(charm_prefix + 'dummy-sink')
    env.juju('add-relation', 'dummy-source', 'dummy-sink')
    env.juju('expose', 'dummy-sink')
    env.wait_for_started()
    # Wait up to 30 seconds for token to be created.
    logging.info('Retrieving token.')
    get_token="""
        for x in $(seq 30); do
          if [ -f /var/run/dummy-sink/token ]; then
            if [ "$(cat /var/run/dummy-sink/token)" != "" ]; then
              break
            fi
          fi
          sleep 1
        done
        cat /var/run/dummy-sink/token
    """
    result = env.client.get_juju_output(env, 'ssh', 'dummy-sink/0', get_token)
    result = re.match(r'([^\n\r]*)\r?\n?', result).group(1)
    if result != token:
        raise ValueError('Token is %r' % result)


def scp_logs(log_names, directory):
    subprocess.check_call([
        'timeout', '5m', 'scp', '-o', 'UserKnownHostsFile', '/dev/null',
        '-o', 'StrictHostKeyChecking', 'no'] +
        log_names + [directory])


def dump_logs(env, host, directory):
    log_names = []
    if env.local:
        local = os.path.join(get_juju_home(), 'local')
        log_names = [os.path.join(local, 'cloud-init-output.log')]
        log_dir = os.path.join(local, 'log')
        log_names.extend(os.path.join(log_dir, l) for l
                         in os.listdir(log_dir) if l.endswith('.log'))
        scp_logs(log_names, directory)
    else:
        log_names = [
            'ubuntu@%s:/var/log/%s' % (host, n)
            for n in ['juju/all-machines.log', 'cloud-init-output.log']]
        try:
            wait_for_port(host, 22, timeout=60)
        except PortTimeoutError:
{'fraction_non_alphanumeric': 0.1263146481622032, 'fraction_numerical': 0.006071777079041526, 'mean_word_length': 2.9117896522476676, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '296924', 'n_tokens_mistral': 3049, 'n_tokens_neox': 2950, 'n_words': 687}