<commit_before># this file is responsible for submitting tests into the queue
# by generating combinations of facets found in
# https://github.com/ceph/ceph-qa-suite.git

import copy
from datetime import datetime
import itertools
import logging
import os
import requests
import pwd
import subprocess
import smtplib
import sys
import yaml
from email.mime.text import MIMEText
from tempfile import NamedTemporaryFile

import teuthology
from . import lock
from .config import config
from .repo_utils import checkout_repo, BranchNotFoundError

log = logging.getLogger(__name__)


def main(args):
    verbose = args['--verbose']
    if verbose:
        teuthology.log.setLevel(logging.DEBUG)
    dry_run = args['--dry-run']

    base_yaml_paths = args['<config_yaml>']
    suite = args['--suite']
    nice_suite = suite.replace('/', ':')
    ceph_branch = args['--ceph']
    kernel_branch = args['--kernel']
    kernel_flavor = args['--flavor']
    teuthology_branch = args['--teuthology-branch']
    machine_type = args['--machine-type']
    distro = args['--distro']

    limit = int(args['--limit'])
    priority = int(args['--priority'])
    num = int(args['--num'])
    owner = args['--owner']
    email = args['--email']
    if email:
        config.email_specified = True
        config.results_email = email
    timeout = args['--timeout']

    name = make_run_name(nice_suite, ceph_branch, kernel_branch, kernel_flavor,
                         machine_type)

    suite_repo_path = fetch_suite_repo(ceph_branch, test_name=name)

    config_string = create_initial_config(nice_suite, ceph_branch,
                                          teuthology_branch, kernel_branch,
                                          kernel_flavor, distro, machine_type)

    with NamedTemporaryFile(prefix='schedule_suite_',
                            delete=False) as base_yaml:
        base_yaml.write(config_string)
        base_yaml_path = base_yaml.name
    base_yaml_paths.insert(0, base_yaml_path)
    prepare_and_schedule(owner=owner,
                         name=name,
                         suite=suite,
                         machine_type=machine_type,
                         suite_repo_path=suite_repo_path,
                         base_yaml_paths=base_yaml_paths,
                         email=email,
                         priority=priority,
                         limit=limit,
                         num=num,
                         timeout=timeout,
                         dry_run=dry_run,
                         verbose=verbose,
                         )
    os.remove(base_yaml_path)


def make_run_name(suite, ceph_branch, kernel_branch, kernel_flavor,
                  machine_type, user=None, timestamp=None):
    """
    Generate a run name based on the parameters. A run name looks like:
        teuthology-2014-06-23_19:00:37-rados-dumpling-testing-basic-plana
    """
    if not user:
        user = pwd.getpwuid(os.getuid()).pw_name
    # We assume timestamp is a datetime.datetime object
    if not timestamp:
        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')

    worker = get_worker(machine_type)
    return '-'.join(
        [user, str(timestamp), suite, ceph_branch,
         kernel_branch, kernel_flavor, worker]
    )


def fetch_suite_repo(branch, test_name):
    """
    Fetch the suite repo (and also the teuthology repo) so that we can use it
    to build jobs. Repos are stored in ~/src/.

    :returns: The path to the repo on disk
    """
    src_base_path = os.path.expanduser('~/src')
    if not os.path.exists(src_base_path):
        os.mkdir(src_base_path)
    suite_repo_path = os.path.join(src_base_path,
                                   'ceph-qa-suite_' + branch)
    try:
        checkout_repo(
            repo_url=os.path.join(config.ceph_git_base_url, 'teuthology.git'),
            dest_path=os.path.join(src_base_path, 'teuthology'),
            branch='master')
        checkout_repo(
            repo_url=os.path.join(config.ceph_git_base_url,
                                  'ceph-qa-suite.git'),
            dest_path=suite_repo_path,
            branch=branch)
    except BranchNotFoundError as exc:
        schedule_fail(message=str(exc), name=test_name)
    return suite_repo_path<commit_msg>Add a note about teuthology scheduling<commit_after># this file is responsible for submitting tests into the queue
# by generating combinations of facets found in
# https://github.com/ceph/ceph-qa-suite.git

import copy
from datetime import datetime
import itertools
import logging
import os
import requests
import pwd
import subprocess
import smtplib
import sys
import yaml
from email.mime.text import MIMEText
from tempfile import NamedTemporaryFile

import teuthology
from . import lock
from .config import config
from .repo_utils import checkout_repo, BranchNotFoundError

log = logging.getLogger(__name__)


def main(args):
    verbose = args['--verbose']
    if verbose:
        teuthology.log.setLevel(logging.DEBUG)
    dry_run = args['--dry-run']

    base_yaml_paths = args['<config_yaml>']
    suite = args['--suite']
    nice_suite = suite.replace('/', ':')
    ceph_branch = args['--ceph']
    kernel_branch = args['--kernel']
    kernel_flavor = args['--flavor']
    teuthology_branch = args['--teuthology-branch']
    machine_type = args['--machine-type']
    distro = args['--distro']

    limit = int(args['--limit'])
    priority = int(args['--priority'])
    num = int(args['--num'])
    owner = args['--owner']
    email = args['--email']
    if email:
        config.email_specified = True
        config.results_email = email
    timeout = args['--timeout']

    name = make_run_name(nice_suite, ceph_branch, kernel_branch, kernel_flavor,
                         machine_type)

    suite_repo_path = fetch_suite_repo(ceph_branch, test_name=name)

    config_string = create_initial_config(nice_suite, ceph_branch,
                                          teuthology_branch, kernel_branch,
                                          kernel_flavor, distro, machine_type)

    with NamedTemporaryFile(prefix='schedule_suite_',
                            delete=False) as base_yaml:
        base_yaml.write(config_string)
        base_yaml_path = base_yaml.name
    base_yaml_paths.insert(0, base_yaml_path)
    prepare_and_schedule(owner=owner,
                         name=name,
                         suite=suite,
                         machine_type=machine_type,
                         suite_repo_path=suite_repo_path,
                         base_yaml_paths=base_yaml_paths,
                         email=email,
                         priority=priority,
                         limit=limit,
                         num=num,
                         timeout=timeout,
                         dry_run=dry_run,
                         verbose=verbose,
                         )
    os.remove(base_yaml_path)


def make_run_name(suite, ceph_branch, kernel_branch, kernel_flavor,
                  machine_type, user=None, timestamp=None):
    """
    Generate a run name based on the parameters. A run name looks like:
        teuthology-2014-06-23_19:00:37-rados-dumpling-testing-basic-plana
    """
    if not user:
        user = pwd.getpwuid(os.getuid()).pw_name
    # We assume timestamp is a datetime.datetime object
    if not timestamp:
        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')

    worker = get_worker(machine_type)
    return '-'.join(
        [user, str(timestamp), suite, ceph_branch,
         kernel_branch, kernel_flavor, worker]
    )


def fetch_suite_repo(branch, test_name):
    """
    Fetch the suite repo (and also the teuthology repo) so that we can use it
    to build jobs. Repos are stored in ~/src/.

    The reason the teuthology repo is also fetched is that currently we use
    subprocess to call teuthology-schedule to schedule jobs so we need to make
    sure it is up-to-date. For that reason we always fetch the master branch
    for test scheduling, regardless of what teuthology branch is requested for
    testing.

    :returns: The path to the repo on disk
    """
    src_base_path = os.path.expanduser('~/src')
    if not os.path.exists(src_base_path):
        os.mkdir(src_base_path)
    suite_repo_path = os.path.join(src_base_path,
                                   'ceph-qa-suite_' + branch)
    try:
        checkout_repo(
            repo_url=os.path.join(config.ceph_git_base_url, 'teuthology.git'),
            dest_path=os.path.join(src_base_path, 'teuthology'),
            branch='master')
        checkout_repo(
            repo_url=os.path.join(config.ceph_git_base_url,
                                  'ceph-qa-suite.git'),
            dest_path=suite_repo_path,
            branch=branch)
    except BranchNotFoundError as exc:
        schedule_fail(message=str(exc), name=test_name)
    return suite_repo_path
{'fraction_non_alphanumeric': 0.09454300471592185, 'fraction_numerical': 0.0033685156074556477, 'mean_word_length': 2.213203463203463, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6232257', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2540, 'n_words': 588}