<commit_before>#!/usr/bin/env python
#
# URL=https://raw.githubusercontent.com/rmyers/ci_scripts/master/run.py
# curl -s $URL | python

import json
import os
import sys
import urllib2
import pprint
import subprocess


def puts(line, **kwargs):
    if isinstance(line, basestring):
        print(line.format(**kwargs))
    else:
        pprint.pprint(line)
    sys.stdout.flush()


puts('\n\nENVIRONMENT:')
puts(dict(os.environ))

TOKEN = os.environ.get('GITHUB_TOKEN', 'Unknown')
GITHUB = 'https://github.rackspace.com/api/v3/repos/Cloud-Database'
TROVE_BIN = '/usr/share/python/trove/bin'
TROVE_PYTHON = os.path.join(TROVE_BIN, 'python')
FAB_CMD = os.path.join(TROVE_BIN, 'fab')
TROVE_FAB = '{fab} --fabfile=/Cloud-Database/fabfile.py'.format(fab=FAB_CMD)
TROVE_PIP = os.path.join(TROVE_BIN, 'pip')
PR_NUMBER = os.environ.get('PR', 'Unknown')
REPO = os.environ.get('REPO', 'Cloud-Database')
JOB_NAME = os.environ.get('JOB_NAME', 'Unknown')
BUILD_URL = os.environ.get('BUILD_URL', 'Unknown')
BUILD_ID = os.environ.get('BUILD_ID', 'Unknown')
WORKSPACE = os.environ.get('WORKSPACE', '/tmp')
DEBUG = os.environ.get('DEBUG', 'true') == 'true'
COMMAND = os.environ.get('COMMAND', 'test')
HEADERS = {'Authorization': 'token {}'.format(TOKEN)}
PREFIX = ''
# Mapping of repo to path
DIRECTORIES = {
    'Cloud-Database': WORKSPACE,
    'clouddb-puppet-v2': os.path.join(WORKSPACE, 'lib/clouddb-puppet-v2'),
}

# Submodules to update
SUBMODULES = [
    'lib/clouddb-puppet-v2',
    'lib/rackspace-monitoring',
    'lib/python-cloudlb',
    'lib/python-cinderclient',
    'lib/python-swiftclient',
    'lib/python-keystoneclient',
    'lib/python-glanceclient',
    'lib/private/python-novaclient',
    'lib/private/secrets',
]

# Log file locations
LOGS = [
    '/var/log/trove',
    '/var/log/nova',
    '/var/log/cinder',
    '/var/log/cdbproxy',
    '/var/log/cdbproxy-endpoint',
]


class cd(object):
    """Run commands in a different directory."""

    def __init__(self, path):
        global PREFIX
        self.original = PREFIX
        self.path = path

    def __enter__(self, *args, **kwargs):
        global PREFIX
        PREFIX = 'cd {path} && '.format(path=self.path)

    def __exit__(self, *args, **kwargs):
        global PREFIX
        PREFIX = self.original


def call(command, **kwargs):
    command = command.format(**kwargs)
    cmd = '{prefix}{command}'.format(prefix=PREFIX, command=command)
    puts(cmd)
    return subprocess.call(cmd, shell=True)


def check_call(command, **kwargs):
    command = command.format(**kwargs)
    cmd = '{prefix}{command}'.format(prefix=PREFIX, command=command)
    puts(cmd)
    return subprocess.check_call(cmd, shell=True)


def update_status(pr, state, job_name=JOB_NAME, build_url=BUILD_URL):
    url = pr['statuses_url']

    if state == 'pending':
        description = "Stashy says check back later..."
    elif state == 'success':
        description = "Stashy says you are golden!"
    elif state == 'failure':
        description = "Stashy says better luck next time :("

    data = {
        'state': state,
        'description': description,
        'context': job_name,
        'target_url': build_url,
    }
    if DEBUG:
        puts(data)

    puts('\nSetting test state to: {state}\n', state=state)
    request = urllib2.Request(url, headers=HEADERS)
    request.add_data(json.dumps(data))
    resp = urllib2.urlopen(request)


class TestCase(object):

    template = 'test_case.xml'

    def __init__(self, name, test_cmd, xunit=True):
        self.name = name
        self.test_cmd = test_cmd
        self.pr = None
        self.xunit = xunit
        self.state = 'pending'

    def fetch_pr(self, repo=REPO, number=PR_NUMBER):
        url = '{github}/{repo}/pulls/{number}'.format(
            github=GITHUB, number=number, repo=repo)
        request = urllib2.Request(url, headers=HEADERS)
        resp = urllib2.urlopen(request)
        pr = json.loads(resp.read())
        if DEBUG:
            puts('\n\nPR DATA:\n')
            puts(pr)
        return pr

    def update_submodules(self):
        with cd(WORKSPACE):
            for sub in SUBMODULES:
                check_call('git submodule update --init {sub}', sub=sub)

    def checkout_pr(self, pr):
        with cd(DIRECTORIES.get(pr['base']['repo']['name'])):
            branch = 'pull/{number}/head:{number}'.format(**pr)
            check_call('git fetch origin {branch} -f'.format(branch=branch))
            check_call('git checkout {number}', **pr)

    def setup(self):
        self.pr = self.fetch_pr()
        puts('\n\nPR {number} INFO:\n', **self.pr)
        puts('URL: {html_url}', **self.pr)
        puts('SHA: {sha}', sha=self.pr['head']['sha'])
        puts('TITLE: {title}', **self.pr)
        puts('INFO: \n{body}\n', **self.pr)
        if not self.pr['mergeable']:
            puts('\n\nTHIS PR CANNOT BE MERGED!!!\n\n')
            raise Exception('Pr Not mergeable')
        # TODO(rmyers): search body for 'DEPENDS ON:' and handle it too
        self.update_submodules()
        self.checkout_pr(self.pr)

    def clean(self):
        puts('\nCLEANING UP:\n')

    def run_tests(self):
        args = {
            'fab': TROVE_FAB,
            'pip': TROVE_PIP,
            'python': TROVE_PYTHON,
            'bin': TROVE_BIN,
            'workspace': WORKSPACE,
            'fab_args': 'tests:stop=True,white_box=True,with_xunit=True',
            'mysql_56': 'datastore=mysql,version=5.6',
            'mysql_51': 'datastore=mysql,version=5.1',
            'mariadb': 'datastore=mariadb,version=10',
            'percona': 'datastore=percona,version=5.6',
            'default_groups': 'group="rax_stable|rax_replication_groups"'
        }
        check_call(self.test_cmd, **args)

    def run(self):
        """Actually run the tests."""
        try:
            self.setup()
            update_status(self.pr, state=self.state)
            self.run_tests()
            self.state = 'success'
        except:
            self.state = 'failure'<commit_msg>Handle depends on links<commit_after>#!/usr/bin/env python
#
# URL=https://raw.githubusercontent.com/rmyers/ci_scripts/master/run.py
# curl -s $URL | python

import json
import os
import re
import sys
import urllib2
import pprint
import subprocess


def puts(line, **kwargs):
    if isinstance(line, basestring):
        print(line.format(**kwargs))
    else:
        pprint.pprint(line)
    sys.stdout.flush()


puts('\n\nENVIRONMENT:')
puts(dict(os.environ))

TOKEN = os.environ.get('GITHUB_TOKEN', 'Unknown')
GITHUB = 'https://github.rackspace.com/api/v3/repos/Cloud-Database'
TROVE_BIN = '/usr/share/python/trove/bin'
TROVE_PYTHON = os.path.join(TROVE_BIN, 'python')
FAB_CMD = os.path.join(TROVE_BIN, 'fab')
TROVE_FAB = '{fab} --fabfile=/Cloud-Database/fabfile.py'.format(fab=FAB_CMD)
TROVE_PIP = os.path.join(TROVE_BIN, 'pip')
PR_NUMBER = os.environ.get('PR', 'Unknown')
REPO = os.environ.get('REPO', 'Cloud-Database')
JOB_NAME = os.environ.get('JOB_NAME', 'Unknown')
BUILD_URL = os.environ.get('BUILD_URL', 'Unknown')
BUILD_ID = os.environ.get('BUILD_ID', 'Unknown')
WORKSPACE = os.environ.get('WORKSPACE', '/tmp')
DEBUG = os.environ.get('DEBUG', 'true') == 'true'
COMMAND = os.environ.get('COMMAND', 'test')
HEADERS = {'Authorization': 'token {}'.format(TOKEN)}
PREFIX = ''
DEPENDS = re.compile('https://github.rackspace.com/[^/]+/([^/]+)/pull/(\d+)')
# Mapping of repo to path
DIRECTORIES = {
    'Cloud-Database': WORKSPACE,
    'clouddb-puppet-v2': os.path.join(WORKSPACE, 'lib/clouddb-puppet-v2'),
}

# Submodules to update
SUBMODULES = [
    'lib/clouddb-puppet-v2',
    'lib/rackspace-monitoring',
    'lib/python-cloudlb',
    'lib/python-cinderclient',
    'lib/python-swiftclient',
    'lib/python-keystoneclient',
    'lib/python-glanceclient',
    'lib/private/python-novaclient',
    'lib/private/secrets',
]

# Log file locations
LOGS = [
    '/var/log/trove',
    '/var/log/nova',
    '/var/log/cinder',
    '/var/log/cdbproxy',
    '/var/log/cdbproxy-endpoint',
]


class cd(object):
    """Run commands in a different directory."""

    def __init__(self, path):
        global PREFIX
        self.original = PREFIX
        self.path = path

    def __enter__(self, *args, **kwargs):
        global PREFIX
        PREFIX = 'cd {path} && '.format(path=self.path)

    def __exit__(self, *args, **kwargs):
        global PREFIX
        PREFIX = self.original


def call(command, **kwargs):
    command = command.format(**kwargs)
    cmd = '{prefix}{command}'.format(prefix=PREFIX, command=command)
    puts(cmd)
    return subprocess.call(cmd, shell=True)


def check_call(command, **kwargs):
    command = command.format(**kwargs)
    cmd = '{prefix}{command}'.format(prefix=PREFIX, command=command)
    puts(cmd)
    return subprocess.check_call(cmd, shell=True)


def update_status(pr, state, job_name=JOB_NAME, build_url=BUILD_URL):
    url = pr['statuses_url']

    if state == 'pending':
        description = "Stashy says check back later..."
    elif state == 'success':
        description = "Stashy says you are golden!"
    elif state == 'failure':
        description = "Stashy says better luck next time :("

    data = {
        'state': state,
        'description': description,
        'context': job_name,
        'target_url': build_url,
    }
    if DEBUG:
        puts(data)

    puts('\nSetting test state to: {state}\n', state=state)
    request = urllib2.Request(url, headers=HEADERS)
    request.add_data(json.dumps(data))
    resp = urllib2.urlopen(request)


def fetch_pr(repo=REPO, number=PR_NUMBER):
    url = '{github}/{repo}/pulls/{number}'.format(
        github=GITHUB, number=number, repo=repo)
    request = urllib2.Request(url, headers=HEADERS)
    resp = urllib2.urlopen(request)
    pr = json.loads(resp.read())
    if DEBUG:
        puts('\n\nPR DATA:\n')
        puts(pr)
    return pr


def parse_depends(body):
    for pull in DEPENDS.finditer(body):
        yield fetch_pr(*pull.groups())


class TestCase(object):

    template = 'test_case.xml'

    def __init__(self, name, test_cmd, xunit=True):
        self.name = name
        self.test_cmd = test_cmd
        self.pr = None
        self.xunit = xunit
        self.state = 'pending'

    def update_submodules(self):
        with cd(WORKSPACE):
            for sub in SUBMODULES:
                check_call('git submodule update --init {sub}', sub=sub)

    def checkout_pr(self, pr):
        with cd(DIRECTORIES.get(pr['base']['repo']['name'])):
            branch = 'pull/{number}/head:{number}'.format(**pr)
            check_call('git fetch origin {branch} -f'.format(branch=branch))
            check_call('git checkout {number}', **pr)

    def setup(self):
        self.pr = fetch_pr()
        puts('\n\nPR {number} INFO:\n', **self.pr)
        puts('URL: {html_url}', **self.pr)
        puts('SHA: {sha}', sha=self.pr['head']['sha'])
        puts('TITLE: {title}', **self.pr)
        puts('INFO: \n{body}\n', **self.pr)
        if not self.pr['mergeable']:
            puts('\n\nTHIS PR CANNOT BE MERGED!!!\n\n')
            raise Exception('Pr Not mergeable')
        self.update_submodules()
        self.checkout_pr(self.pr)
        for pull in parse_depends(self.pr['body']):
            self.checkout_pr(pull)

    def clean(self):
        puts('\nCLEANING UP:\n')

    def run_tests(self):
        args = {
            'fab': TROVE_FAB,
            'pip': TROVE_PIP,
            'python': TROVE_PYTHON,
            'bin': TROVE_BIN,
            'workspace': WORKSPACE,
            'fab_args': 'tests:stop=True,white_box=True,with_xunit=True',
            'mysql_56': 'datastore=mysql,version=5.6',
            'mysql_51': 'datastore=mysql,version=5.1',
            'mariadb': 'datastore=mariadb,version=10',
            'percona': 'datastore=percona,version=5.6',
            'default_groups': 'group="rax_stable|rax_replication_groups"'
        }
        check_call(self.test_cmd, **args)

    def run(self):
        """Actually run the tests."""
        try:
            self.setup()
            update_status(self.pr, state=self.state)
            self.run_tests()
            self.state = 'success'
        except:
            self.state = 'failure'
{'fraction_non_alphanumeric': 0.1592789840229414, 'fraction_numerical': 0.0034412126177795987, 'mean_word_length': 2.828732747804266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1226836', 'n_tokens_mistral': 4258, 'n_tokens_neox': 4219, 'n_words': 798}