<commit_before>import datetime
import logging
import math
import os
from hashlib import sha1

import simplejson as json
from jsonschema import validate

from treeherder.model.models import (MachinePlatform,
                                     OptionCollection,
                                     Repository)
from treeherder.perf.models import (PerformanceDatum,
                                    PerformanceFramework,
                                    PerformanceSignature)
from treeherder.perf.tasks import generate_alerts

logger = logging.getLogger(__name__)


PERFORMANCE_ARTIFACT_TYPES = set([
    'performance_data',
    'talos_data'
])


# keys useful for creating a non-redundant performance signature
SIGNIFICANT_REFERENCE_DATA_KEYS = ['option_collection_hash',
                                   'machine_platform']


PERFHERDER_SCHEMA = json.load(open(os.path.join('schemas',
                                                'performance-artifact.json')))
TALOS_SCHEMA = json.load(open(os.path.join('schemas',
                                           'talos-artifact.json')))


def _transform_signature_properties(properties, significant_keys=None):
    if significant_keys is None:
        significant_keys = SIGNIFICANT_REFERENCE_DATA_KEYS
    transformed_properties = {k: v for k, v in properties.iteritems() if
                              k in significant_keys}

    # HACK: determine if e10s is in job_group_symbol, and add an "e10s"
    # property to a 'test_options' property if so (we should probably
    # make talos produce this information somehow and consume it in the
    # future)
    if 'e10s' in properties.get('job_group_symbol', ''):
        transformed_properties['test_options'] = json.dumps(['e10s'])

    return transformed_properties


def _get_signature_hash(signature_properties):
    signature_prop_values = signature_properties.keys()
    str_values = []
    for value in signature_properties.values():
        if not isinstance(value, basestring):
            str_values.append(json.dumps(value))
        else:
            str_values.append(value)
    signature_prop_values.extend(str_values)

    sha = sha1()
    sha.update(''.join(map(lambda x: str(x), sorted(signature_prop_values))))

    return sha.hexdigest()


def _load_perf_artifact(project_name, reference_data, job_data, job_guid,
                        perf_datum):
    validate(perf_datum, PERFHERDER_SCHEMA)

    if 'e10s' in reference_data.get('job_group_symbol', ''):
        extra_properties = {'test_options': ['e10s']}
    else:
        extra_properties = {}

    # transform the reference data so it only contains what we actually
    # care about (for calculating the signature hash reproducibly), then
    # get the associated models
    reference_data = _transform_signature_properties(reference_data)
    option_collection = OptionCollection.objects.get(
        option_collection_hash=reference_data['option_collection_hash'])
    # there may be multiple machine platforms with the same platform: use
    # the first
    platform = MachinePlatform.objects.filter(
        platform=reference_data['machine_platform'])[0]
    repository = Repository.objects.get(
        name=project_name)
    is_try_repository = repository.repository_group.name == 'try'

    # data for performance series
    job_id = job_data[job_guid]['id']
    result_set_id = job_data[job_guid]['result_set_id']
    push_timestamp = datetime.datetime.fromtimestamp(
        job_data[job_guid]['push_timestamp'])

    try:
        framework = PerformanceFramework.objects.get(
            name=perf_datum['framework']['name'])
    except PerformanceFramework.DoesNotExist:
        logger.warning("Performance framework {} does not exist, skipping "
                       "load of performance artifacts".format(
                           perf_datum['framework']['name']))
        return
    for suite in perf_datum['suites']:
        subtest_properties = []
        summary_signature_hash = None
        for subtest in suite['subtests']:
            subtest_metadata = {
                'suite': suite['name'],
                'test': subtest['name'],
                'lowerIsBetter': subtest.get('lowerIsBetter', True)
            }
            subtest_metadata.update(reference_data)
            subtest_properties.append(subtest_metadata)

        # if we have a summary value, create or get its signature by all its subtest
        # properties.
        if suite.get('value') is not None:
            # summary series
            summary_properties = {
                'suite': suite['name'],
                'subtest_properties': subtest_properties
            }
            summary_properties.update(reference_data)
            summary_properties.update(extra_properties)
            summary_signature_hash = _get_signature_hash(
                    summary_properties)
            signature, _ = PerformanceSignature.objects.get_or_create(
                repository=repository, signature_hash=summary_signature_hash,
                defaults={
                    'test': '',
                    'suite': suite['name'],
                    'option_collection': option_collection,
                    'platform': platform,
                    'framework': framework,
                    'extra_properties': extra_properties,
                    'lower_is_better': suite.get('lowerIsBetter', True),
                    'last_updated': push_timestamp
                })
            (_, datum_created) = PerformanceDatum.objects.get_or_create(
                repository=repository,
                result_set_id=result_set_id,
                job_id=job_id,
                signature=signature,
                push_timestamp=push_timestamp,
                defaults={'value': suite['value']})
            if datum_created and (not is_try_repository):
                generate_alerts.apply_async(args=[signature.id],
                                            routing_key='generate_perf_alerts')

        for subtest_metadata in subtest_properties:
            # we calculate the subtest signature incorporate
            # the hash of the parent.
            summary_signature = None
            if summary_signature_hash is not None:
                subtest_metadata.update({'parent_signature': summary_signature_hash})
                summary_signature = PerformanceSignature.objects.get(
                        signature_hash=summary_signature_hash)
            subtest_signature_hash = _get_signature_hash(subtest_metadata)
            value = list(subtest['value'] for subtest in suite['subtests'] if<commit_msg>Bug 1226693 - Fix perf ETL problem noticed after stage deploy<commit_after>import datetime
import logging
import math
import os
from hashlib import sha1

import simplejson as json
from jsonschema import validate

from treeherder.model.models import (MachinePlatform,
                                     OptionCollection,
                                     Repository)
from treeherder.perf.models import (PerformanceDatum,
                                    PerformanceFramework,
                                    PerformanceSignature)
from treeherder.perf.tasks import generate_alerts

logger = logging.getLogger(__name__)


PERFORMANCE_ARTIFACT_TYPES = set([
    'performance_data',
    'talos_data'
])


# keys useful for creating a non-redundant performance signature
SIGNIFICANT_REFERENCE_DATA_KEYS = ['option_collection_hash',
                                   'machine_platform']


PERFHERDER_SCHEMA = json.load(open(os.path.join('schemas',
                                                'performance-artifact.json')))
TALOS_SCHEMA = json.load(open(os.path.join('schemas',
                                           'talos-artifact.json')))


def _transform_signature_properties(properties, significant_keys=None):
    if significant_keys is None:
        significant_keys = SIGNIFICANT_REFERENCE_DATA_KEYS
    transformed_properties = {k: v for k, v in properties.iteritems() if
                              k in significant_keys}

    # HACK: determine if e10s is in job_group_symbol, and add an "e10s"
    # property to a 'test_options' property if so (we should probably
    # make talos produce this information somehow and consume it in the
    # future)
    if 'e10s' in properties.get('job_group_symbol', ''):
        transformed_properties['test_options'] = json.dumps(['e10s'])

    return transformed_properties


def _get_signature_hash(signature_properties):
    signature_prop_values = signature_properties.keys()
    str_values = []
    for value in signature_properties.values():
        if not isinstance(value, basestring):
            str_values.append(json.dumps(value))
        else:
            str_values.append(value)
    signature_prop_values.extend(str_values)

    sha = sha1()
    sha.update(''.join(map(lambda x: str(x), sorted(signature_prop_values))))

    return sha.hexdigest()


def _load_perf_artifact(project_name, reference_data, job_data, job_guid,
                        perf_datum):
    validate(perf_datum, PERFHERDER_SCHEMA)

    if 'e10s' in reference_data.get('job_group_symbol', ''):
        extra_properties = {'test_options': ['e10s']}
    else:
        extra_properties = {}

    # transform the reference data so it only contains what we actually
    # care about (for calculating the signature hash reproducibly), then
    # get the associated models
    reference_data = _transform_signature_properties(reference_data)
    option_collection = OptionCollection.objects.get(
        option_collection_hash=reference_data['option_collection_hash'])
    # there may be multiple machine platforms with the same platform: use
    # the first
    platform = MachinePlatform.objects.filter(
        platform=reference_data['machine_platform'])[0]
    repository = Repository.objects.get(
        name=project_name)
    is_try_repository = repository.repository_group.name == 'try'

    # data for performance series
    job_id = job_data[job_guid]['id']
    result_set_id = job_data[job_guid]['result_set_id']
    push_timestamp = datetime.datetime.fromtimestamp(
        job_data[job_guid]['push_timestamp'])

    try:
        framework = PerformanceFramework.objects.get(
            name=perf_datum['framework']['name'])
    except PerformanceFramework.DoesNotExist:
        logger.warning("Performance framework {} does not exist, skipping "
                       "load of performance artifacts".format(
                           perf_datum['framework']['name']))
        return
    for suite in perf_datum['suites']:
        subtest_properties = []
        summary_signature_hash = None
        for subtest in suite['subtests']:
            subtest_metadata = {
                'suite': suite['name'],
                'test': subtest['name'],
                'lowerIsBetter': subtest.get('lowerIsBetter', True)
            }
            subtest_metadata.update(reference_data)
            subtest_properties.append(subtest_metadata)

        # if we have a summary value, create or get its signature by all its subtest
        # properties.
        if suite.get('value') is not None:
            # summary series
            summary_properties = {
                'suite': suite['name'],
                'subtest_properties': subtest_properties
            }
            summary_properties.update(reference_data)
            summary_properties.update(extra_properties)
            summary_signature_hash = _get_signature_hash(
                    summary_properties)
            signature, _ = PerformanceSignature.objects.get_or_create(
                repository=repository, signature_hash=summary_signature_hash,
                defaults={
                    'test': '',
                    'suite': suite['name'],
                    'option_collection': option_collection,
                    'platform': platform,
                    'framework': framework,
                    'extra_properties': extra_properties,
                    'lower_is_better': suite.get('lowerIsBetter', True),
                    'last_updated': push_timestamp
                })
            (_, datum_created) = PerformanceDatum.objects.get_or_create(
                repository=repository,
                result_set_id=result_set_id,
                job_id=job_id,
                signature=signature,
                push_timestamp=push_timestamp,
                defaults={'value': suite['value']})
            if datum_created and (not is_try_repository):
                generate_alerts.apply_async(args=[signature.id],
                                            routing_key='generate_perf_alerts')

        for subtest_metadata in subtest_properties:
            # we calculate the subtest signature incorporate
            # the hash of the parent.
            summary_signature = None
            if summary_signature_hash is not None:
                subtest_metadata.update({'parent_signature': summary_signature_hash})
                summary_signature = PerformanceSignature.objects.get(
                    repository=repository,
                    signature_hash=summary_signature_hash)
            subtest_signature_hash = _get_signature_hash(subtest_metadata)
            value = list(subtest['value'] for subtest in suite['subtests'] if
{'fraction_non_alphanumeric': 0.08078913809916735, 'fraction_numerical': 0.0027754857099992498, 'mean_word_length': 2.3222028407675057, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '201013', 'n_tokens_mistral': 3449, 'n_tokens_neox': 3297, 'n_words': 749}