<commit_before>import copy
import logging
import os
from hashlib import sha1

import simplejson as json
from jsonschema import validate

from treeherder.model.models import OptionCollection
from treeherder.perf.models import (PerformanceDatum,
                                    PerformanceFramework,
                                    PerformanceSignature)
from treeherder.perf.tasks import generate_alerts

logger = logging.getLogger(__name__)


PERFHERDER_SCHEMA = json.load(open(os.path.join('schemas',
                                                'performance-artifact.json')))


def _get_signature_hash(signature_properties):
    signature_prop_values = signature_properties.keys()
    str_values = []
    for value in signature_properties.values():
        if not isinstance(value, basestring):
            str_values.append(json.dumps(value, sort_keys=True))
        else:
            str_values.append(value)
    signature_prop_values.extend(str_values)

    sha = sha1()
    sha.update(''.join(map(lambda x: str(x), sorted(signature_prop_values))))

    return sha.hexdigest()


def _load_perf_datum(job, perf_datum):
    validate(perf_datum, PERFHERDER_SCHEMA)

    extra_properties = {}
    reference_data = {
        'option_collection_hash': job.signature.option_collection_hash,
        'machine_platform': job.signature.machine_platform
    }
    if 'e10s' in job.job_type.job_group.symbol:
        extra_properties = {'test_options': ['e10s']}
        reference_data['test_options'] = json.dumps(['e10s'])

    option_collection = OptionCollection.objects.get(
        option_collection_hash=job.signature.option_collection_hash)

    try:
        framework = PerformanceFramework.objects.get(
            name=perf_datum['framework']['name'])
    except PerformanceFramework.DoesNotExist:
        logger.warning("Performance framework {} does not exist, skipping "
                       "load of performance artifacts".format(
                           perf_datum['framework']['name']))
        return
    if not framework.enabled:
        logger.info("Performance framework {} is not enabled, skipping"
                    .format(perf_datum['framework']['name']))
        return
    for suite in perf_datum['suites']:
        suite_extra_properties = copy.copy(extra_properties)
        if suite.get('extraOptions'):<commit_msg>Bug 1318474 - Remove obsolete e10s-special case handling in perf etl<commit_after>import copy
import logging
import os
from hashlib import sha1

import simplejson as json
from jsonschema import validate

from treeherder.model.models import OptionCollection
from treeherder.perf.models import (PerformanceDatum,
                                    PerformanceFramework,
                                    PerformanceSignature)
from treeherder.perf.tasks import generate_alerts

logger = logging.getLogger(__name__)


PERFHERDER_SCHEMA = json.load(open(os.path.join('schemas',
                                                'performance-artifact.json')))


def _get_signature_hash(signature_properties):
    signature_prop_values = signature_properties.keys()
    str_values = []
    for value in signature_properties.values():
        if not isinstance(value, basestring):
            str_values.append(json.dumps(value, sort_keys=True))
        else:
            str_values.append(value)
    signature_prop_values.extend(str_values)

    sha = sha1()
    sha.update(''.join(map(lambda x: str(x), sorted(signature_prop_values))))

    return sha.hexdigest()


def _load_perf_datum(job, perf_datum):
    validate(perf_datum, PERFHERDER_SCHEMA)

    extra_properties = {}
    reference_data = {
        'option_collection_hash': job.signature.option_collection_hash,
        'machine_platform': job.signature.machine_platform
    }

    option_collection = OptionCollection.objects.get(
        option_collection_hash=job.signature.option_collection_hash)

    try:
        framework = PerformanceFramework.objects.get(
            name=perf_datum['framework']['name'])
    except PerformanceFramework.DoesNotExist:
        logger.warning("Performance framework {} does not exist, skipping "
                       "load of performance artifacts".format(
                           perf_datum['framework']['name']))
        return
    if not framework.enabled:
        logger.info("Performance framework {} is not enabled, skipping"
                    .format(perf_datum['framework']['name']))
        return
    for suite in perf_datum['suites']:
        suite_extra_properties = copy.copy(extra_properties)
        if suite.get('extraOptions'):
{'fraction_non_alphanumeric': 0.09128450336883286, 'fraction_numerical': 0.0041295370571614864, 'mean_word_length': 2.8802698145025296, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4786942', 'n_tokens_mistral': 1251, 'n_tokens_neox': 1197, 'n_words': 245}