<commit_before>import json
import MySQLdb

from warnings import filterwarnings, resetwarnings
from django.conf import settings

from treeherder.model.models import Datasource
from treeherder.model import utils

from .refdata import RefDataManager
from .base import TreeherderModelBase

class JobsModel(TreeherderModelBase):
    """
    Represent a job repository with objectstore

    content-types:
        jobs
        objectstore

    """

    # content types that every project will have
    CT_JOBS = "jobs"
    CT_OBJECTSTORE = "objectstore"
    CONTENT_TYPES = [CT_JOBS, CT_OBJECTSTORE]


    @classmethod
    def create(cls, project, hosts=None, types=None):
        """
        Create all the datasource tables for this project.

        ``hosts`` is an optional dictionary mapping contenttype names to the
        database server host on which the database for that contenttype should
        be created. Not all contenttypes need to be represented; any that
        aren't will use the default (``TREEHERDER_DATABASE_HOST``).

        ``types`` is an optional dictionary mapping contenttype names to the
        type of database that should be created. For MySQL/MariaDB databases,
        use "MySQL-Engine", where "Engine" could be "InnoDB", "Aria", etc. Not
        all contenttypes need to be represented; any that aren't will use the
        default (``MySQL-InnoDB``).


        """
        hosts = hosts or {}
        types = types or {}

        for ct in cls.CONTENT_TYPES:
            Datasource.create(
                project,
                ct,
                host=hosts.get(ct),
                db_type=types.get(ct),
                )

        return cls(project=project)


    def get_oauth_consumer_secret(self, key):
        ds = self.sources[self.CT_OBJECTSTORE].datasource
        secret = ds.get_oauth_consumer_secret(key)
        return secret


    def _get_last_insert_id(self, source=None):
        """Return last-inserted ID."""
        if not source:
            source = self.CT_JOBS
        return self.sources[source].dhub.execute(
            proc='generic.selects.get_last_insert_id',
            debug_show=self.DEBUG,
            return_type='iter',
            ).get_column_data('id')


    def store_job_data(self, json_data, error=None):
        """Write the JSON to the objectstore to be queued for processing."""

        loaded_timestamp = utils.get_now_timestamp()
        error = "N" if error is None else "Y"
        error_msg = error or ""

        self.sources[self.CT_OBJECTSTORE].dhub.execute(
            proc='objectstore.inserts.store_json',
            placeholders=[loaded_timestamp, json_data, error, error_msg],
            debug_show=self.DEBUG
        )

        return self._get_last_insert_id()


    def retrieve_job_data(self, limit):
        """
        Retrieve JSON blobs from the objectstore.

        Does not claim rows for processing; should not be used for actually
        processing JSON blobs into perftest schema.

        Used only by the `transfer_data` management command.

        """
        proc = "objectstore.selects.get_unprocessed"
        json_blobs = self.sources[self.CT_OBJECTSTORE].dhub.execute(
            proc=proc,
            placeholders=[limit],
            debug_show=self.DEBUG,
            return_type='tuple'
        )

        return json_blobs


    def load_job_data(self, data):
        """
        Load JobData instance into jobs db, return job_id.

        @@@: should I return the job_guid instead?

        Example:
            {
                "sources": [
                    {
                        "commit_timestamp": 1365732271,
                        "push_timestamp": 1365732271,
                        "comments": "Bug 854583 - Use _pointer_ instead of _cursor_ for mouse lock, r=dolske, approval-aurora=gavin",
                        "repository": "mozilla-aurora",
                        "revision": "c91ee0e8a980"
                    }
                ],
                "revision_hash": "24fd64b8251fac5cf60b54a915bffa7e51f636b5",
                "jobs": [
                    {
                        "build_platform": {
                            "platform": "Ubuntu VM 12.04",
                            "os_name": "linux",
                            "architecture": "x86_64",
                            "vm": true
                        },
                        "submit_timestamp": 1365732271,
                        "start_timestamp": "20130411165317",
                        "name": "xpcshell",
                        "option_collection": {
                            "opt": true
                        },
                        "log_references": [
                            {
                                "url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-aurora-linux64-pgo/1365724397/mozilla-aurora_ubuntu64_vm_test_pgo-xpcshell-bm54-tests1-linux-build4.txt.gz",
                                "name": "unittest"
                            }
                        ],
                        "who": "sendchange-unittest",
                        "reason": "scheduler",
                        "artifact": {},
                        "machine_platform": {
                            "platform": "Ubuntu VM 12.04",
                            "os_name": "linux",
                            "architecture": "x86_64",
                            "vm": true
                        },
                        "machine": "tst-linux64-ec2-314",
                        "state": "TODO",
                        "result": 0,
                        "job_guid": "d19375ce775f0dc166de01daa5d2e8a73a8e8ebf",
                        "product_name": "firefox",
                        "end_timestamp": "1365733932"
                    }
                ]
            }

        """

        # Get/Set reference info, all inserts use ON DUPLICATE KEY

        # @@@ tempted to coalesce these ids into a dict to pass to _set_job_data...
        rdm = RefDataManager(self.project)

        build_platform_id = rdm.get_or_create_build_platform(
            **data["jobs"]["build_platform"])
        machine_platform_id = rdm.get_or_create_machine_platform(
            **data["jobs"]["machine_platform"])
        machine_id = rdm.get_or_create_machine(
            data["machine"],
        )
        # @@@ need to straighten out with mdoglio
        option_collection_id = rdm.get_or_create_option_collection(
            data)

        # @@@ need these fields in the job structure
        job_type_id = rdm.get_or_create_job_type(data)
        product_id = rdm.get_or_create_product(
            data["jobs"]["product_name"],
        )

        result_set_id = self._set_result_set(data["revision_hash"])

        job_id = self._set_job_data(
            data,
            result_set_id,
            build_platform_id,
            machine_platform_id,
            machine_id,
            option_collection_id,
            job_type_id,
            product_id,
        )

        return job_id


    def _set_result_set(self, revision_hash):

        job_id = self._insert_data_and_get_id(
            'set_result_set',
            [
                revision_hash,
            ]
        )

        return job_id

<commit_msg>updates to usage of refdata methods<commit_after>import json
import MySQLdb

from warnings import filterwarnings, resetwarnings
from django.conf import settings

from treeherder.model.models import Datasource
from treeherder.model import utils

from .refdata import RefDataManager
from .base import TreeherderModelBase

class JobsModel(TreeherderModelBase):
    """
    Represent a job repository with objectstore

    content-types:
        jobs
        objectstore

    """

    # content types that every project will have
    CT_JOBS = "jobs"
    CT_OBJECTSTORE = "objectstore"
    CONTENT_TYPES = [CT_JOBS, CT_OBJECTSTORE]


    @classmethod
    def create(cls, project, hosts=None, types=None):
        """
        Create all the datasource tables for this project.

        ``hosts`` is an optional dictionary mapping contenttype names to the
        database server host on which the database for that contenttype should
        be created. Not all contenttypes need to be represented; any that
        aren't will use the default (``TREEHERDER_DATABASE_HOST``).

        ``types`` is an optional dictionary mapping contenttype names to the
        type of database that should be created. For MySQL/MariaDB databases,
        use "MySQL-Engine", where "Engine" could be "InnoDB", "Aria", etc. Not
        all contenttypes need to be represented; any that aren't will use the
        default (``MySQL-InnoDB``).


        """
        hosts = hosts or {}
        types = types or {}

        for ct in cls.CONTENT_TYPES:
            Datasource.create(
                project,
                ct,
                host=hosts.get(ct),
                db_type=types.get(ct),
                )

        return cls(project=project)


    def get_oauth_consumer_secret(self, key):
        ds = self.sources[self.CT_OBJECTSTORE].datasource
        secret = ds.get_oauth_consumer_secret(key)
        return secret


    def _get_last_insert_id(self, source=None):
        """Return last-inserted ID."""
        if not source:
            source = self.CT_JOBS
        return self.sources[source].dhub.execute(
            proc='generic.selects.get_last_insert_id',
            debug_show=self.DEBUG,
            return_type='iter',
            ).get_column_data('id')


    def store_job_data(self, json_data, error=None):
        """Write the JSON to the objectstore to be queued for processing."""

        loaded_timestamp = utils.get_now_timestamp()
        error = "N" if error is None else "Y"
        error_msg = error or ""

        self.sources[self.CT_OBJECTSTORE].dhub.execute(
            proc='objectstore.inserts.store_json',
            placeholders=[loaded_timestamp, json_data, error, error_msg],
            debug_show=self.DEBUG
        )

        return self._get_last_insert_id()


    def retrieve_job_data(self, limit):
        """
        Retrieve JSON blobs from the objectstore.

        Does not claim rows for processing; should not be used for actually
        processing JSON blobs into perftest schema.

        Used only by the `transfer_data` management command.

        """
        proc = "objectstore.selects.get_unprocessed"
        json_blobs = self.sources[self.CT_OBJECTSTORE].dhub.execute(
            proc=proc,
            placeholders=[limit],
            debug_show=self.DEBUG,
            return_type='tuple'
        )

        return json_blobs


    def load_job_data(self, data):
        """
        Load JobData instance into jobs db, return job_id.

        @@@: should I return the job_guid instead?

        Example:
            {
                "sources": [
                    {
                        "commit_timestamp": 1365732271,
                        "push_timestamp": 1365732271,
                        "comments": "Bug 854583 - Use _pointer_ instead of _cursor_ for mouse lock, r=dolske, approval-aurora=gavin",
                        "repository": "mozilla-aurora",
                        "revision": "c91ee0e8a980"
                    }
                ],
                "revision_hash": "24fd64b8251fac5cf60b54a915bffa7e51f636b5",
                "jobs": [
                    {
                        "build_platform": {
                            "platform": "Ubuntu VM 12.04",
                            "os_name": "linux",
                            "architecture": "x86_64",
                            "vm": true
                        },
                        "submit_timestamp": 1365732271,
                        "start_timestamp": "20130411165317",
                        "name": "xpcshell",
                        "option_collection": {
                            "opt": true
                        },
                        "log_references": [
                            {
                                "url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-aurora-linux64-pgo/1365724397/mozilla-aurora_ubuntu64_vm_test_pgo-xpcshell-bm54-tests1-linux-build4.txt.gz",
                                "name": "unittest"
                            }
                        ],
                        "who": "sendchange-unittest",
                        "reason": "scheduler",
                        "artifact": {},
                        "machine_platform": {
                            "platform": "Ubuntu VM 12.04",
                            "os_name": "linux",
                            "architecture": "x86_64",
                            "vm": true
                        },
                        "machine": "tst-linux64-ec2-314",
                        "state": "TODO",
                        "result": 0,
                        "job_guid": "d19375ce775f0dc166de01daa5d2e8a73a8e8ebf",
                        "product_name": "firefox",
                        "end_timestamp": "1365733932"
                    }
                ]
            }

        """

        # Get/Set reference info, all inserts use ON DUPLICATE KEY

        rdm = RefDataManager(self.project)

        build_platform_id = rdm.get_or_create_build_platform(
            **data["jobs"]["build_platform"])
        machine_platform_id = rdm.get_or_create_machine_platform(
            **data["jobs"]["machine_platform"])

        machine_id = rdm.get_or_create_machine(
            data["machine"],
            timestamp=max([
                data["start_timestamp"],
                data["submit_timestamp"],
                data["end_timestamp"],
                ])
        )
        option_collection_id = rdm.get_or_create_option_collection(
            [k for k, v in data["option_collection"].items() if v],
        )

        job_type_id = rdm.get_or_create_job_type(data["name"])
        product_id = rdm.get_or_create_product(
            data["jobs"]["product_name"],
            description=None  # @@@ remove this when api changes
        )

        result_set_id = self._set_result_set(data["revision_hash"])

        job_id = self._set_job_data(
            data,
            result_set_id,
            build_platform_id,
            machine_platform_id,
            machine_id,
            option_collection_id,
            job_type_id,
            product_id,
        )

        return job_id


    def _set_result_set(self, revision_hash):

        job_id = self._insert_data_and_get_id(
            'set_result_set',
            [
                revision_hash,
            ]
        )

        return job_id


{'fraction_non_alphanumeric': 0.08752542372881356, 'fraction_numerical': 0.020610169491525422, 'mean_word_length': 1.5476683937823834, 'pattern_counts': {'":': 70, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '798000', 'n_tokens_mistral': 4312, 'n_tokens_neox': 3904, 'n_words': 889}