<commit_before>"""
Module with non-database helper classes
"""

from uuid import UUID
import base64
import logging
import time

from django.core.files.base import ContentFile
from django.db import connection, transaction, IntegrityError
from linaro_dashboard_bundle.errors import DocumentFormatError
from linaro_dashboard_bundle.evolution import DocumentEvolution
from linaro_dashboard_bundle.io import DocumentIO
from linaro_json.extensions import datetime_extension, timedelta_extension


PROFILE_LOGGING = False


def is_postgres():
    return (connection.settings_dict['ENGINE']
            == 'django.db.backends.postgresql_psycopg2')


class IBundleFormatImporter(object):
    """
    Interface for bundle format importers.
    """

    def import_document(self, s_bundle, doc):
        """
        Import document in a supported format and tie it to the
        specified server side DashboardBundle model (s_bundle)

        :Discussion:
            Imports specified document in a supported format and tie it
            to the specified server side DashboardBundle model
            (s_bundle)

        :Return value:
            None

        :Exceptions raised:
            ValueError
            TypeError
            others (?)
        """
        raise NotImplementedError(self.import_document)


class BundleFormatImporter_1_0(IBundleFormatImporter):
    """
    IFormatImporter subclass capable of loading "Dashboard Bundle Format 1.0"
    """

    def import_document(self, s_bundle, doc):
        """
        Import specified bundle document into the database.
        """
        self._content_files = []
        self._import_sanity_check(doc)
        try:
            self._import_document_with_transaction(s_bundle, doc)
        except IntegrityError:
            self._remove_created_files()
            raise

    def _remove_created_files(self):
        """
        Remove any files that may have already been flushed to disk. Otherwise
        the transaction handling should rollback anything evil that was
        happening while we were here
        """
        for content_file in self._content_files:
            content_file.delete(save=False)

    def _import_sanity_check(self, doc):
        """
        Sanity check before any import is attempted.

        This prevents InternalError (but is racy with other transactions).
        Still it's a little bit better to report the exception raised below
        rather than the IntegrityError that would have been raised otherwise.

        The code copes with both (using transactions around _import_document()
        and _remove_created_files() that gets called if something is wrong)
        """
        from dashboard_app.models import TestRun

        for test_run in doc.get("test_runs", []):
            analyzer_assigned_uuid = test_run["analyzer_assigned_uuid"]
            if TestRun.objects.filter(
                analyzer_assigned_uuid=analyzer_assigned_uuid).exists():
                raise ValueError("A test with UUID {0} already exists".format(analyzer_assigned_uuid))

    @transaction.commit_on_success
    def _import_document_with_transaction(self, s_bundle, doc):
        """
        Note: This function uses commit_on_success to ensure the database is in
        a consistent state after IntegrityErrors that would clog the
        transaction on pgsql. Since transactions will not rollback any files we
        created in the meantime there is a helper that cleans attachments in
        case something goes wrong
        """
        self._import_document(s_bundle, doc)

    def _import_document(self, s_bundle, doc):
        for c_test_run in doc.get("test_runs", []):
            self._import_test_run(c_test_run, s_bundle)

    def __init__(self):
        self._qc = 0
        self._time = time.time()

    def _log(self, method_name):
        if PROFILE_LOGGING:
            t = time.time() - self._time
            if t > 0.1:
                p = '**'
            else:
                p = '  '
            logging.warning(
                '%s %s %.2f %d', p, method_name, t,
                len(connection.queries) - self._qc)
            self._qc = len(connection.queries)
            self._time = time.time()

    def _import_test_run(self, c_test_run, s_bundle):
        """
        Import TestRun
        """
        from dashboard_app.models import TestRun

        s_test = self._import_test(c_test_run)
        analyzer_assigned_uuid = UUID(c_test_run["analyzer_assigned_uuid"])
        s_test_run = TestRun.objects.create(
            bundle = s_bundle,
            test = s_test,
            analyzer_assigned_uuid = str(analyzer_assigned_uuid),
            analyzer_assigned_date = datetime_extension.from_json(
                # required by schema
                c_test_run["analyzer_assigned_date"]),
            time_check_performed = (
                # required by schema
                c_test_run["time_check_performed"]),
        )
        # needed for foreign key models below
        s_test_run.save()
        # import all the bits and pieces
        self._log('starting')
        self._import_test_results(c_test_run, s_test_run)
        self._log('results')
        self._import_attachments(c_test_run, s_test_run)
        self._log('attachments')
        self._import_hardware_context(c_test_run, s_test_run)
        self._log('hardware')
        self._import_software_context(c_test_run, s_test_run)
        self._log('software')
        self._import_attributes(c_test_run, s_test_run)
        self._log('attributes')
        # collect all the changes that happen before the previous save
        s_test_run.save()
        return s_test_run

    def _import_software_context(self, c_test_run, s_test_run):
        """
        Import software context.

        In format 1.0 that's just a list of packages and software image
        description
        """
        self._import_packages(c_test_run, s_test_run)
        s_test_run.sw_image_desc = self._get_sw_context(c_test_run).get(
                "sw_image", {}).get("desc", "")

    def _import_hardware_context(self, c_test_run, s_test_run):
        """
        Import hardware context.

        In format 1.0 that's just a list of devices
        """
        self._import_devices(c_test_run, s_test_run)

    def _import_test(self, c_test_run):
        """
        Import dashboard_app.models.Test into the database
        based on a client-side description of a TestRun
        """
        from dashboard_app.models import Test

        s_test, test_created = Test.objects.get_or_create(
            test_id = c_test_run["test_id"]) # required by schema<commit_msg>Denormalize each test run on deserialization<commit_after>"""
Module with non-database helper classes
"""

from uuid import UUID
import base64
import logging
import time

from django.core.files.base import ContentFile
from django.db import connection, transaction, IntegrityError
from linaro_dashboard_bundle.errors import DocumentFormatError
from linaro_dashboard_bundle.evolution import DocumentEvolution
from linaro_dashboard_bundle.io import DocumentIO
from linaro_json.extensions import datetime_extension, timedelta_extension


PROFILE_LOGGING = False


def is_postgres():
    return (connection.settings_dict['ENGINE']
            == 'django.db.backends.postgresql_psycopg2')


class IBundleFormatImporter(object):
    """
    Interface for bundle format importers.
    """

    def import_document(self, s_bundle, doc):
        """
        Import document in a supported format and tie it to the
        specified server side DashboardBundle model (s_bundle)

        :Discussion:
            Imports specified document in a supported format and tie it
            to the specified server side DashboardBundle model
            (s_bundle)

        :Return value:
            None

        :Exceptions raised:
            ValueError
            TypeError
            others (?)
        """
        raise NotImplementedError(self.import_document)


class BundleFormatImporter_1_0(IBundleFormatImporter):
    """
    IFormatImporter subclass capable of loading "Dashboard Bundle Format 1.0"
    """

    def import_document(self, s_bundle, doc):
        """
        Import specified bundle document into the database.
        """
        self._content_files = []
        self._import_sanity_check(doc)
        try:
            self._import_document_with_transaction(s_bundle, doc)
        except IntegrityError:
            self._remove_created_files()
            raise

    def _remove_created_files(self):
        """
        Remove any files that may have already been flushed to disk. Otherwise
        the transaction handling should rollback anything evil that was
        happening while we were here
        """
        for content_file in self._content_files:
            content_file.delete(save=False)

    def _import_sanity_check(self, doc):
        """
        Sanity check before any import is attempted.

        This prevents InternalError (but is racy with other transactions).
        Still it's a little bit better to report the exception raised below
        rather than the IntegrityError that would have been raised otherwise.

        The code copes with both (using transactions around _import_document()
        and _remove_created_files() that gets called if something is wrong)
        """
        from dashboard_app.models import TestRun

        for test_run in doc.get("test_runs", []):
            analyzer_assigned_uuid = test_run["analyzer_assigned_uuid"]
            if TestRun.objects.filter(
                analyzer_assigned_uuid=analyzer_assigned_uuid).exists():
                raise ValueError("A test with UUID {0} already exists".format(analyzer_assigned_uuid))

    @transaction.commit_on_success
    def _import_document_with_transaction(self, s_bundle, doc):
        """
        Note: This function uses commit_on_success to ensure the database is in
        a consistent state after IntegrityErrors that would clog the
        transaction on pgsql. Since transactions will not rollback any files we
        created in the meantime there is a helper that cleans attachments in
        case something goes wrong
        """
        self._import_document(s_bundle, doc)

    def _import_document(self, s_bundle, doc):
        for c_test_run in doc.get("test_runs", []):
            self._import_test_run(c_test_run, s_bundle)

    def __init__(self):
        self._qc = 0
        self._time = time.time()

    def _log(self, method_name):
        if PROFILE_LOGGING:
            t = time.time() - self._time
            if t > 0.1:
                p = '**'
            else:
                p = '  '
            logging.warning(
                '%s %s %.2f %d', p, method_name, t,
                len(connection.queries) - self._qc)
            self._qc = len(connection.queries)
            self._time = time.time()

    def _import_test_run(self, c_test_run, s_bundle):
        """
        Import TestRun
        """
        from dashboard_app.models import TestRun

        s_test = self._import_test(c_test_run)
        analyzer_assigned_uuid = UUID(c_test_run["analyzer_assigned_uuid"])
        s_test_run = TestRun.objects.create(
            bundle = s_bundle,
            test = s_test,
            analyzer_assigned_uuid = str(analyzer_assigned_uuid),
            analyzer_assigned_date = datetime_extension.from_json(
                # required by schema
                c_test_run["analyzer_assigned_date"]),
            time_check_performed = (
                # required by schema
                c_test_run["time_check_performed"]),
        )
        # needed for foreign key models below
        s_test_run.save()
        # import all the bits and pieces
        self._log('starting')
        self._import_test_results(c_test_run, s_test_run)
        self._log('results')
        self._import_attachments(c_test_run, s_test_run)
        self._log('attachments')
        self._import_hardware_context(c_test_run, s_test_run)
        self._log('hardware')
        self._import_software_context(c_test_run, s_test_run)
        self._log('software')
        self._import_attributes(c_test_run, s_test_run)
        self._log('attributes')
        # collect all the changes that happen before the previous save
        s_test_run.save()
        s_test_run.denormalize()
        return s_test_run

    def _import_software_context(self, c_test_run, s_test_run):
        """
        Import software context.

        In format 1.0 that's just a list of packages and software image
        description
        """
        self._import_packages(c_test_run, s_test_run)
        s_test_run.sw_image_desc = self._get_sw_context(c_test_run).get(
                "sw_image", {}).get("desc", "")

    def _import_hardware_context(self, c_test_run, s_test_run):
        """
        Import hardware context.

        In format 1.0 that's just a list of devices
        """
        self._import_devices(c_test_run, s_test_run)

    def _import_test(self, c_test_run):
        """
        Import dashboard_app.models.Test into the database
        based on a client-side description of a TestRun
        """
        from dashboard_app.models import Test

        s_test, test_created = Test.objects.get_or_create(
            test_id = c_test_run["test_id"]) # required by schema
{'fraction_non_alphanumeric': 0.07239988045427376, 'fraction_numerical': 0.002390914524805738, 'mean_word_length': 2.421523517382413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6240049', 'n_tokens_mistral': 3766, 'n_tokens_neox': 3645, 'n_words': 997}