<commit_before># pylint: disable=too-many-lines

import logging
import os
import uuid
import json
import simplejson
import urlparse
import smtplib
import socket
import sys
import yaml
from django.db.models import Q
from django.conf import settings
from django.contrib.auth.models import User, Group
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.utils.safestring import mark_safe
from django.core.exceptions import (
    ImproperlyConfigured,
    ValidationError,
    ObjectDoesNotExist,
    MultipleObjectsReturned,
)
from django.core.mail import send_mail
from django.core.validators import validate_email
from django.db import models
from django.template.loader import render_to_string
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import get_object_or_404
from django_kvstore import models as kvmodels
from django_kvstore import get_kvstore
from django.utils import timezone

from django_restricted_resource.models import (
    RestrictedResource,
    RestrictedResourceManager
)
from lava_scheduler_app.managers import RestrictedTestJobQuerySet
from lava_scheduler_app.schema import validate_submission, SubmissionException
from dashboard_app.models import Bundle, BundleStream, NamedAttribute

from lava_dispatcher.job import validate_job_data
from lava_scheduler_app import utils

from linaro_django_xmlrpc.models import AuthToken

# pylint: disable=invalid-name,no-self-use,too-many-public-methods,too-few-public-methods

# Make the open function accept encodings in python < 3.x
if sys.version_info[0] < 3:
    import codecs
    open = codecs.open  # pylint: disable=redefined-builtin


class JSONDataError(ValueError):
    """Error raised when JSON is syntactically valid but ill-formed."""


class DevicesUnavailableException(UserWarning):
    """Error raised when required number of devices are unavailable."""


class Tag(models.Model):

    name = models.SlugField(unique=True)

    description = models.TextField(null=True, blank=True)

    def __unicode__(self):
        return self.name


def is_deprecated_json(data):
    """ Deprecated """
    deprecated_json = True
    try:
        ob = simplejson.loads(data)
        # calls deprecated lava_dispatcher code
        validate_job_data(ob)
    except (AttributeError, simplejson.JSONDecodeError, ValueError):
        deprecated_json = False
    except (JSONDataError, simplejson.JSONDecodeError, ValueError) as exc:
        raise SubmissionException("Decoding job submission failed: %s." % exc)
    return deprecated_json


def validate_job(data):
    if not is_deprecated_json(data):
        try:
            # only try YAML if this is not JSON
            # YAML can parse JSON as YAML, JSON cannot parse YAML at all
            yaml_data = yaml.load(data)
        except yaml.YAMLError as exc:
            # neither yaml nor json loaders were able to process the submission.
            raise SubmissionException("Loading job submission failed: %s." % exc)

        # validate against the submission schema.
        validate_submission(yaml_data)  # raises SubmissionException if invalid.


def validate_job_json(data):
    """ Deprecated """
    try:
        ob = simplejson.loads(data)
        validate_job_data(ob)
    except ValueError as e:
        raise ValidationError(e)


class Architecture(models.Model):
    name = models.CharField(
        primary_key=True,
        verbose_name=u'Architecture version',
        help_text=u'e.g. ARMv7',
        max_length=100,
        editable=True,
    )

    def __unicode__(self):
        return self.pk


class ProcessorFamily(models.Model):
    name = models.CharField(
        primary_key=True,
        verbose_name=u'Processor Family',
        help_text=u'e.g. OMAP4, Exynos',
        max_length=100,
        editable=True,
    )

    def __unicode__(self):
        return self.pk


class BitWidth(models.Model):
    width = models.PositiveSmallIntegerField(
        primary_key=True,
        verbose_name=u'Processor bit width',
        help_text=u'integer: e.g. 32 or 64',
        editable=True,
    )

    def __unicode__(self):
        return "%d" % self.pk


class Core(models.Model):
    name = models.CharField(
        primary_key=True,
        verbose_name=u'CPU core',
        help_text=u'Name of a specific CPU core, e.g. Cortex-A9',
        editable=True,
        max_length=100,
    )

    def __unicode__(self):
        return self.pk


class DeviceType(models.Model):
    """
    A class of device, for example a pandaboard or a snowball.
    """

    name = models.SlugField(primary_key=True)

    architecture = models.ForeignKey(
        Architecture,
        related_name='device_types',
        blank=True,
        null=True,
    )

    processor = models.ForeignKey(
        ProcessorFamily,
        related_name='device_types',
        blank=True,
        null=True,
    )

    cpu_model = models.CharField(
        verbose_name=u'CPU model',
        help_text=u'e.g. a list of CPU model descriptive strings: OMAP4430 / OMAP4460',
        max_length=100,
        blank=True,
        null=True,
        editable=True,
    )

    bits = models.ForeignKey(
        BitWidth,
        related_name='device_types',
        blank=True,
        null=True,
    )

    cores = models.ManyToManyField(
        Core,
        related_name='device_types',
        blank=True,
    )

    core_count = models.PositiveSmallIntegerField(
        verbose_name=u'Total number of cores',
        help_text=u'Must be an equal number of each type(s) of core(s).',
        blank=True,
        null=True,
    )

    def __unicode__(self):
        return self.name

    description = models.TextField(
        verbose_name=_(u"Device Type Description"),
        max_length=200,
        null=True,
        blank=True,
        default=None
    )

    health_check_job = models.TextField(
        null=True, blank=True, default=None, validators=[validate_job])

    health_frequency = models.IntegerField(
        verbose_name="How often to run health checks",
        default=24
    )

    HEALTH_PER_HOUR = 0
    HEALTH_PER_JOB = 1
    HEALTH_DENOMINATOR = (
        (HEALTH_PER_HOUR, 'hours'),
        (HEALTH_PER_JOB, 'jobs'),
    )

    health_denominator = models.IntegerField(
        choices=HEALTH_DENOMINATOR,
        default=HEALTH_PER_HOUR,
        verbose_name="Initiate health checks by hours or by jobs.",
        help_text=("Choose to submit a health check every N hours "
                   "or every N jobs. Balance against the duration of "
                   "a health check job and the average job duration."))

    display = models.BooleanField(default=True,
                                  help_text=("Should this be displayed in the GUI or not. This can be "
                                             "useful if you are removing all devices of this type but don't "
                                             "want to loose the test results generated by the devices."))

    owners_only = models.BooleanField(default=False,
                                      help_text="Hide this device type for all users except owners of "
                                                "devices of this type.")

    @models.permalink
    def get_absolute_url(self):
        return ("lava.scheduler.device_type.detail", [self.pk])

    def num_devices_visible_to(self, user):
        """
        Prepare a list of devices of this DeviceType which
        this user can see. If the DeviceType is not hidden,
        returns all devices of this type.
        :param user: User to check
        :return: the number of devices of this DeviceType which the
        user can see. This may be 0 if the type is hidden
        and the user owns none of the devices of this type.
        """
        devices = Device.objects.filter(device_type=self)
        if self.owners_only:
            return len([d for d in devices if d.is_owned_by(user)])
        else:<commit_msg>ORM: use select_related when possible<commit_after># pylint: disable=too-many-lines

import logging
import os
import uuid
import json
import simplejson
import urlparse
import smtplib
import socket
import sys
import yaml
from django.db.models import Q
from django.conf import settings
from django.contrib.auth.models import User, Group
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.utils.safestring import mark_safe
from django.core.exceptions import (
    ImproperlyConfigured,
    ValidationError,
    ObjectDoesNotExist,
    MultipleObjectsReturned,
)
from django.core.mail import send_mail
from django.core.validators import validate_email
from django.db import models
from django.template.loader import render_to_string
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import get_object_or_404
from django_kvstore import models as kvmodels
from django_kvstore import get_kvstore
from django.utils import timezone

from django_restricted_resource.models import (
    RestrictedResource,
    RestrictedResourceManager
)
from lava_scheduler_app.managers import RestrictedTestJobQuerySet
from lava_scheduler_app.schema import validate_submission, SubmissionException
from dashboard_app.models import Bundle, BundleStream, NamedAttribute

from lava_dispatcher.job import validate_job_data
from lava_scheduler_app import utils

from linaro_django_xmlrpc.models import AuthToken

# pylint: disable=invalid-name,no-self-use,too-many-public-methods,too-few-public-methods

# Make the open function accept encodings in python < 3.x
if sys.version_info[0] < 3:
    import codecs
    open = codecs.open  # pylint: disable=redefined-builtin


class JSONDataError(ValueError):
    """Error raised when JSON is syntactically valid but ill-formed."""


class DevicesUnavailableException(UserWarning):
    """Error raised when required number of devices are unavailable."""


class Tag(models.Model):

    name = models.SlugField(unique=True)

    description = models.TextField(null=True, blank=True)

    def __unicode__(self):
        return self.name


def is_deprecated_json(data):
    """ Deprecated """
    deprecated_json = True
    try:
        ob = simplejson.loads(data)
        # calls deprecated lava_dispatcher code
        validate_job_data(ob)
    except (AttributeError, simplejson.JSONDecodeError, ValueError):
        deprecated_json = False
    except (JSONDataError, simplejson.JSONDecodeError, ValueError) as exc:
        raise SubmissionException("Decoding job submission failed: %s." % exc)
    return deprecated_json


def validate_job(data):
    if not is_deprecated_json(data):
        try:
            # only try YAML if this is not JSON
            # YAML can parse JSON as YAML, JSON cannot parse YAML at all
            yaml_data = yaml.load(data)
        except yaml.YAMLError as exc:
            # neither yaml nor json loaders were able to process the submission.
            raise SubmissionException("Loading job submission failed: %s." % exc)

        # validate against the submission schema.
        validate_submission(yaml_data)  # raises SubmissionException if invalid.


def validate_job_json(data):
    """ Deprecated """
    try:
        ob = simplejson.loads(data)
        validate_job_data(ob)
    except ValueError as e:
        raise ValidationError(e)


class Architecture(models.Model):
    name = models.CharField(
        primary_key=True,
        verbose_name=u'Architecture version',
        help_text=u'e.g. ARMv7',
        max_length=100,
        editable=True,
    )

    def __unicode__(self):
        return self.pk


class ProcessorFamily(models.Model):
    name = models.CharField(
        primary_key=True,
        verbose_name=u'Processor Family',
        help_text=u'e.g. OMAP4, Exynos',
        max_length=100,
        editable=True,
    )

    def __unicode__(self):
        return self.pk


class BitWidth(models.Model):
    width = models.PositiveSmallIntegerField(
        primary_key=True,
        verbose_name=u'Processor bit width',
        help_text=u'integer: e.g. 32 or 64',
        editable=True,
    )

    def __unicode__(self):
        return "%d" % self.pk


class Core(models.Model):
    name = models.CharField(
        primary_key=True,
        verbose_name=u'CPU core',
        help_text=u'Name of a specific CPU core, e.g. Cortex-A9',
        editable=True,
        max_length=100,
    )

    def __unicode__(self):
        return self.pk


class DeviceType(models.Model):
    """
    A class of device, for example a pandaboard or a snowball.
    """

    name = models.SlugField(primary_key=True)

    architecture = models.ForeignKey(
        Architecture,
        related_name='device_types',
        blank=True,
        null=True,
    )

    processor = models.ForeignKey(
        ProcessorFamily,
        related_name='device_types',
        blank=True,
        null=True,
    )

    cpu_model = models.CharField(
        verbose_name=u'CPU model',
        help_text=u'e.g. a list of CPU model descriptive strings: OMAP4430 / OMAP4460',
        max_length=100,
        blank=True,
        null=True,
        editable=True,
    )

    bits = models.ForeignKey(
        BitWidth,
        related_name='device_types',
        blank=True,
        null=True,
    )

    cores = models.ManyToManyField(
        Core,
        related_name='device_types',
        blank=True,
    )

    core_count = models.PositiveSmallIntegerField(
        verbose_name=u'Total number of cores',
        help_text=u'Must be an equal number of each type(s) of core(s).',
        blank=True,
        null=True,
    )

    def __unicode__(self):
        return self.name

    description = models.TextField(
        verbose_name=_(u"Device Type Description"),
        max_length=200,
        null=True,
        blank=True,
        default=None
    )

    health_check_job = models.TextField(
        null=True, blank=True, default=None, validators=[validate_job])

    health_frequency = models.IntegerField(
        verbose_name="How often to run health checks",
        default=24
    )

    HEALTH_PER_HOUR = 0
    HEALTH_PER_JOB = 1
    HEALTH_DENOMINATOR = (
        (HEALTH_PER_HOUR, 'hours'),
        (HEALTH_PER_JOB, 'jobs'),
    )

    health_denominator = models.IntegerField(
        choices=HEALTH_DENOMINATOR,
        default=HEALTH_PER_HOUR,
        verbose_name="Initiate health checks by hours or by jobs.",
        help_text=("Choose to submit a health check every N hours "
                   "or every N jobs. Balance against the duration of "
                   "a health check job and the average job duration."))

    display = models.BooleanField(default=True,
                                  help_text=("Should this be displayed in the GUI or not. This can be "
                                             "useful if you are removing all devices of this type but don't "
                                             "want to loose the test results generated by the devices."))

    owners_only = models.BooleanField(default=False,
                                      help_text="Hide this device type for all users except owners of "
                                                "devices of this type.")

    @models.permalink
    def get_absolute_url(self):
        return ("lava.scheduler.device_type.detail", [self.pk])

    def num_devices_visible_to(self, user):
        """
        Prepare a list of devices of this DeviceType which
        this user can see. If the DeviceType is not hidden,
        returns all devices of this type.
        :param user: User to check
        :return: the number of devices of this DeviceType which the
        user can see. This may be 0 if the type is hidden
        and the user owns none of the devices of this type.
        """
        devices = Device.objects.filter(device_type=self) \
                                .only('user', 'group') \
                                .select_related('user', 'group')
        if self.owners_only:
            return len([d for d in devices if d.is_owned_by(user)])
        else:
{'fraction_non_alphanumeric': 0.07659442724458204, 'fraction_numerical': 0.005077399380804953, 'mean_word_length': 2.605938825630721, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7631720', 'n_tokens_mistral': 4734, 'n_tokens_neox': 4475, 'n_words': 1297}