<commit_before>import hashlib
import json
import os
import posixpath
import re
import time
import unicodedata
import uuid
import zipfile

from collections import namedtuple

from django.core.files.storage import default_storage as storage
from django.db import models
from django.dispatch import receiver
from django.template.defaultfilters import slugify
from django.utils.encoding import force_bytes, force_text
from django.utils.functional import cached_property
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext, ugettext_lazy as _

from django_extensions.db.fields.json import JSONField
from django_statsd.clients import statsd
from jinja2 import escape as jinja2_escape

import olympia.core.logger

from olympia import amo
from olympia.lib.cache import memoize
from olympia.amo.decorators import use_primary_db
from olympia.amo.models import ModelBase, OnChangeMixin, ManagerBase
from olympia.amo.storage_utils import copy_stored_file, move_stored_file
from olympia.amo.templatetags.jinja_helpers import (
    absolutify, urlparams, user_media_path, user_media_url)
from olympia.amo.urlresolvers import reverse
from olympia.applications.models import AppVersion
from olympia.files.utils import SafeZip, write_crx_as_xpi
from olympia.translations.fields import TranslatedField


log = olympia.core.logger.getLogger('z.files')

# Acceptable extensions.
EXTENSIONS = ('.crx', '.xpi', '.jar', '.xml', '.json', '.zip')


class File(OnChangeMixin, ModelBase):
    STATUS_CHOICES = amo.STATUS_CHOICES_FILE

    version = models.ForeignKey(
        'versions.Version', related_name='files',
        on_delete=models.CASCADE)
    platform = models.PositiveIntegerField(
        choices=amo.SUPPORTED_PLATFORMS_CHOICES,
        default=amo.PLATFORM_ALL.id,
        db_column="platform_id"
    )
    filename = models.CharField(max_length=255, default='')
    size = models.PositiveIntegerField(default=0)  # In bytes.
    hash = models.CharField(max_length=255, default='')
    # The original hash of the file, before we sign it, or repackage it in
    # any other way.
    original_hash = models.CharField(max_length=255, default='')
    jetpack_version = models.CharField(max_length=10, null=True)
    status = models.PositiveSmallIntegerField(
        choices=STATUS_CHOICES.items(), default=amo.STATUS_AWAITING_REVIEW)
    datestatuschanged = models.DateTimeField(null=True, auto_now_add=True)
    is_restart_required = models.BooleanField(default=False)
    strict_compatibility = models.BooleanField(default=False)
    # The XPI contains JS that calls require("chrome").
    requires_chrome = models.BooleanField(default=False)
    reviewed = models.DateTimeField(null=True)
    # The `binary` field is used to store the flags from amo-validator when it
    # files files with binary extensions or files that may contain binary
    # content.
    binary = models.BooleanField(default=False)
    # The `binary_components` field is used to store the flag from
    # amo-validator when it finds "binary-components" in the chrome manifest
    # file, used for default to compatible.
    binary_components = models.BooleanField(default=False, db_index=True)
    # Serial number of the certificate use for the signature.
    cert_serial_num = models.TextField(blank=True)
    # Is the file signed by Mozilla?
    is_signed = models.BooleanField(default=False)
    # Is the file a multi-package?
    #     https://developer.mozilla.org/en-US/docs/Multiple_Item_Packaging
    is_multi_package = models.BooleanField(default=False)
    # Is the file an experiment (see bug 1220097)?
    is_experiment = models.BooleanField(default=False)
    # Is the file a WebExtension?<commit_msg>Make files.jetpack_version and .reviewed optional in django admin<commit_after>import hashlib
import json
import os
import posixpath
import re
import time
import unicodedata
import uuid
import zipfile

from collections import namedtuple

from django.core.files.storage import default_storage as storage
from django.db import models
from django.dispatch import receiver
from django.template.defaultfilters import slugify
from django.utils.encoding import force_bytes, force_text
from django.utils.functional import cached_property
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext, ugettext_lazy as _

from django_extensions.db.fields.json import JSONField
from django_statsd.clients import statsd
from jinja2 import escape as jinja2_escape

import olympia.core.logger

from olympia import amo
from olympia.lib.cache import memoize
from olympia.amo.decorators import use_primary_db
from olympia.amo.models import ModelBase, OnChangeMixin, ManagerBase
from olympia.amo.storage_utils import copy_stored_file, move_stored_file
from olympia.amo.templatetags.jinja_helpers import (
    absolutify, urlparams, user_media_path, user_media_url)
from olympia.amo.urlresolvers import reverse
from olympia.applications.models import AppVersion
from olympia.files.utils import SafeZip, write_crx_as_xpi
from olympia.translations.fields import TranslatedField


log = olympia.core.logger.getLogger('z.files')

# Acceptable extensions.
EXTENSIONS = ('.crx', '.xpi', '.jar', '.xml', '.json', '.zip')


class File(OnChangeMixin, ModelBase):
    STATUS_CHOICES = amo.STATUS_CHOICES_FILE

    version = models.ForeignKey(
        'versions.Version', related_name='files',
        on_delete=models.CASCADE)
    platform = models.PositiveIntegerField(
        choices=amo.SUPPORTED_PLATFORMS_CHOICES,
        default=amo.PLATFORM_ALL.id,
        db_column="platform_id"
    )
    filename = models.CharField(max_length=255, default='')
    size = models.PositiveIntegerField(default=0)  # In bytes.
    hash = models.CharField(max_length=255, default='')
    # The original hash of the file, before we sign it, or repackage it in
    # any other way.
    original_hash = models.CharField(max_length=255, default='')
    jetpack_version = models.CharField(max_length=10, null=True, blank=True)
    status = models.PositiveSmallIntegerField(
        choices=STATUS_CHOICES.items(), default=amo.STATUS_AWAITING_REVIEW)
    datestatuschanged = models.DateTimeField(null=True, auto_now_add=True)
    is_restart_required = models.BooleanField(default=False)
    strict_compatibility = models.BooleanField(default=False)
    # The XPI contains JS that calls require("chrome").
    requires_chrome = models.BooleanField(default=False)
    reviewed = models.DateTimeField(null=True, blank=True)
    # The `binary` field is used to store the flags from amo-validator when it
    # files files with binary extensions or files that may contain binary
    # content.
    binary = models.BooleanField(default=False)
    # The `binary_components` field is used to store the flag from
    # amo-validator when it finds "binary-components" in the chrome manifest
    # file, used for default to compatible.
    binary_components = models.BooleanField(default=False, db_index=True)
    # Serial number of the certificate use for the signature.
    cert_serial_num = models.TextField(blank=True)
    # Is the file signed by Mozilla?
    is_signed = models.BooleanField(default=False)
    # Is the file a multi-package?
    #     https://developer.mozilla.org/en-US/docs/Multiple_Item_Packaging
    is_multi_package = models.BooleanField(default=False)
    # Is the file an experiment (see bug 1220097)?
    is_experiment = models.BooleanField(default=False)
    # Is the file a WebExtension?
{'fraction_non_alphanumeric': 0.07749667110519308, 'fraction_numerical': 0.00559254327563249, 'mean_word_length': 4.733587786259542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5689722', 'n_tokens_mistral': 2233, 'n_tokens_neox': 2199, 'n_words': 608}