<commit_before>import os
import posixpath
import urlparse
import uuid

import lxml.html

from docutils.core import publish_string

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import Sum
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.encoding import smart_str
from django.utils.translation import ugettext_lazy as _

from model_utils import Choices
from model_utils.fields import AutoCreatedField, AutoLastModifiedField
from model_utils.models import TimeStampedModel

from crate.fields import JSONField
from packages.utils import verlib


def release_file_upload_to(instance, filename):
    dsplit = instance.digest.split("$")
    if len(dsplit) == 2:
        directory = dsplit[1]
    else:
        directory = str(uuid.uuid4()).replace("-", "")

    return posixpath.join("packages", directory, filename)


# @@@ These are by Nature Hierarchical. Would we benefit from a tree structure?
class TroveClassifier(models.Model):
    trove = models.CharField(max_length=350, unique=True)

    def __unicode__(self):
        return self.trove


class Package(TimeStampedModel):
    name = models.SlugField(max_length=150, unique=True)
    package_uri_migrated = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse("package_detail", kwargs={"package": self.name})

    @property
    def downloads(self):
        total_downloads = ReleaseFile.objects.filter(release__package__pk=self.pk).aggregate(total_downloads=Sum("downloads"))["total_downloads"]
        if total_downloads is None:
            return 0
        return total_downloads

    @property
    def latest(self):
        if not hasattr(self, "_latest_release"):
            releases = self.releases.order_by("-order")[:1]
            if releases:
                self._latest_release = releases[0]
            else:
                self._latest_release = None
        return self._latest_release

    @property
    def install_command(self):
        return "pip install %(package)s" % {"package": self.name}

    @property
    def requirement_line(self):
        if self.latest is not None:
            # @@@ Should This Be Major/Minor/Patch/Exact Version?
            #       For Now we'll use Minor if verlib can parse it, else exact
            normalized = verlib.suggest_normalized_version(self.latest.version)
            if normalized is not None:
                ver = verlib.NormalizedVersion(normalized)
                next_version = "%(major)s.%(minor)s" % {"major": ver.parts[0][0], "minor": ver.parts[0][1] + 1}
                return "%(package)s>=%(current_version)s,<%(next_version)s" % {
                    "package": self.name,
                    "current_version": self.latest.version,
                    "next_version": next_version,
                }
            return "%(package)s==%(version)s" % {"package": self.name, "version": self.latest.version}

    def get_package_links(self):
        if not self.package_uri_migrated:
            for release in self.releases.all():
                release.save()
            self.package_uri_migrated = True
            self.save()
            return PackageURI.objects.filter(package=self)
        return self.package_links.all()


class PackageURI(models.Model):
    package = models.ForeignKey(Package, related_name="package_links")
    uri = models.URLField(max_length=400)

    class Meta:
        unique_together = ["package", "uri"]

    def __unicode__(self):
        return self.uri


class Release(models.Model):
    created = AutoCreatedField(_("created"), db_index=True)
    modified = AutoLastModifiedField(_("modified"))

    package = models.ForeignKey(Package, related_name="releases")
    version = models.CharField(max_length=512)

    hidden = models.BooleanField(default=False)
<commit_msg>wrap the release migration in a transaction<commit_after>import os
import posixpath
import urlparse
import uuid

import lxml.html

from docutils.core import publish_string

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models, transaction
from django.db.models import Sum
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.encoding import smart_str
from django.utils.translation import ugettext_lazy as _

from model_utils import Choices
from model_utils.fields import AutoCreatedField, AutoLastModifiedField
from model_utils.models import TimeStampedModel

from crate.fields import JSONField
from packages.utils import verlib


def release_file_upload_to(instance, filename):
    dsplit = instance.digest.split("$")
    if len(dsplit) == 2:
        directory = dsplit[1]
    else:
        directory = str(uuid.uuid4()).replace("-", "")

    return posixpath.join("packages", directory, filename)


# @@@ These are by Nature Hierarchical. Would we benefit from a tree structure?
class TroveClassifier(models.Model):
    trove = models.CharField(max_length=350, unique=True)

    def __unicode__(self):
        return self.trove


class Package(TimeStampedModel):
    name = models.SlugField(max_length=150, unique=True)
    package_uri_migrated = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse("package_detail", kwargs={"package": self.name})

    @property
    def downloads(self):
        total_downloads = ReleaseFile.objects.filter(release__package__pk=self.pk).aggregate(total_downloads=Sum("downloads"))["total_downloads"]
        if total_downloads is None:
            return 0
        return total_downloads

    @property
    def latest(self):
        if not hasattr(self, "_latest_release"):
            releases = self.releases.order_by("-order")[:1]
            if releases:
                self._latest_release = releases[0]
            else:
                self._latest_release = None
        return self._latest_release

    @property
    def install_command(self):
        return "pip install %(package)s" % {"package": self.name}

    @property
    def requirement_line(self):
        if self.latest is not None:
            # @@@ Should This Be Major/Minor/Patch/Exact Version?
            #       For Now we'll use Minor if verlib can parse it, else exact
            normalized = verlib.suggest_normalized_version(self.latest.version)
            if normalized is not None:
                ver = verlib.NormalizedVersion(normalized)
                next_version = "%(major)s.%(minor)s" % {"major": ver.parts[0][0], "minor": ver.parts[0][1] + 1}
                return "%(package)s>=%(current_version)s,<%(next_version)s" % {
                    "package": self.name,
                    "current_version": self.latest.version,
                    "next_version": next_version,
                }
            return "%(package)s==%(version)s" % {"package": self.name, "version": self.latest.version}

    def get_package_links(self):
        if not self.package_uri_migrated:
            with transaction.commit_on_success():
                for release in self.releases.all():
                    release.save()
                self.package_uri_migrated = True
                self.save()
            return PackageURI.objects.filter(package=self)
        return self.package_links.all()


class PackageURI(models.Model):
    package = models.ForeignKey(Package, related_name="package_links")
    uri = models.URLField(max_length=400)

    class Meta:
        unique_together = ["package", "uri"]

    def __unicode__(self):
        return self.uri


class Release(models.Model):
    created = AutoCreatedField(_("created"), db_index=True)
    modified = AutoLastModifiedField(_("modified"))

    package = models.ForeignKey(Package, related_name="releases")
    version = models.CharField(max_length=512)

    hidden = models.BooleanField(default=False)

{'fraction_non_alphanumeric': 0.10053436063129116, 'fraction_numerical': 0.005716416055672921, 'mean_word_length': 3.0099651220727455, 'pattern_counts': {'":': 18, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3207806', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2215, 'n_words': 539}