<commit_before>import os
from django.db import models
from django.utils.translation import ugettext as _

#from django_extensions.db.fields import AutoSlugField


class BaseModel(models.Model):
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)

    class Meta(object):
        abstract = True


class Collection(BaseModel):
    """Collection of images in a certain base_dir"""

    SORT_NAME_ASC = 'name_asc'
    SORT_NAME_DESC = 'name_desc'
    SORT_DATE_ASC = 'date_asc'
    SORT_DATE_DESC = 'date_desc'
    SORT_OPTIONS = (
        (SORT_NAME_ASC, 'Name ascending'),
        (SORT_NAME_DESC, 'Name descending'),
        (SORT_DATE_ASC, 'Date ascending'),
        (SORT_DATE_DESC, 'Date descending'),
    )

    FEATURED_FIRST = 'first'
    FEATURED_LATEST = 'latest'
    FEATURED_OPTIONS = (
        (FEATURED_FIRST, 'First image in (sub)album'),
        (FEATURED_LATEST, 'Newest image in collection'),
    )

    title = models.CharField(max_length=255)
    slug = models.SlugField(max_length=255, unique=True)
    base_dir = models.CharField(max_length=255, unique=True)
    archive_dir = models.CharField(max_length=255, blank=True)
    base_uri = models.CharField(max_length=255, blank=True, help_text='Base URI of original images')
    archive_uri = models.CharField(max_length=255, blank=True, help_text='Base URI of image archive')

    # Flat or nested into directories
    flat = models.BooleanField(default=False, help_text='Flatten a collection, or keep the nesting in directories')

    sortmethod = models.CharField(max_length=10, choices=SORT_OPTIONS, default=SORT_DATE_DESC)
    featured_default = models.CharField(max_length=10, choices=FEATURED_OPTIONS, default=FEATURED_LATEST)

    description = models.TextField(null=True, blank=True)

    is_public = models.BooleanField(default=False, help_text='If public, the collection is visible for the world')
    needs_authentication = models.BooleanField(default=False, help_text='User needs to be logged in to view the images')

    def nr_directories(self):
        return Directory.objects.filter(collection__pk=self.pk).count()

    def nr_images(self):
        total = 0
        for directory in Directory.objects.filter(collection__pk=self.pk):
            total += directory.nr_images()
        return total

    def images(self):
        images = []
        directories = Directory.objects.filter(collection=self)
        for directory in directories:
            images += directory.images(self.sortmethod)
        return images

    def get_featured_image(self):
        directories = Directory.objects.filter(collection=self).order_by('relative_path')
        if directories:
            return directories[0].get_featured_image()
        else:
            return None

    def __unicode__(self):
        return '{0} ({1})'.format(self.title, self.base_dir)


class Directory(BaseModel):
    """Directory/collection umbrella object"""
    directory = models.CharField(max_length=255)
    relative_path = models.CharField(max_length=255, help_text='Path relative to Collection base dir')
    collection = models.ForeignKey(Collection, on_delete=models.CASCADE)

    title = models.CharField(max_length=255, blank=True, null=True)
    description = models.TextField(null=True, blank=True)
    featured_image = models.ForeignKey('Image', blank=True, null=True, related_name='featured_image', on_delete=models.SET_NULL)

    parent_directory = models.ForeignKey('self', null=True, blank=True, related_name='children', on_delete=models.CASCADE)

    def get_filepath(self, filename):
        return '{0}{1}'.format(self.directory, filename)

    @property
    def dir_name(self):
        if self.title:
            return self.title
        else:
            title = os.path.basename(self.relative_path)
            title = title.replace('_', ' ')
            return title[:1].upper() + title[1:]

    def nr_images(self):
        return Image.objects.filter(directory__pk=self.pk).count()

    def images(self, sortmethod):
        """ Return all images from this dir, sorted as Collection.sortmethod """
        result = Image.objects.filter(directory=self)
        if sortmethod == Collection.SORT_NAME_DESC:
            return result.order_by('-filename')
        elif sortmethod == Collection.SORT_NAME_ASC:
            return result.order_by('filename')
        elif sortmethod == Collection.SORT_DATE_DESC:
            return result.order_by('-filter_modified')
        elif sortmethod == Collection.SORT_DATE_ASC:
            return result.order_by('filter_modified')

    def get_featured_image(self):
        if self.featured_image:
            return self.featured_image
        else:
            if self.collection.featured_default == self.collection.FEATURED_FIRST:
                images = self.images(self.collection.sortmethod)
                if images:
                    return images[0]
                else:
                    try:
                        for directory in Directory.objects.filter(parent_directory=self).order_by('relative_path'):
                            images = directory.images(self.collection.sortmethod)
                            if images:
                                return images[0]
                    except Directory.DoesNotExist:<commit_msg>Fixed crash on feature_image lookup when still indexing/incomplete<commit_after>import os
from django.db import models
from django.utils.translation import ugettext as _

#from django_extensions.db.fields import AutoSlugField


class BaseModel(models.Model):
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)

    class Meta(object):
        abstract = True


class Collection(BaseModel):
    """Collection of images in a certain base_dir"""

    SORT_NAME_ASC = 'name_asc'
    SORT_NAME_DESC = 'name_desc'
    SORT_DATE_ASC = 'date_asc'
    SORT_DATE_DESC = 'date_desc'
    SORT_OPTIONS = (
        (SORT_NAME_ASC, 'Name ascending'),
        (SORT_NAME_DESC, 'Name descending'),
        (SORT_DATE_ASC, 'Date ascending'),
        (SORT_DATE_DESC, 'Date descending'),
    )

    FEATURED_FIRST = 'first'
    FEATURED_LATEST = 'latest'
    FEATURED_OPTIONS = (
        (FEATURED_FIRST, 'First image in (sub)album'),
        (FEATURED_LATEST, 'Newest image in collection'),
    )

    title = models.CharField(max_length=255)
    slug = models.SlugField(max_length=255, unique=True)
    base_dir = models.CharField(max_length=255, unique=True)
    archive_dir = models.CharField(max_length=255, blank=True)
    base_uri = models.CharField(max_length=255, blank=True, help_text='Base URI of original images')
    archive_uri = models.CharField(max_length=255, blank=True, help_text='Base URI of image archive')

    # Flat or nested into directories
    flat = models.BooleanField(default=False, help_text='Flatten a collection, or keep the nesting in directories')

    sortmethod = models.CharField(max_length=10, choices=SORT_OPTIONS, default=SORT_DATE_DESC)
    featured_default = models.CharField(max_length=10, choices=FEATURED_OPTIONS, default=FEATURED_LATEST)

    description = models.TextField(null=True, blank=True)

    is_public = models.BooleanField(default=False, help_text='If public, the collection is visible for the world')
    needs_authentication = models.BooleanField(default=False, help_text='User needs to be logged in to view the images')

    def nr_directories(self):
        return Directory.objects.filter(collection__pk=self.pk).count()

    def nr_images(self):
        total = 0
        for directory in Directory.objects.filter(collection__pk=self.pk):
            total += directory.nr_images()
        return total

    def images(self):
        images = []
        directories = Directory.objects.filter(collection=self)
        for directory in directories:
            images += directory.images(self.sortmethod)
        return images

    def get_featured_image(self):
        directories = list(Directory.objects.filter(collection=self).order_by('relative_path'))
        if directories:
            return directories[0].get_featured_image()
        else:
            return None

    def __unicode__(self):
        return '{0} ({1})'.format(self.title, self.base_dir)


class Directory(BaseModel):
    """Directory/collection umbrella object"""
    directory = models.CharField(max_length=255)
    relative_path = models.CharField(max_length=255, help_text='Path relative to Collection base dir')
    collection = models.ForeignKey(Collection, on_delete=models.CASCADE)

    title = models.CharField(max_length=255, blank=True, null=True)
    description = models.TextField(null=True, blank=True)
    featured_image = models.ForeignKey('Image', blank=True, null=True, related_name='featured_image', on_delete=models.SET_NULL)

    parent_directory = models.ForeignKey('self', null=True, blank=True, related_name='children', on_delete=models.CASCADE)

    def get_filepath(self, filename):
        return '{0}{1}'.format(self.directory, filename)

    @property
    def dir_name(self):
        if self.title:
            return self.title
        else:
            title = os.path.basename(self.relative_path)
            title = title.replace('_', ' ')
            return title[:1].upper() + title[1:]

    def nr_images(self):
        return Image.objects.filter(directory__pk=self.pk).count()

    def images(self, sortmethod):
        """ Return all images from this dir, sorted as Collection.sortmethod """
        result = Image.objects.filter(directory=self)
        if sortmethod == Collection.SORT_NAME_DESC:
            return result.order_by('-filename')
        elif sortmethod == Collection.SORT_NAME_ASC:
            return result.order_by('filename')
        elif sortmethod == Collection.SORT_DATE_DESC:
            return result.order_by('-filter_modified')
        elif sortmethod == Collection.SORT_DATE_ASC:
            return result.order_by('filter_modified')

    def get_featured_image(self):
        if self.featured_image:
            return self.featured_image
        else:
            if self.collection.featured_default == self.collection.FEATURED_FIRST:
                images = list(self.images(self.collection.sortmethod))
                if images:
                    return images[0]
                else:
                    try:
                        for directory in Directory.objects.filter(parent_directory=self).order_by('relative_path'):
                            images = list(directory.images(self.collection.sortmethod))
                            if images:
                                return images[0]
                    except Directory.DoesNotExist:
{'fraction_non_alphanumeric': 0.09943155344329513, 'fraction_numerical': 0.007641412729475352, 'mean_word_length': 3.0666919287608945, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1534380', 'n_tokens_mistral': 3147, 'n_tokens_neox': 3028, 'n_words': 644}