<commit_before>import os

from django.db import models
from django.contrib.auth.models import User
from django.conf import settings
from django.utils.translation import gettext_lazy as _

from explorer import fields


class EditorSettings(models.Model):
    user = models.ForeignKey(User, unique=True)
    settings = fields.JSONField()
    
    class Meta:
        verbose_name, verbose_name_plural = _("editor settings"), _("editor settings")

    def __unicode__(self):
        return u"Editor settings for %s" % self.user.username


class Book(models.Model):
    class Meta:
        permissions = (
            ("can_add_files", "Can do hg add."),
        )
    pass


class PullRequest(models.Model):
    comitter = models.ForeignKey(User) # the user who request the pull 
    file = models.CharField(max_length=256) # the file to request
    source_rev = models.CharField(max_length=40) # revision number of the commiter

    comment = models.TextField() # addtional comments to the request

    # revision number in which the changes were merged (if any)
    merged_rev = models.CharField(max_length=40, null=True) 
    
    def __unicode__(self):
        return u"Pull request from %s, source: %s %s, status: %s." % \
            (self.commiter, self.file, self.source_rev, \
                (("merged into "+self.merged_rev) if self.merged_rev else "pending") )

    
def get_image_folders():
    return sorted(fn for fn in os.listdir(os.path.join(settings.MEDIA_ROOT, settings.IMAGE_DIR)) if not fn.startswith('.'))


def get_images_from_folder(folder):
    return sorted(settings.MEDIA_URL + settings.IMAGE_DIR + '/' + folder + '/' + fn for fn 
            in os.listdir(os.path.join(settings.MEDIA_ROOT, settings.IMAGE_DIR, folder))
            if not fn.startswith('.'))
<commit_msg>Patch to handle unicode image names.<commit_after>import os

from django.db import models
from django.contrib.auth.models import User
from django.conf import settings
from django.utils.translation import gettext_lazy as _

from explorer import fields


class EditorSettings(models.Model):
    user = models.ForeignKey(User, unique=True)
    settings = fields.JSONField()
    
    class Meta:
        verbose_name, verbose_name_plural = _("editor settings"), _("editor settings")

    def __unicode__(self):
        return u"Editor settings for %s" % self.user.username


class Book(models.Model):
    class Meta:
        permissions = (
            ("can_add_files", "Can do hg add."),
        )
    pass


class PullRequest(models.Model):
    comitter = models.ForeignKey(User) # the user who request the pull 
    file = models.CharField(max_length=256) # the file to request
    source_rev = models.CharField(max_length=40) # revision number of the commiter

    comment = models.TextField() # addtional comments to the request

    # revision number in which the changes were merged (if any)
    merged_rev = models.CharField(max_length=40, null=True) 
    
    def __unicode__(self):
        return u"Pull request from %s, source: %s %s, status: %s." % \
            (self.commiter, self.file, self.source_rev, \
                (("merged into "+self.merged_rev) if self.merged_rev else "pending") )

    
def get_image_folders():
    return sorted(fn for fn in os.listdir(os.path.join(settings.MEDIA_ROOT, settings.IMAGE_DIR)) if not fn.startswith('.'))


def get_images_from_folder(folder):
    return sorted(settings.MEDIA_URL + settings.IMAGE_DIR + u'/' + folder + u'/' + fn.decode('utf-8') for fn
            in os.listdir(os.path.join(settings.MEDIA_ROOT, settings.IMAGE_DIR, folder))
            if not fn.decode('utf-8').startswith('.'))

{'fraction_non_alphanumeric': 0.10778935820076796, 'fraction_numerical': 0.004388370817334065, 'mean_word_length': 3.2805164319248825, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2722926', 'n_tokens_mistral': 1129, 'n_tokens_neox': 1080, 'n_words': 307}