<commit_before># -*- coding: utf-8 -*-
from django.db import models
from django.db.models import permalink
from django.utils.translation import ugettext_lazy as _
from django.contrib.auth.models import User
from django.core.files import File

from newtagging.models import TagBase
from newtagging import managers


TAG_CATEGORIES = (
    ('author', _('author')),
    ('epoch', _('epoch')),
    ('kind', _('kind')),
    ('genre', _('genre')),
    ('theme', _('theme')),
    ('set', _('set')),
)


class TagSubcategoryManager(models.Manager):
    def __init__(self, subcategory):
        super(TagSubcategoryManager, self).__init__()
        self.subcategory = subcategory
        
    def get_query_set(self):
        return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)


class Tag(TagBase):
    name = models.CharField(_('name'), max_length=50, unique=True, db_index=True)
    slug = models.SlugField(_('slug'), unique=True, db_index=True)
    sort_key = models.SlugField(_('sort key'), db_index=True)
    category = models.CharField(_('category'), max_length=50, blank=False, null=False, 
        db_index=True, choices=TAG_CATEGORIES)
    description = models.TextField(blank=True)
    
    user = models.ForeignKey(User, blank=True, null=True)
    
    def has_description(self):
        return len(self.description) > 0
    has_description.short_description = _('Has description')
    has_description.boolean = True

    @permalink
    def get_absolute_url(self):
        return ('catalogue.views.tagged_book_list', [self.slug])
    
    class Meta:
        ordering = ('sort_key',)
        verbose_name = _('tag')
        verbose_name_plural = _('tags')
    
    def __unicode__(self):
        return self.name

    @staticmethod
    def get_tag_list(tags):
        if isinstance(tags, basestring):
            tag_slugs = tags.split('/')
            return [Tag.objects.get(slug=slug) for slug in tag_slugs]
        else:
            return TagBase.get_tag_list(tags)


class Book(models.Model):
    title = models.CharField(_('title'), max_length=120)
    slug = models.SlugField(_('slug'), unique=True, db_index=True)
    description = models.TextField(_('description'), blank=True)
    created_at = models.DateTimeField(_('creation date'), auto_now=True)
    
    # Formats
    xml_file = models.FileField(_('XML file'), upload_to='books/xml', blank=True)
    pdf_file = models.FileField(_('PDF file'), upload_to='books/pdf', blank=True)
    odt_file = models.FileField(_('ODT file'), upload_to='books/odt', blank=True)
    html_file = models.FileField(_('HTML file'), upload_to='books/html', blank=True)
    
    objects = managers.ModelTaggedItemManager(Tag)
    tags = managers.TagDescriptor(Tag)
    
    def has_description(self):
        return len(self.description) > 0
    has_description.short_description = _('Has description')
    has_description.boolean = True
    
    def has_pdf_file(self):
        return bool(self.pdf_file)
    has_pdf_file.short_description = 'PDF'
    has_pdf_file.boolean = True
    
    def has_odt_file(self):
        return bool(self.odt_file)
    has_odt_file.short_description = 'ODT'
    has_odt_file.boolean = True
    
    def has_html_file(self):
        return bool(self.html_file)
    has_html_file.short_description = 'HTML'
    has_html_file.boolean = True
    
    def save(self, **kwargs):
        try:
            from bin import book2html
            from django.conf import settings
            from os.path import splitext, basename
            from tempfile import NamedTemporaryFile
            
            html_file = NamedTemporaryFile()
            book2html.transform(self.xml_file.path, html_file)
            
            html_filename = '%s.html' % splitext(basename(self.xml_file.path))[0]
            self.html_file.save(html_filename, File(html_file), save=False)
        except ValueError:
            pass

        book = super(Book, self).save(**kwargs)
    
    @permalink
    def get_absolute_url(self):
        return ('catalogue.views.book_detail', [self.slug])
        
    class Meta:
        ordering = ('title',)
        verbose_name = _('book')<commit_msg>Removed unnecessary import.<commit_after># -*- coding: utf-8 -*-
from django.db import models
from django.db.models import permalink
from django.utils.translation import ugettext_lazy as _
from django.contrib.auth.models import User
from django.core.files import File

from newtagging.models import TagBase
from newtagging import managers


TAG_CATEGORIES = (
    ('author', _('author')),
    ('epoch', _('epoch')),
    ('kind', _('kind')),
    ('genre', _('genre')),
    ('theme', _('theme')),
    ('set', _('set')),
)


class TagSubcategoryManager(models.Manager):
    def __init__(self, subcategory):
        super(TagSubcategoryManager, self).__init__()
        self.subcategory = subcategory
        
    def get_query_set(self):
        return super(TagSubcategoryManager, self).get_query_set().filter(category=self.subcategory)


class Tag(TagBase):
    name = models.CharField(_('name'), max_length=50, unique=True, db_index=True)
    slug = models.SlugField(_('slug'), unique=True, db_index=True)
    sort_key = models.SlugField(_('sort key'), db_index=True)
    category = models.CharField(_('category'), max_length=50, blank=False, null=False, 
        db_index=True, choices=TAG_CATEGORIES)
    description = models.TextField(blank=True)
    
    user = models.ForeignKey(User, blank=True, null=True)
    
    def has_description(self):
        return len(self.description) > 0
    has_description.short_description = _('Has description')
    has_description.boolean = True

    @permalink
    def get_absolute_url(self):
        return ('catalogue.views.tagged_book_list', [self.slug])
    
    class Meta:
        ordering = ('sort_key',)
        verbose_name = _('tag')
        verbose_name_plural = _('tags')
    
    def __unicode__(self):
        return self.name

    @staticmethod
    def get_tag_list(tags):
        if isinstance(tags, basestring):
            tag_slugs = tags.split('/')
            return [Tag.objects.get(slug=slug) for slug in tag_slugs]
        else:
            return TagBase.get_tag_list(tags)


class Book(models.Model):
    title = models.CharField(_('title'), max_length=120)
    slug = models.SlugField(_('slug'), unique=True, db_index=True)
    description = models.TextField(_('description'), blank=True)
    created_at = models.DateTimeField(_('creation date'), auto_now=True)
    
    # Formats
    xml_file = models.FileField(_('XML file'), upload_to='books/xml', blank=True)
    pdf_file = models.FileField(_('PDF file'), upload_to='books/pdf', blank=True)
    odt_file = models.FileField(_('ODT file'), upload_to='books/odt', blank=True)
    html_file = models.FileField(_('HTML file'), upload_to='books/html', blank=True)
    
    objects = managers.ModelTaggedItemManager(Tag)
    tags = managers.TagDescriptor(Tag)
    
    def has_description(self):
        return len(self.description) > 0
    has_description.short_description = _('Has description')
    has_description.boolean = True
    
    def has_pdf_file(self):
        return bool(self.pdf_file)
    has_pdf_file.short_description = 'PDF'
    has_pdf_file.boolean = True
    
    def has_odt_file(self):
        return bool(self.odt_file)
    has_odt_file.short_description = 'ODT'
    has_odt_file.boolean = True
    
    def has_html_file(self):
        return bool(self.html_file)
    has_html_file.short_description = 'HTML'
    has_html_file.boolean = True
    
    def save(self, **kwargs):
        try:
            from bin import book2html
            from os.path import splitext, basename
            from tempfile import NamedTemporaryFile
            
            html_file = NamedTemporaryFile()
            book2html.transform(self.xml_file.path, html_file)
            
            html_filename = '%s.html' % splitext(basename(self.xml_file.path))[0]
            self.html_file.save(html_filename, File(html_file), save=False)
        except ValueError:
            pass

        book = super(Book, self).save(**kwargs)
    
    @permalink
    def get_absolute_url(self):
        return ('catalogue.views.book_detail', [self.slug])
        
    class Meta:
        ordering = ('title',)
        verbose_name = _('book')
{'fraction_non_alphanumeric': 0.12132837789233905, 'fraction_numerical': 0.0031171322383407264, 'mean_word_length': 3.119506172839506, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3055274', 'n_tokens_mistral': 2620, 'n_tokens_neox': 2593, 'n_words': 509}