<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import json
import logging
import datetime
import random
from flask import url_for, current_app
from flask.ext.security import current_user
from flask.ext.admin.babel import lazy_gettext
from flask.ext.admin import form
from jinja2 import Markup
from quokka.core.db import db
from quokka import admin
from quokka.core.admin.models import ModelAdmin
from quokka.modules.accounts.models import User
from quokka.utils.text import slugify
from quokka import settings

logger = logging.getLogger()


###############################################################
# Commom extendable base classes
###############################################################


class Publishable(object):
    published = db.BooleanField(default=False)
    available_at = db.DateTimeField(default=datetime.datetime.now)
    available_until = db.DateTimeField(required=False)
    created_at = db.DateTimeField(default=datetime.datetime.now)
    updated_at = db.DateTimeField(default=datetime.datetime.now)
    created_by = db.ReferenceField(User, reverse_delete_rule=db.DENY)
    last_updated_by = db.ReferenceField(User, reverse_delete_rule=db.DENY)

    def save(self, *args, **kwargs):
        self.updated_at = datetime.datetime.now()

        try:
            user = User.objects.get(id=current_user.id)
            if not self.id:
                self.created_by = user
            self.last_updated_by = user
        except Exception as e:
            logger.warning("No user to save the model: %s" % e.message)

        super(Publishable, self).save(*args, **kwargs)


class Slugged(object):
    slug = db.StringField(max_length=255, required=True)
    long_slug = db.StringField(unique=True, required=True)
    mpath = db.StringField()

    def _create_mpath_long_slug(self):
        try:
            if self.parent and self.parent != self:
                self.long_slug = "/".join(
                    [self.parent.long_slug, self.slug]
                )

                self.mpath = "".join(
                    [self.parent.mpath, self.slug, ',']
                )
            else:
                self.long_slug = self.slug
                self.mpath = ",%s," % self.slug
        except:
            logger.info("excepting to content validate_long_slug")
            self.long_slug = "/".join(
                [self.channel.long_slug, self.slug]
            )

            self.mpath = "".join([self.channel.mpath, self.slug, ','])

    def validate_long_slug(self):

        self._create_mpath_long_slug()

        filters = dict(long_slug=self.long_slug)
        if self.id:
            filters['id__ne'] = self.id

        exist = self.__class__.objects(**filters)
        if exist.count():
            if current_app.config.get('SMART_SLUG_ENABLED', False):
                self.slug = "{0}-{1}".format(self.slug, random.getrandbits(32))
                self._create_mpath_long_slug()
            else:
                raise db.ValidationError(
                    lazy_gettext("%(slug)s slug already exists",
                                 slug=self.long_slug)
                )

    def validate_slug(self, title=None):
        if self.slug:
            self.slug = slugify(self.slug)
        else:
            self.slug = slugify(title or self.title)


class Comment(db.EmbeddedDocument):
    body = db.StringField(verbose_name="Comment", required=True)
    author = db.StringField(verbose_name="Name", max_length=255, required=True)
    published = db.BooleanField(default=True)
    created_at = db.DateTimeField(default=datetime.datetime.now)
    created_by = db.ReferenceField(User)  # reverse_delete_rule not supported

    def __unicode__(self):
        return "{0}-{1}...".format(self.author, self.body[:10])

    meta = {
        'indexes': ['-created_at', '-available_at'],
        'ordering': ['-created_at']
    }


class Commentable(object):
    comments = db.ListField(db.EmbeddedDocumentField(Comment))


class Tagged(object):
    tags = db.ListField(db.StringField(max_length=50))


class CustomValue(db.EmbeddedDocument):

    FORMATS = (
        ('json', "json"),
        ('text', "text"),
        ('int', "int"),
        ('float', "float"),
    )

    DEFAULT_FORMATTER = lambda value: value

    FORMATTERS = {
        'json': lambda value: json.loads(value),
        'text': DEFAULT_FORMATTER,
        'int': lambda value: int(value),
        'float': lambda value: float(value)
    }

    REVERSE_FORMATTERS = {
        'json': lambda value:
        value if isinstance(value, str) else json.dumps(value),
        'text': DEFAULT_FORMATTER,
        'int': DEFAULT_FORMATTER,
        'float': DEFAULT_FORMATTER
    }

    name = db.StringField(max_length=50, required=True)
    rawvalue = db.StringField(verbose_name=lazy_gettext("Value"),
                              required=True)
    formatter = db.StringField(choices=FORMATS, default="text", required=True)

    @property
    def value(self):
        return self.FORMATTERS.get(self.formatter,
                                   self.DEFAULT_FORMATTER)(self.rawvalue)

    @value.setter
    def value(self, value):  # lint:ok
        self.rawvalue = self.REVERSE_FORMATTERS.get(self.formatter,
                                                    self.STR_FORMATTER)(value)

    def clean(self):
        try:
            self.value
        except Exception as e:
            raise Exception(e.message)
        super(CustomValue, self).clean()

    def __unicode__(self):
        return self.name


class HasCustomValue(object):
    values = db.ListField(db.EmbeddedDocumentField(CustomValue))

    def clean(self):
        current_names = [value.name for value in self.values]
        for name in current_names:
            if current_names.count(name) > 1:
                raise Exception(lazy_gettext("%(name)s already exists",
                                             name=name))
        super(HasCustomValue, self).clean()


class Imaged(object):
    main_image = db.ReferenceField("Image", required=False,
                                   reverse_delete_rule=db.NULLIFY)
    main_image_caption = db.StringField(max_length=255)


class Channel(HasCustomValue, Publishable, Slugged, db.DynamicDocument):
    title = db.StringField(max_length=255, required=True)
    description = db.StringField()
    show_in_menu = db.BooleanField(default=False)
    is_homepage = db.BooleanField(default=False)
    include_in_rss = db.BooleanField(default=False)
    indexable = db.BooleanField(default=True)
    canonical_url = db.StringField()
    order = db.IntField(default=0)

    # MPTT
    parent = db.ReferenceField('self', required=False, default=None,
                               reverse_delete_rule=db.DENY)

    def get_ancestors(self, menu=True):
        return self.__class__.objects(parent=self, show_in_menu=menu)
<commit_msg>implemented channel_type and channel_aliases<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import json
import logging
import datetime
import random
from flask import url_for, current_app
from flask.ext.security import current_user
from flask.ext.admin.babel import lazy_gettext
from flask.ext.admin import form
from jinja2 import Markup
from quokka.core.db import db
from quokka import admin
from quokka.core.admin.models import ModelAdmin
from quokka.modules.accounts.models import User
from quokka.utils.text import slugify
from quokka import settings

logger = logging.getLogger()


###############################################################
# Commom extendable base classes
###############################################################


class Publishable(object):
    published = db.BooleanField(default=False)
    available_at = db.DateTimeField(default=datetime.datetime.now)
    available_until = db.DateTimeField(required=False)
    created_at = db.DateTimeField(default=datetime.datetime.now)
    updated_at = db.DateTimeField(default=datetime.datetime.now)
    created_by = db.ReferenceField(User, reverse_delete_rule=db.DENY)
    last_updated_by = db.ReferenceField(User, reverse_delete_rule=db.DENY)

    def save(self, *args, **kwargs):
        self.updated_at = datetime.datetime.now()

        try:
            user = User.objects.get(id=current_user.id)
            if not self.id:
                self.created_by = user
            self.last_updated_by = user
        except Exception as e:
            logger.warning("No user to save the model: %s" % e.message)

        super(Publishable, self).save(*args, **kwargs)


class Slugged(object):
    slug = db.StringField(max_length=255, required=True)
    long_slug = db.StringField(unique=True, required=True)
    mpath = db.StringField()

    def _create_mpath_long_slug(self):
        try:
            if self.parent and self.parent != self:
                self.long_slug = "/".join(
                    [self.parent.long_slug, self.slug]
                )

                self.mpath = "".join(
                    [self.parent.mpath, self.slug, ',']
                )
            else:
                self.long_slug = self.slug
                self.mpath = ",%s," % self.slug
        except:
            logger.info("excepting to content validate_long_slug")
            self.long_slug = "/".join(
                [self.channel.long_slug, self.slug]
            )

            self.mpath = "".join([self.channel.mpath, self.slug, ','])

    def validate_long_slug(self):

        self._create_mpath_long_slug()

        filters = dict(long_slug=self.long_slug)
        if self.id:
            filters['id__ne'] = self.id

        exist = self.__class__.objects(**filters)
        if exist.count():
            if current_app.config.get('SMART_SLUG_ENABLED', False):
                self.slug = "{0}-{1}".format(self.slug, random.getrandbits(32))
                self._create_mpath_long_slug()
            else:
                raise db.ValidationError(
                    lazy_gettext("%(slug)s slug already exists",
                                 slug=self.long_slug)
                )

    def validate_slug(self, title=None):
        if self.slug:
            self.slug = slugify(self.slug)
        else:
            self.slug = slugify(title or self.title)


class Comment(db.EmbeddedDocument):
    body = db.StringField(verbose_name="Comment", required=True)
    author = db.StringField(verbose_name="Name", max_length=255, required=True)
    published = db.BooleanField(default=True)
    created_at = db.DateTimeField(default=datetime.datetime.now)
    created_by = db.ReferenceField(User)  # reverse_delete_rule not supported

    def __unicode__(self):
        return "{0}-{1}...".format(self.author, self.body[:10])

    meta = {
        'indexes': ['-created_at', '-available_at'],
        'ordering': ['-created_at']
    }


class Commentable(object):
    comments = db.ListField(db.EmbeddedDocumentField(Comment))


class Tagged(object):
    tags = db.ListField(db.StringField(max_length=50))


class CustomValue(db.EmbeddedDocument):

    FORMATS = (
        ('json', "json"),
        ('text', "text"),
        ('int', "int"),
        ('float', "float"),
    )

    DEFAULT_FORMATTER = lambda value: value

    FORMATTERS = {
        'json': lambda value: json.loads(value),
        'text': DEFAULT_FORMATTER,
        'int': lambda value: int(value),
        'float': lambda value: float(value)
    }

    REVERSE_FORMATTERS = {
        'json': lambda value:
        value if isinstance(value, str) else json.dumps(value),
        'text': DEFAULT_FORMATTER,
        'int': DEFAULT_FORMATTER,
        'float': DEFAULT_FORMATTER
    }

    name = db.StringField(max_length=50, required=True)
    rawvalue = db.StringField(verbose_name=lazy_gettext("Value"),
                              required=True)
    formatter = db.StringField(choices=FORMATS, default="text", required=True)

    @property
    def value(self):
        return self.FORMATTERS.get(self.formatter,
                                   self.DEFAULT_FORMATTER)(self.rawvalue)

    @value.setter
    def value(self, value):  # lint:ok
        self.rawvalue = self.REVERSE_FORMATTERS.get(self.formatter,
                                                    self.STR_FORMATTER)(value)

    def clean(self):
        try:
            self.value
        except Exception as e:
            raise Exception(e.message)
        super(CustomValue, self).clean()

    def __unicode__(self):
        return self.name


class HasCustomValue(object):
    values = db.ListField(db.EmbeddedDocumentField(CustomValue))

    def clean(self):
        current_names = [value.name for value in self.values]
        for name in current_names:
            if current_names.count(name) > 1:
                raise Exception(lazy_gettext("%(name)s already exists",
                                             name=name))
        super(HasCustomValue, self).clean()


class Imaged(object):
    main_image = db.ReferenceField("Image", required=False,
                                   reverse_delete_rule=db.NULLIFY)
    main_image_caption = db.StringField(max_length=255)


class ChannelConfigs(object):
    content_filters = db.DictField(required=False)
    inherit_parent = db.BooleanField(default=False)


class ChannelType(HasCustomValue, ChannelConfigs, db.DynamicDocument):
    title = db.StringField(max_length=255, required=True)
    identifier = db.StringField(max_length=255, required=True, unique=True)
    template_suffix = db.StringField(max_length=255, required=True)
    theme_name = db.StringField(max_length=255, required=False)

    def __unicode__(self):
        return self.title


class Channel(HasCustomValue, Publishable, Slugged,
              ChannelConfigs, db.DynamicDocument):
    title = db.StringField(max_length=255, required=True)
    description = db.StringField()
    show_in_menu = db.BooleanField(default=False)
    is_homepage = db.BooleanField(default=False)
    include_in_rss = db.BooleanField(default=False)
    indexable = db.BooleanField(default=True)
    canonical_url = db.StringField()
    order = db.IntField(default=0)

    # MPTT
    parent = db.ReferenceField('self', required=False, default=None,
                               reverse_delete_rule=db.DENY)

    aliases = db.ListField(db.StringField())
    channel_type = db.ReferenceField(ChannelType, required=False,
                                     reverse_delete_rule=db.DENY)

    def get_ancestors(self, menu=True):
        return self.__class__.objects(parent=self, show_in_menu=menu)

{'fraction_non_alphanumeric': 0.12567892746648332, 'fraction_numerical': 0.0046751460983155725, 'mean_word_length': 2.5486704074164432, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6573722', 'n_tokens_mistral': 4359, 'n_tokens_neox': 4222, 'n_words': 815}