<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import math
import os.path
import uuid
from collections import defaultdict

import magic
import requests

from datetime import date

from bs4 import BeautifulSoup
from django.utils.lru_cache import lru_cache

from six.moves.urllib.parse import urlparse, urlencode

from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import AnonymousUser
from django.utils.encoding import smart_text
from django.core.urlresolvers import reverse
from django.core.files.base import ContentFile
from django.utils.html import strip_tags
from django.utils.translation import ugettext_lazy as _
from django.db.models import Q, signals, F
from django.contrib.contenttypes.fields import GenericForeignKey
from django.db.models import ObjectDoesNotExist
from django.utils.functional import cached_property
from django.conf import settings

from pybb.compat import update_fields, AUTH_USER_MODEL, queryset
from pybb.util import unescape, get_model_string, tznow
from pybb.base import ModelBase, ManagerBase, QuerySetBase
from pybb.models.mixins import ParentForumQuerysetMixin, ParentForumManagerMixin, ParentForumBase
from pybb.subscription import notify_topic_subscribers
from pybb import defaults
from pybb.fields import ContentTypeRestrictedFileField, CAStorage
from pybb.tasks import generate_markup, sync_cover
from pybb.processors import markup

from autoslug import AutoSlugField

if 'sorl.thumbnail' in settings.INSTALLED_APPS:
    from sorl.thumbnail import ImageField
else:
    from django.db.models import ImageField


try:
    from south.modelsinspector import add_introspection_rules
    add_introspection_rules([], ["^annoying\.fields\.JSONField"])
    add_introspection_rules([], ["^annoying\.fields\.AutoOneToOneField"])
except ImportError:
    pass


class ModeratorManager(ManagerBase):
    def contribute_to_class(self, cls, name):
        signals.post_delete.connect(self.post_delete, sender=cls)
        return super(ModeratorManager, self).contribute_to_class(cls, name)

    def post_delete(self, instance, **kwargs):
        from pybb.proxies import UserObjectPermission

        UserObjectPermission.objects.get_for_object(instance.user, instance.forum).delete()

        (instance.user.user_permissions.through.objects.filter(permission__codename__in=defaults.PYBB_USER_PERMISSIONS,
                                                               user=instance.user)
         .delete())


@python_2_unicode_compatible
class BaseModerator(ModelBase):
    forum = models.ForeignKey(get_model_string('Forum'))
    user = models.ForeignKey(AUTH_USER_MODEL)

    objects = ModeratorManager()

    class Meta:
        app_label = 'pybb'
        verbose_name = _('Moderator')
        verbose_name_plural = _('Moderators')
        abstract = True

    def __str__(self):
        return _('Moderator %(user)s of %(forum)s') % {
            'user': self.user,
            'forum': self.forum
        }

    @property
    def permissions(self):
        from pybb.proxies import UserObjectPermission

        permissions = list(self.user.user_permissions.all())
        permissions += [obj_perm.permission
                        for obj_perm in (UserObjectPermission.objects.get_for_object(user=self.user,
                                                                                     obj=self.forum)
                                         .select_related('permission'))]

        return permissions


class ForumQuerySet(ParentForumQuerysetMixin, QuerySetBase):
    def filter_by_user(self, user, hidden=True):
        if user.is_staff or user.is_superuser:
            return self

        if user.is_authenticated():
            return self.filter(staff=False)

        if hidden:
            return self.filter(hidden=False, staff=False)

        return self.filter(staff=False)


@queryset
class ForumManager(ParentForumManagerMixin, ManagerBase):
    def contribute_to_class(self, cls, name):
        signals.post_save.connect(self.post_save, sender=cls)
        signals.post_delete.connect(self.post_delete, sender=cls)
        return super(ForumManager, self).contribute_to_class(cls, name)

    def post_save(self, instance, **kwargs):
        if kwargs.get('created', False) and instance.forum_id:
            instance.forum.compute()

    def post_delete(self, instance, **kwargs):
        if instance.forum_id:
            instance.forum.compute()

    def get_queryset(self):
        return ForumQuerySet(self.model)

    def filter_by_user(self, *args, **kwargs):
        return self.get_queryset().filter_by_user(*args, **kwargs)


@lru_cache()
def get_moderator_ids_by_forum():
    from pybb.models import Moderator

    result = defaultdict(list)
    for moderator in Moderator.objects.all():
        result[moderator.forum_id].append(moderator.user_id)

    return result


@python_2_unicode_compatible
class BaseForum(ParentForumBase):
    forum = models.ForeignKey('Forum', related_name='forums',
                              verbose_name=_('Parent'), null=True, blank=True)
    name = models.CharField(_('Name'), max_length=80)
    slug = AutoSlugField(populate_from='name', max_length=80)
    position = models.IntegerField(_('Position'), blank=True, default=0, db_index=True)
    description = models.TextField(_('Description'), blank=True)
    moderators = models.ManyToManyField(AUTH_USER_MODEL, blank=True, verbose_name=_('Moderators'), through=get_model_string('Moderator'))
    updated = models.DateTimeField(_('Updated'), blank=True, null=True)
    post_count = models.IntegerField(_('Post count'), blank=True, default=0, db_index=True)
    topic_count = models.IntegerField(_('Topic count'), blank=True, default=0, db_index=True)
    forum_count = models.PositiveIntegerField(default=0, db_index=True)
    readed_by = models.ManyToManyField(AUTH_USER_MODEL, through=get_model_string('ForumReadTracker'), related_name='readed_forums')
    headline = models.TextField(_('Headline'), blank=True, null=True)
    hidden = models.BooleanField(_('Hidden'), blank=False, null=False,
                                 default=False, db_index=True)
    staff = models.BooleanField(_('Staff only'), blank=False,
                                null=False, default=False,
                                db_index=True)
    empty = models.BooleanField(_('Empty'),
                                default=False,
                                db_index=True,
                                help_text=_('If the forum is empty, you can\'t post into it'))

    last_topic = models.ForeignKey(get_model_string('Topic'),
                                   null=True,
                                   blank=True,
                                   related_name='last_forums',
                                   on_delete=models.SET_NULL)

    last_post = models.ForeignKey(get_model_string('Post'),
                                  null=True,
                                  blank=True,
                                  on_delete=models.SET_NULL)

    objects = ForumManager()

    class Meta(object):
        verbose_name = _('Forum')
        verbose_name_plural = _('Forums')
        ordering = ['position']
        permissions = (
            ('can_unstick_topic', _('Can unstick topic')),
            ('can_stick_topic', _('Can stick topic')),
            ('can_open_topic', _('Can open topic')),
            ('can_close_topic', _('Can close topic')),
            ('can_merge_topic', _('Can merge topic')),
            ('can_delete_topic', _('Can delete topic')),
            ('can_change_topic', _('Can edit topic')),
            ('can_move_topic', _('Can move topic')),
            ('can_move_post', _('Can move post')),
            ('can_change_poll', _('Can edit a poll')),
            ('can_publish_announce', _('Can publish an announce')),
            ('can_change_post', _('Can edit post')),
            ('can_delete_post', _('Can delete post')),
            ('can_see_user_ip', _('Can see IP Address')),
            ('can_change_attachment', _('Can moderate attachments')),
        )
        app_label = 'pybb'
        abstract = True

    def compute(self, commit=True):
        forum_count = self.forums.count()

        from pybb.models import Topic

        res = Topic.objects.visible().filter(forum_id=self.pk).aggregate(post_count=models.Sum('post_count'))

        post_count = res['post_count'] or 0

        topic_count = Topic.objects.filter(forum_id=self.pk).visible().count() or 0

        res = self.__class__.objects.filter(forum_id=self.pk).aggregate(post_count=models.Sum('post_count'),
                                                                        topic_count=models.Sum('topic_count'),
                                                                        forum_count=models.Sum('forum_count'))

        self.post_count = post_count + (res['post_count'] or 0)

        self.topic_count = topic_count + (res['topic_count'] or 0)

        self.forum_count = forum_count + (res['forum_count'] or 0)

        if commit:
            self.save()

        if self.forum_id and self.forum_id != self.pk:
            self.forum.compute(commit=commit)

    def is_moderated_by(self, user, permission=None):
        if user.is_superuser or user.is_staff:
            return True

        if (user.is_authenticated() and
                user.pk in get_moderator_ids_by_forum()[self.pk]):

            if permission:
                return user.has_perm(permission, self)

            return True

        return False

    def is_hidden(self):
        return self.hidden or (self.forum_id and self.forum.is_hidden())

    def is_accessible_by(self, user, hidden=True):
        if self.forum_id and not self.forum.is_accessible_by(user, hidden=hidden):
            return False

        if self.staff and not user.is_staff:
            return False

        if hidden and self.is_hidden() and not user.is_authenticated():
            return False

        return True

    def __str__(self):
        return self.name

    def update_counters(self, commit=True):
        last_post = self.get_last_post()

        if last_post:
            self.updated = last_post.created

            self.last_post = last_post
            self.last_topic = last_post.topic

        self.compute(commit=commit)

        if commit:
            self.save()

    def get_absolute_url(self):
        return reverse('pybb:forum_detail', kwargs={'slug': self.slug})

    @property
    def posts(self):
        from pybb.models import Post

        return Post.objects.filter(topic__forum=self).visible().select_related()

    def get_last_post(self):
        from pybb.models import Post

        post = None

        try:
            topic = self.topics.visible().order_by('-updated')[0]
            post = topic.last_post

            if post:
                post.topic = topic
        except (IndexError, Post.DoesNotExist):
            pass

        if post and post.topic.poll_id:
            try:
                other = self.topics.visible().filter(poll__isnull=True).order_by('-updated')[0].last_post
            except (IndexError, Post.DoesNotExist):
                pass
            else:
                if post:
                    if other.created > post.created:
                        post = other
                else:
                    post = other

        return post

    def get_last_topic(self):
        topic = None

        try:
            topic = self.topics.order_by('-updated').visible().select_related()[0]
        except IndexError:
            return None

        try:
            forum_topic = self.forums.order_by('-updated').filter(last_topic__isnull=False)[0].last_topic

            if topic:
                if forum_topic.updated > topic.updated:
                    topic = forum_topic
            else:
                topic = forum_topic
        except IndexError:
            pass

        return topic

    @classmethod
    def watch_forum(cls, old_attr={}, new_attr={}, instance=None, sender=None, **kw):
        from pybb.models import Forum

        if (old_attr.get('forum_id', None) and
                old_attr.get('forum_id') != new_attr.get('forum_id')):
            instance.save(_signal=False)

            try:
                Forum.objects.get(pk=old_attr.get('forum_id')).update_counters()
            except ObjectDoesNotExist:
                pass


class BaseTopicRedirection(ModelBase):
    TYPE_PERMANENT_REDIRECT = 1
    TYPE_NO_REDIRECT = 2
    TYPE_EXPIRING_REDIRECT = 3

    TYPE_CHOICES = (
        (TYPE_PERMANENT_REDIRECT, _('Permanent redirect')),
        (TYPE_NO_REDIRECT, _('Leave no redirect')),
        (TYPE_EXPIRING_REDIRECT, _('Expiring redirect')),
    )

    from_topic = models.OneToOneField(get_model_string('Topic'),
                                      related_name='redirection')
    to_topic = models.ForeignKey(get_model_string('Topic'),
                                 related_name='redirections')
    created = models.DateTimeField(_('Created'), auto_now_add=True)
    type = models.PositiveSmallIntegerField(choices=TYPE_CHOICES,
                                            default=TYPE_PERMANENT_REDIRECT,
                                            db_index=True)
    expired = models.DateField(_('Expired'), null=True)

    class Meta(object):
        verbose_name = _('Redirection')
        verbose_name_plural = _('Redirections')
        app_label = 'pybb'
        abstract = True
        ordering = ['-created']

    def is_expired(self):
        return self.expired < date.today()

    def is_type_permanent(self):
        return self.type == self.TYPE_PERMANENT_REDIRECT

    def is_type_no(self):
        return self.type == self.TYPE_NO_REDIRECT

    def is_type_expiring(self):
        return self.type == self.TYPE_EXPIRING_REDIRECT


class TopicQuerySetMixin(object):
    def filter_by_user(self, user, forum=None):
        if forum is not None:
            if not forum.is_moderated_by(user):
                if user.is_authenticated():
                    return (self.filter(Q(user=user) | ~Q(on_moderation=BaseTopic.MODERATION_IS_IN_MODERATION))
                            .exclude(deleted=True))
            else:
                return self

            return self.exclude(on_moderation=BaseTopic.MODERATION_IS_IN_MODERATION).exclude(deleted=True)

        if user.is_staff or user.is_superuser:
            return self

        if user.is_authenticated():
            return (self.filter(Q(user=user) | ~Q(on_moderation=BaseTopic.MODERATION_IS_IN_MODERATION))
                    .filter(forum__staff=False)
                    .exclude(deleted=True))

        return (self.filter(forum__hidden=False, forum__staff=False)
                .exclude(on_moderation=BaseTopic.MODERATION_IS_IN_MODERATION)
                .exclude(deleted=True))

    def visible(self):
        return self.filter(deleted=False,
                           redirect=False).exclude(on_moderation=BaseTopic.MODERATION_IS_IN_MODERATION)


class TopicQuerySet(ParentForumQuerysetMixin, TopicQuerySetMixin, QuerySetBase):
    pass


@queryset
class TopicManager(ParentForumManagerMixin, ManagerBase):
    def get_queryset(self):
        return TopicQuerySet(self.model)

    def filter_by_user(self, user, forum=None):
        return self.get_queryset().filter_by_user(user, forum=forum)

    def visible(self):
        return self.get_queryset().visible()


class SubscriptionQuerySet(QuerySetBase):
    def visible(self):
        return self.filter(topic__deleted=False,<commit_msg>add join parameter to topic.filter_by_user<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import math
import os.path
import uuid
from collections import defaultdict

import magic
import requests

from datetime import date

from bs4 import BeautifulSoup
from django.utils.lru_cache import lru_cache

from six.moves.urllib.parse import urlparse, urlencode

from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import AnonymousUser
from django.utils.encoding import smart_text
from django.core.urlresolvers import reverse
from django.core.files.base import ContentFile
from django.utils.html import strip_tags
from django.utils.translation import ugettext_lazy as _
from django.db.models import Q, signals, F
from django.contrib.contenttypes.fields import GenericForeignKey
from django.db.models import ObjectDoesNotExist
from django.utils.functional import cached_property
from django.conf import settings

from pybb.compat import update_fields, AUTH_USER_MODEL, queryset
from pybb.util import unescape, get_model_string, tznow
from pybb.base import ModelBase, ManagerBase, QuerySetBase
from pybb.models.mixins import ParentForumQuerysetMixin, ParentForumManagerMixin, ParentForumBase
from pybb.subscription import notify_topic_subscribers
from pybb import defaults
from pybb.fields import ContentTypeRestrictedFileField, CAStorage
from pybb.tasks import generate_markup, sync_cover
from pybb.processors import markup

from autoslug import AutoSlugField

if 'sorl.thumbnail' in settings.INSTALLED_APPS:
    from sorl.thumbnail import ImageField
else:
    from django.db.models import ImageField


try:
    from south.modelsinspector import add_introspection_rules
    add_introspection_rules([], ["^annoying\.fields\.JSONField"])
    add_introspection_rules([], ["^annoying\.fields\.AutoOneToOneField"])
except ImportError:
    pass


class ModeratorManager(ManagerBase):
    def contribute_to_class(self, cls, name):
        signals.post_delete.connect(self.post_delete, sender=cls)
        return super(ModeratorManager, self).contribute_to_class(cls, name)

    def post_delete(self, instance, **kwargs):
        from pybb.proxies import UserObjectPermission

        UserObjectPermission.objects.get_for_object(instance.user, instance.forum).delete()

        (instance.user.user_permissions.through.objects.filter(permission__codename__in=defaults.PYBB_USER_PERMISSIONS,
                                                               user=instance.user)
         .delete())


@python_2_unicode_compatible
class BaseModerator(ModelBase):
    forum = models.ForeignKey(get_model_string('Forum'))
    user = models.ForeignKey(AUTH_USER_MODEL)

    objects = ModeratorManager()

    class Meta:
        app_label = 'pybb'
        verbose_name = _('Moderator')
        verbose_name_plural = _('Moderators')
        abstract = True

    def __str__(self):
        return _('Moderator %(user)s of %(forum)s') % {
            'user': self.user,
            'forum': self.forum
        }

    @property
    def permissions(self):
        from pybb.proxies import UserObjectPermission

        permissions = list(self.user.user_permissions.all())
        permissions += [obj_perm.permission
                        for obj_perm in (UserObjectPermission.objects.get_for_object(user=self.user,
                                                                                     obj=self.forum)
                                         .select_related('permission'))]

        return permissions


class ForumQuerySet(ParentForumQuerysetMixin, QuerySetBase):
    def filter_by_user(self, user, hidden=True):
        if user.is_staff or user.is_superuser:
            return self

        if user.is_authenticated():
            return self.filter(staff=False)

        if hidden:
            return self.filter(hidden=False, staff=False)

        return self.filter(staff=False)


@queryset
class ForumManager(ParentForumManagerMixin, ManagerBase):
    def contribute_to_class(self, cls, name):
        signals.post_save.connect(self.post_save, sender=cls)
        signals.post_delete.connect(self.post_delete, sender=cls)
        return super(ForumManager, self).contribute_to_class(cls, name)

    def post_save(self, instance, **kwargs):
        if kwargs.get('created', False) and instance.forum_id:
            instance.forum.compute()

    def post_delete(self, instance, **kwargs):
        if instance.forum_id:
            instance.forum.compute()

    def get_queryset(self):
        return ForumQuerySet(self.model)

    def filter_by_user(self, *args, **kwargs):
        return self.get_queryset().filter_by_user(*args, **kwargs)


@lru_cache()
def get_moderator_ids_by_forum():
    from pybb.models import Moderator

    result = defaultdict(list)
    for moderator in Moderator.objects.all():
        result[moderator.forum_id].append(moderator.user_id)

    return result


@python_2_unicode_compatible
class BaseForum(ParentForumBase):
    forum = models.ForeignKey('Forum', related_name='forums',
                              verbose_name=_('Parent'), null=True, blank=True)
    name = models.CharField(_('Name'), max_length=80)
    slug = AutoSlugField(populate_from='name', max_length=80)
    position = models.IntegerField(_('Position'), blank=True, default=0, db_index=True)
    description = models.TextField(_('Description'), blank=True)
    moderators = models.ManyToManyField(AUTH_USER_MODEL, blank=True, verbose_name=_('Moderators'), through=get_model_string('Moderator'))
    updated = models.DateTimeField(_('Updated'), blank=True, null=True)
    post_count = models.IntegerField(_('Post count'), blank=True, default=0, db_index=True)
    topic_count = models.IntegerField(_('Topic count'), blank=True, default=0, db_index=True)
    forum_count = models.PositiveIntegerField(default=0, db_index=True)
    readed_by = models.ManyToManyField(AUTH_USER_MODEL, through=get_model_string('ForumReadTracker'), related_name='readed_forums')
    headline = models.TextField(_('Headline'), blank=True, null=True)
    hidden = models.BooleanField(_('Hidden'), blank=False, null=False,
                                 default=False, db_index=True)
    staff = models.BooleanField(_('Staff only'), blank=False,
                                null=False, default=False,
                                db_index=True)
    empty = models.BooleanField(_('Empty'),
                                default=False,
                                db_index=True,
                                help_text=_('If the forum is empty, you can\'t post into it'))

    last_topic = models.ForeignKey(get_model_string('Topic'),
                                   null=True,
                                   blank=True,
                                   related_name='last_forums',
                                   on_delete=models.SET_NULL)

    last_post = models.ForeignKey(get_model_string('Post'),
                                  null=True,
                                  blank=True,
                                  on_delete=models.SET_NULL)

    objects = ForumManager()

    class Meta(object):
        verbose_name = _('Forum')
        verbose_name_plural = _('Forums')
        ordering = ['position']
        permissions = (
            ('can_unstick_topic', _('Can unstick topic')),
            ('can_stick_topic', _('Can stick topic')),
            ('can_open_topic', _('Can open topic')),
            ('can_close_topic', _('Can close topic')),
            ('can_merge_topic', _('Can merge topic')),
            ('can_delete_topic', _('Can delete topic')),
            ('can_change_topic', _('Can edit topic')),
            ('can_move_topic', _('Can move topic')),
            ('can_move_post', _('Can move post')),
            ('can_change_poll', _('Can edit a poll')),
            ('can_publish_announce', _('Can publish an announce')),
            ('can_change_post', _('Can edit post')),
            ('can_delete_post', _('Can delete post')),
            ('can_see_user_ip', _('Can see IP Address')),
            ('can_change_attachment', _('Can moderate attachments')),
        )
        app_label = 'pybb'
        abstract = True

    def compute(self, commit=True):
        forum_count = self.forums.count()

        from pybb.models import Topic

        res = Topic.objects.visible().filter(forum_id=self.pk).aggregate(post_count=models.Sum('post_count'))

        post_count = res['post_count'] or 0

        topic_count = Topic.objects.filter(forum_id=self.pk).visible().count() or 0

        res = self.__class__.objects.filter(forum_id=self.pk).aggregate(post_count=models.Sum('post_count'),
                                                                        topic_count=models.Sum('topic_count'),
                                                                        forum_count=models.Sum('forum_count'))

        self.post_count = post_count + (res['post_count'] or 0)

        self.topic_count = topic_count + (res['topic_count'] or 0)

        self.forum_count = forum_count + (res['forum_count'] or 0)

        if commit:
            self.save()

        if self.forum_id and self.forum_id != self.pk:
            self.forum.compute(commit=commit)

    def is_moderated_by(self, user, permission=None):
        if user.is_superuser or user.is_staff:
            return True

        if (user.is_authenticated() and
                user.pk in get_moderator_ids_by_forum()[self.pk]):

            if permission:
                return user.has_perm(permission, self)

            return True

        return False

    def is_hidden(self):
        return self.hidden or (self.forum_id and self.forum.is_hidden())

    def is_accessible_by(self, user, hidden=True):
        if self.forum_id and not self.forum.is_accessible_by(user, hidden=hidden):
            return False

        if self.staff and not user.is_staff:
            return False

        if hidden and self.is_hidden() and not user.is_authenticated():
            return False

        return True

    def __str__(self):
        return self.name

    def update_counters(self, commit=True):
        last_post = self.get_last_post()

        if last_post:
            self.updated = last_post.created

            self.last_post = last_post
            self.last_topic = last_post.topic

        self.compute(commit=commit)

        if commit:
            self.save()

    def get_absolute_url(self):
        return reverse('pybb:forum_detail', kwargs={'slug': self.slug})

    @property
    def posts(self):
        from pybb.models import Post

        return Post.objects.filter(topic__forum=self).visible().select_related()

    def get_last_post(self):
        from pybb.models import Post

        post = None

        try:
            topic = self.topics.visible().order_by('-updated')[0]
            post = topic.last_post

            if post:
                post.topic = topic
        except (IndexError, Post.DoesNotExist):
            pass

        if post and post.topic.poll_id:
            try:
                other = self.topics.visible().filter(poll__isnull=True).order_by('-updated')[0].last_post
            except (IndexError, Post.DoesNotExist):
                pass
            else:
                if post:
                    if other.created > post.created:
                        post = other
                else:
                    post = other

        return post

    def get_last_topic(self):
        topic = None

        try:
            topic = self.topics.order_by('-updated').visible().select_related()[0]
        except IndexError:
            return None

        try:
            forum_topic = self.forums.order_by('-updated').filter(last_topic__isnull=False)[0].last_topic

            if topic:
                if forum_topic.updated > topic.updated:
                    topic = forum_topic
            else:
                topic = forum_topic
        except IndexError:
            pass

        return topic

    @classmethod
    def watch_forum(cls, old_attr={}, new_attr={}, instance=None, sender=None, **kw):
        from pybb.models import Forum

        if (old_attr.get('forum_id', None) and
                old_attr.get('forum_id') != new_attr.get('forum_id')):
            instance.save(_signal=False)

            try:
                Forum.objects.get(pk=old_attr.get('forum_id')).update_counters()
            except ObjectDoesNotExist:
                pass


class BaseTopicRedirection(ModelBase):
    TYPE_PERMANENT_REDIRECT = 1
    TYPE_NO_REDIRECT = 2
    TYPE_EXPIRING_REDIRECT = 3

    TYPE_CHOICES = (
        (TYPE_PERMANENT_REDIRECT, _('Permanent redirect')),
        (TYPE_NO_REDIRECT, _('Leave no redirect')),
        (TYPE_EXPIRING_REDIRECT, _('Expiring redirect')),
    )

    from_topic = models.OneToOneField(get_model_string('Topic'),
                                      related_name='redirection')
    to_topic = models.ForeignKey(get_model_string('Topic'),
                                 related_name='redirections')
    created = models.DateTimeField(_('Created'), auto_now_add=True)
    type = models.PositiveSmallIntegerField(choices=TYPE_CHOICES,
                                            default=TYPE_PERMANENT_REDIRECT,
                                            db_index=True)
    expired = models.DateField(_('Expired'), null=True)

    class Meta(object):
        verbose_name = _('Redirection')
        verbose_name_plural = _('Redirections')
        app_label = 'pybb'
        abstract = True
        ordering = ['-created']

    def is_expired(self):
        return self.expired < date.today()

    def is_type_permanent(self):
        return self.type == self.TYPE_PERMANENT_REDIRECT

    def is_type_no(self):
        return self.type == self.TYPE_NO_REDIRECT

    def is_type_expiring(self):
        return self.type == self.TYPE_EXPIRING_REDIRECT


class TopicQuerySetMixin(object):
    def filter_by_user(self, user, forum=None, join=True):
        if forum is not None:
            if not forum.is_moderated_by(user):
                if user.is_authenticated():
                    return (self.filter(Q(user=user) | ~Q(on_moderation=BaseTopic.MODERATION_IS_IN_MODERATION))
                            .exclude(deleted=True))
            else:
                return self

            return self.exclude(on_moderation=BaseTopic.MODERATION_IS_IN_MODERATION).exclude(deleted=True)

        if user.is_staff or user.is_superuser:
            return self

        qs = self
        if user.is_authenticated():
            if join:
                qs = qs.filter(forum__staff=False)
            return (qs.filter(Q(user=user) | ~Q(on_moderation=BaseTopic.MODERATION_IS_IN_MODERATION))
                    .exclude(deleted=True))

        if join:
            qs = qs.filter(forum__hidden=False, forum__staff=False)
        return (qs.exclude(on_moderation=BaseTopic.MODERATION_IS_IN_MODERATION)
                .exclude(deleted=True))

    def visible(self):
        return self.filter(deleted=False,
                           redirect=False).exclude(on_moderation=BaseTopic.MODERATION_IS_IN_MODERATION)


class TopicQuerySet(ParentForumQuerysetMixin, TopicQuerySetMixin, QuerySetBase):
    pass


@queryset
class TopicManager(ParentForumManagerMixin, ManagerBase):
    def get_queryset(self):
        return TopicQuerySet(self.model)

    def filter_by_user(self, user, forum=None):
        return self.get_queryset().filter_by_user(user, forum=forum)

    def visible(self):
        return self.get_queryset().visible()


class SubscriptionQuerySet(QuerySetBase):
    def visible(self):
        return self.filter(topic__deleted=False,
{'fraction_non_alphanumeric': 0.09750523709769568, 'fraction_numerical': 0.0015869993017203073, 'mean_word_length': 2.429146713104049, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 54, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5897383', 'n_tokens_mistral': 9184, 'n_tokens_neox': 8787, 'n_words': 1792}