<commit_before>from time import sleep
from datetime import datetime

import tg
from pylons import g #g is a namespace for globally accessable app helpers
from pylons import c as context
from pylons import request

from pymongo.errors import OperationFailure

from ming import schema
from ming.orm.base import state, session
from ming.orm.mapped_class import MappedClass
from ming.orm.property import FieldProperty, ForeignIdProperty, RelationProperty

from allura.model import VersionedArtifact, Snapshot, Message, Feed, Thread, Post, User, BaseAttachment
from allura.model import Notification, project_orm_session
from allura.lib import helpers as h
from allura.lib import patience

common_suffix = tg.config.get('forgemail.domain', '.sourceforge.net')

class Globals(MappedClass):

    class __mongometa__:
        name = 'wiki-globals'
        session = project_orm_session
        indexes = [ 'app_config_id' ]

    type_s = 'WikiGlobals'
    _id = FieldProperty(schema.ObjectId)
    app_config_id = ForeignIdProperty('AppConfig', if_missing=lambda:context.app.config._id)
    root = FieldProperty(str)


class PageHistory(Snapshot):
    class __mongometa__:
        name='page_history'

    def original(self):
        return Page.query.get(_id=self.artifact_id)

    def authors(self):
        return self.original().authors()
        
    def shorthand_id(self):
        return '%s#%s' % (self.original().shorthand_id(), self.version)

    def url(self):
        return self.original().url() + '?version=%d' % self.version

    def index(self):
        result = Snapshot.index(self)
        result.update(
            title_s='Version %d of %s' % (
                self.version,self.original().title),
            type_s='WikiPage Snapshot',
            text=self.data.text)
        return result

    @property
    def html_text(self):
        """A markdown processed version of the page text"""
        return g.markdown_wiki.convert(self.data.text)

    @property
    def attachments(self):
        return self.original().attachments

    @property
    def email_address(self):
        return self.original().email_address

class Page(VersionedArtifact):
    class __mongometa__:
        name='page'
        history_class = PageHistory

    title=FieldProperty(str)
    text=FieldProperty(schema.String, if_missing='')
    viewable_by=FieldProperty([str])
    deleted=FieldProperty(bool, if_missing=False)
    type_s = 'Wiki'

    def commit(self):
        self.subscribe()
        VersionedArtifact.commit(self)
        if self.version > 1:
            v1 = self.get_version(self.version-1)
            v2 = self
            la = [ line + '\n'  for line in v1.text.splitlines() ]
            lb = [ line + '\n'  for line in v2.text.splitlines() ]
            diff = ''.join(patience.unified_diff(
                    la, lb,
                    'v%d' % v1.version,
                    'v%d' % v2.version))
            description = '<pre>' + diff + '</pre>'
            if v1.title != v2.title:
                subject = '%s renamed page %s to %s' % (
                    context.user.username, v2.title, v1.title)
            else:
                subject = '%s modified page %s' % (
                    context.user.username, self.title)
        else:
            description = self.text
            subject = '%s created page %s' % (
                context.user.username, self.title)
        Feed.post(self, description)
        Notification.post(
            artifact=self, topic='metadata', text=description, subject=subject)

    @property
    def email_address(self):
        domain = '.'.join(reversed(self.app.url[1:-1].split('/'))).replace('_', '-')
        return '%s@%s%s' % (self.title.replace('/', '.'), domain, common_suffix)

    @property
    def email_subject(self):
        return 'Discussion for %s page' % self.title

    def url(self):
        s = self.app_config.url() + h.urlquote(self.title) + '/'
        if self.deleted:
            s += '?deleted=True'
        return s

    def shorthand_id(self):
        return self.title

    def index(self):
        result = VersionedArtifact.index(self)
        result.update(
            title_s='WikiPage %s' % self.title,
            version_i=self.version,
            type_s='WikiPage',
            text=self.text)
        return result<commit_msg>[#1343] Add unicode encoding to wiki page urls<commit_after>from time import sleep
from datetime import datetime

import tg
from pylons import g #g is a namespace for globally accessable app helpers
from pylons import c as context
from pylons import request

from pymongo.errors import OperationFailure

from ming import schema
from ming.orm.base import state, session
from ming.orm.mapped_class import MappedClass
from ming.orm.property import FieldProperty, ForeignIdProperty, RelationProperty

from allura.model import VersionedArtifact, Snapshot, Message, Feed, Thread, Post, User, BaseAttachment
from allura.model import Notification, project_orm_session
from allura.lib import helpers as h
from allura.lib import patience

common_suffix = tg.config.get('forgemail.domain', '.sourceforge.net')

class Globals(MappedClass):

    class __mongometa__:
        name = 'wiki-globals'
        session = project_orm_session
        indexes = [ 'app_config_id' ]

    type_s = 'WikiGlobals'
    _id = FieldProperty(schema.ObjectId)
    app_config_id = ForeignIdProperty('AppConfig', if_missing=lambda:context.app.config._id)
    root = FieldProperty(str)


class PageHistory(Snapshot):
    class __mongometa__:
        name='page_history'

    def original(self):
        return Page.query.get(_id=self.artifact_id)

    def authors(self):
        return self.original().authors()
        
    def shorthand_id(self):
        return '%s#%s' % (self.original().shorthand_id(), self.version)

    def url(self):
        return self.original().url() + '?version=%d' % self.version

    def index(self):
        result = Snapshot.index(self)
        result.update(
            title_s='Version %d of %s' % (
                self.version,self.original().title),
            type_s='WikiPage Snapshot',
            text=self.data.text)
        return result

    @property
    def html_text(self):
        """A markdown processed version of the page text"""
        return g.markdown_wiki.convert(self.data.text)

    @property
    def attachments(self):
        return self.original().attachments

    @property
    def email_address(self):
        return self.original().email_address

class Page(VersionedArtifact):
    class __mongometa__:
        name='page'
        history_class = PageHistory

    title=FieldProperty(str)
    text=FieldProperty(schema.String, if_missing='')
    viewable_by=FieldProperty([str])
    deleted=FieldProperty(bool, if_missing=False)
    type_s = 'Wiki'

    def commit(self):
        self.subscribe()
        VersionedArtifact.commit(self)
        if self.version > 1:
            v1 = self.get_version(self.version-1)
            v2 = self
            la = [ line + '\n'  for line in v1.text.splitlines() ]
            lb = [ line + '\n'  for line in v2.text.splitlines() ]
            diff = ''.join(patience.unified_diff(
                    la, lb,
                    'v%d' % v1.version,
                    'v%d' % v2.version))
            description = '<pre>' + diff + '</pre>'
            if v1.title != v2.title:
                subject = '%s renamed page %s to %s' % (
                    context.user.username, v2.title, v1.title)
            else:
                subject = '%s modified page %s' % (
                    context.user.username, self.title)
        else:
            description = self.text
            subject = '%s created page %s' % (
                context.user.username, self.title)
        Feed.post(self, description)
        Notification.post(
            artifact=self, topic='metadata', text=description, subject=subject)

    @property
    def email_address(self):
        domain = '.'.join(reversed(self.app.url[1:-1].split('/'))).replace('_', '-')
        return '%s@%s%s' % (self.title.replace('/', '.'), domain, common_suffix)

    @property
    def email_subject(self):
        return 'Discussion for %s page' % self.title

    def url(self):
        s = self.app_config.url() + h.urlquote(self.title.encode('utf-8')) + '/'
        if self.deleted:
            s += '?deleted=True'
        return s

    def shorthand_id(self):
        return self.title

    def index(self):
        result = VersionedArtifact.index(self)
        result.update(
            title_s='WikiPage %s' % self.title,
            version_i=self.version,
            type_s='WikiPage',
            text=self.text)
        return result
{'fraction_non_alphanumeric': 0.11348250143430866, 'fraction_numerical': 0.0037865748709122204, 'mean_word_length': 2.3835403726708075, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4647826', 'n_tokens_mistral': 2641, 'n_tokens_neox': 2541, 'n_words': 614}