<commit_before>import re
import os
import logging
from collections import defaultdict
from time import sleep
from datetime import datetime
import cPickle as pickle

import pymongo
from pylons import c, g
from ming import Document, Session, Field
from ming import schema as S
from ming.orm.base import mapper, session, state
from ming.orm.mapped_class import MappedClass, MappedClassMeta
from ming.orm.property import FieldProperty, ForeignIdProperty, RelationProperty
from pymongo.errors import OperationFailure
from webhelpers import feedgenerator as FG

from pyforge.lib.helpers import nonce, push_config, push_context, absurl
from .session import ProjectSession
from .session import main_doc_session, main_orm_session
from .session import project_doc_session, project_orm_session
from .session import artifact_orm_session
from .types import ArtifactReference, ArtifactReferenceType

log = logging.getLogger(__name__)

def gen_message_id():
    parts = c.project.url().split('/')[1:-1]
    return '%s.<EMAIL>' % (nonce(40),
                                         c.app.config.options['mount_point'],
                                         '.'.join(reversed(parts)))

class ArtifactLink(MappedClass):
    class __mongometa__:
        session = project_orm_session
        name='artifact_link'
        indexes = [
            ('link', 'project_id') ]

    core_re = r'''(\[
            (?:(?P<project_id>.*?):)?      # optional project ID
            (?:(?P<app_id>.*?):)?      # optional plugin ID
            (?P<artifact_id>.*)             # artifact ID
    \])'''

    re_link_1 = re.compile(r'\s' + core_re, re.VERBOSE)
    re_link_2 = re.compile(r'^' +  core_re, re.VERBOSE)

    _id = FieldProperty(str)
    link = FieldProperty(str)
    project_id = ForeignIdProperty('Project')
    plugin_name = FieldProperty(str)
    mount_point = FieldProperty(str)
    url = FieldProperty(str)
    artifact_reference = FieldProperty(ArtifactReferenceType)

    @classmethod
    def add(cls, artifact):
        aid = artifact.index_id()
        entry = cls.query.get(_id=aid)
        if entry is None:
            entry = cls(_id=aid)
        entry.link=artifact.shorthand_id()
        entry.project_id=artifact.project_id
        entry.plugin_name=artifact.app_config.plugin_name
        entry.mount_point=artifact.app_config.options.mount_point
        entry.url=artifact.url()
        entry.artifact_reference = artifact.dump_ref()

    @classmethod
    def remove(cls, artifact):
        mapper(cls).remove(dict(_id=artifact.index_id()))

    @classmethod
    def lookup(cls, link):
        from .project import Project
        #
        # Parse the link syntax
        #
        m = cls.re_link_1.match(link)
        if m is None: m = cls.re_link_2.match(link)
        if m is None: return None
        groups = m.groupdict()
        project_id = groups.get('project_id', None)
        app_id = groups.get('app_id', None)
        artifact_id = groups.get('artifact_id', None)
        if app_id is None:
            app_id = project_id
            project_id = None

        #
        # Find the projects to search
        #
        if project_id is None:
            if c.project:
                projects = list(c.project.parent_iter())
            else:
                return None
        elif project_id.startswith('/'):
            projects = Project.query.find(dict(shortname=project_id[1:])).all()
        else:
            project_id = os.path.normpath(
                os.path.join('/' + c.project.shortname, project_id))
            projects = Project.query.find(dict(shortname=project_id[1:])).all()
        if not projects: return None
        #
        # Actually search the projects
        #
        with push_config(c, project=projects[0]):
            for p in projects:
                links = cls.query.find(dict(project_id=p._id, link=artifact_id)).all()<commit_msg>FORGE-339 - fix another markdown bug<commit_after>import re
import os
import logging
from collections import defaultdict
from time import sleep
from datetime import datetime
import cPickle as pickle

import pymongo
from pylons import c, g
from ming import Document, Session, Field
from ming import schema as S
from ming.orm.base import mapper, session, state
from ming.orm.mapped_class import MappedClass, MappedClassMeta
from ming.orm.property import FieldProperty, ForeignIdProperty, RelationProperty
from pymongo.errors import OperationFailure
from webhelpers import feedgenerator as FG

from pyforge.lib.helpers import nonce, push_config, push_context, absurl
from .session import ProjectSession
from .session import main_doc_session, main_orm_session
from .session import project_doc_session, project_orm_session
from .session import artifact_orm_session
from .types import ArtifactReference, ArtifactReferenceType

log = logging.getLogger(__name__)

def gen_message_id():
    parts = c.project.url().split('/')[1:-1]
    return '%s.%s<EMAIL>' % (nonce(40),
                                         c.app.config.options['mount_point'],
                                         '.'.join(reversed(parts)))

class ArtifactLink(MappedClass):
    class __mongometa__:
        session = project_orm_session
        name='artifact_link'
        indexes = [
            ('link', 'project_id') ]

    core_re = r'''(\[
            (?:(?P<project_id>.*?):)?      # optional project ID
            (?:(?P<app_id>.*?):)?      # optional plugin ID
            (?P<artifact_id>.*)             # artifact ID
    \])'''

    re_link_1 = re.compile(r'\s' + core_re, re.VERBOSE)
    re_link_2 = re.compile(r'^' +  core_re, re.VERBOSE)

    _id = FieldProperty(str)
    link = FieldProperty(str)
    project_id = ForeignIdProperty('Project')
    plugin_name = FieldProperty(str)
    mount_point = FieldProperty(str)
    url = FieldProperty(str)
    artifact_reference = FieldProperty(ArtifactReferenceType)

    @classmethod
    def add(cls, artifact):
        aid = artifact.index_id()
        entry = cls.query.get(_id=aid)
        if entry is None:
            entry = cls(_id=aid)
        entry.link=artifact.shorthand_id()
        entry.project_id=artifact.project_id
        entry.plugin_name=artifact.app_config.plugin_name
        entry.mount_point=artifact.app_config.options.mount_point
        entry.url=artifact.url()
        entry.artifact_reference = artifact.dump_ref()

    @classmethod
    def remove(cls, artifact):
        mapper(cls).remove(dict(_id=artifact.index_id()))

    @classmethod
    def lookup(cls, link):
        from .project import Project
        #
        # Parse the link syntax
        #
        m = cls.re_link_1.match(link)
        if m is None: m = cls.re_link_2.match(link)
        if m is None: return None
        groups = m.groupdict()
        project_id = groups.get('project_id', None)
        app_id = groups.get('app_id', None)
        artifact_id = groups.get('artifact_id', None)
        if app_id is None:
            app_id = project_id
            project_id = None

        #
        # Find the projects to search
        #
        if project_id is None:
            if c.project:
                projects = list(c.project.parent_iter())
            else:
                return None
        elif project_id.startswith('/'):
            projects = Project.query.find(dict(shortname=project_id[1:])).all()
        else:
            if c.project:
                project_id = os.path.normpath(
                    os.path.join('/' + c.project.shortname, project_id))
            else:
                project_id = '/' + project_id
            projects = Project.query.find(dict(shortname=project_id[1:])).all()
        if not projects: return None
        #
        # Actually search the projects
        #
        with push_config(c, project=projects[0]):
            for p in projects:
                links = cls.query.find(dict(project_id=p._id, link=artifact_id)).all()
{'fraction_non_alphanumeric': 0.10186005314437556, 'fraction_numerical': 0.0031633556877135264, 'mean_word_length': 2.4485165794066317, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6939414', 'n_tokens_mistral': 2364, 'n_tokens_neox': 2278, 'n_words': 552}