<commit_before># -*- coding: utf-8 -*-
import os
import re
import uuid
import urllib
import logging
import datetime
import urlparse
from collections import OrderedDict

import pytz
import blinker
from flask import request
from HTMLParser import HTMLParser

from modularodm import Q
from modularodm import fields
from modularodm.validators import MaxLengthValidator
from modularodm.exceptions import ValidationTypeError
from modularodm.exceptions import ValidationValueError

from framework import status
from framework.mongo import ObjectId
from framework.mongo import StoredObject
from framework.addons import AddonModelMixin
from framework.auth import get_user, User, Auth
from framework.auth import signals as auth_signals
from framework.exceptions import PermissionsError
from framework.guid.model import GuidStoredObject
from framework.auth.utils import privacy_info_handle
from framework.analytics import tasks as piwik_tasks
from framework.mongo.utils import to_mongo, to_mongo_key
from framework.analytics import (
    get_basic_counters, increment_user_activity_counters
)
from framework.sentry import log_exception
from framework.transactions.context import TokuTransaction

from website import language
from website import settings
from website.util import web_url_for
from website.util import api_url_for
from website.exceptions import NodeStateError
from website.citations.utils import datetime_to_csl
from website.util.permissions import expand_permissions
from website.util.permissions import CREATOR_PERMISSIONS
from website.project.metadata.schemas import OSF_META_SCHEMAS
from website.util.permissions import DEFAULT_CONTRIBUTOR_PERMISSIONS

html_parser = HTMLParser()

logger = logging.getLogger(__name__)


def has_anonymous_link(node, auth):
    """check if the node is anonymous to the user

    :param Node node: Node which the user wants to visit
    :param str link: any view-only link in the current url
    :return bool anonymous: Whether the node is anonymous to the user or not
    """
    view_only_link = auth.private_key or request.args.get('view_only', '').strip('/')
    if not view_only_link:
        return False
    if node.is_public:
        return False
    return any(
        link.anonymous
        for link in node.private_links_active
        if link.key == view_only_link
    )


signals = blinker.Namespace()
contributor_added = signals.signal('contributor-added')
unreg_contributor_added = signals.signal('unreg-contributor-added')
write_permissions_revoked = signals.signal('write-permissions-revoked')


class MetaSchema(StoredObject):

    _id = fields.StringField(default=lambda: str(ObjectId()))
    name = fields.StringField()
    schema = fields.DictionaryField()
    category = fields.StringField()

    # Version of the Knockout metadata renderer to use (e.g. if data binds
    # change)
    metadata_version = fields.IntegerField()
    # Version of the schema to use (e.g. if questions, responses change)
    schema_version = fields.IntegerField()


def ensure_schemas(clear=True):
    """Import meta-data schemas from JSON to database, optionally clearing
    database first.

    :param clear: Clear schema database before import
    """
    if clear:
        try:
            MetaSchema.remove()
        except AttributeError:
            if not settings.DEBUG_MODE:
                raise
    for schema in OSF_META_SCHEMAS:
        try:
            MetaSchema.find_one(
                Q('name', 'eq', schema['name']) &
                Q('schema_version', 'eq', schema['schema_version'])
            )
        except:
            schema['name'] = schema['name'].replace(' ', '_')
            schema_obj = MetaSchema(**schema)
            schema_obj.save()


class MetaData(GuidStoredObject):

    _id = fields.StringField(primary=True)

    target = fields.AbstractForeignField(backref='metadata')
    data = fields.DictionaryField()

    date_created = fields.DateTimeField(auto_now_add=datetime.datetime.utcnow)
    date_modified = fields.DateTimeField(auto_now=datetime.datetime.utcnow)


def validate_comment_reports(value, *args, **kwargs):
    for key, val in value.iteritems():
        if not User.load(key):
            raise ValidationValueError('Keys must be user IDs')
        if not isinstance(val, dict):
            raise ValidationTypeError('Values must be dictionaries')
        if 'category' not in val or 'text' not in val:
            raise ValidationValueError(
                'Values must include `category` and `text` keys'
            )


class Comment(GuidStoredObject):

    OVERVIEW = "node"
    FILES = "files"
    WIKI = "wiki"

    _id = fields.StringField(primary=True)

    user = fields.ForeignField('user', required=True, backref='commented')
    node = fields.ForeignField('node', required=True, backref='comment_owner')
    target = fields.AbstractForeignField(required=True, backref='commented')
    # The file/wiki/overview's page that the comment is for
    root_target = fields.AbstractForeignField(backref='comment_target')

    date_created = fields.DateTimeField(auto_now_add=datetime.datetime.utcnow)
    date_modified = fields.DateTimeField(auto_now=datetime.datetime.utcnow)
    modified = fields.BooleanField()

    is_deleted = fields.BooleanField(default=False)
    # Whether the original file/wiki is deleted
    is_hidden = fields.BooleanField(default=False)
    # The type root_target: node/files/wiki
    page = fields.StringField()
    content = fields.StringField()

    # Dictionary field mapping user IDs to dictionaries of report details:
    # {
    #   'icpnw': {'category': 'hate', 'message': 'offensive'},
    #   'cdi38': {'category': 'spam', 'message': 'godwins law'},
    # }
    reports = fields.DictionaryField(validate=validate_comment_reports)

    @property
    def url(self):
        return '/{}/'.format(self._id)

    @property
    def deep_url(self):
        return '/{0}discussions/{1}/'.format(self.node.deep_url, self._id)

    @classmethod
    def create(cls, auth, **kwargs):

        comment = cls(**kwargs)
        if isinstance(comment.target, Comment):
            comment.root_target = comment.target.root_target
        else:
            comment.root_target = comment.target
        comment.save()<commit_msg>Add a docstring and will not affect travis [skip ci]<commit_after># -*- coding: utf-8 -*-
import os
import re
import uuid
import urllib
import logging
import datetime
import urlparse
from collections import OrderedDict

import pytz
import blinker
from flask import request
from HTMLParser import HTMLParser

from modularodm import Q
from modularodm import fields
from modularodm.validators import MaxLengthValidator
from modularodm.exceptions import ValidationTypeError
from modularodm.exceptions import ValidationValueError

from framework import status
from framework.mongo import ObjectId
from framework.mongo import StoredObject
from framework.addons import AddonModelMixin
from framework.auth import get_user, User, Auth
from framework.auth import signals as auth_signals
from framework.exceptions import PermissionsError
from framework.guid.model import GuidStoredObject
from framework.auth.utils import privacy_info_handle
from framework.analytics import tasks as piwik_tasks
from framework.mongo.utils import to_mongo, to_mongo_key
from framework.analytics import (
    get_basic_counters, increment_user_activity_counters
)
from framework.sentry import log_exception
from framework.transactions.context import TokuTransaction

from website import language
from website import settings
from website.util import web_url_for
from website.util import api_url_for
from website.exceptions import NodeStateError
from website.citations.utils import datetime_to_csl
from website.util.permissions import expand_permissions
from website.util.permissions import CREATOR_PERMISSIONS
from website.project.metadata.schemas import OSF_META_SCHEMAS
from website.util.permissions import DEFAULT_CONTRIBUTOR_PERMISSIONS

html_parser = HTMLParser()

logger = logging.getLogger(__name__)


def has_anonymous_link(node, auth):
    """check if the node is anonymous to the user

    :param Node node: Node which the user wants to visit
    :param str link: any view-only link in the current url
    :return bool anonymous: Whether the node is anonymous to the user or not
    """
    view_only_link = auth.private_key or request.args.get('view_only', '').strip('/')
    if not view_only_link:
        return False
    if node.is_public:
        return False
    return any(
        link.anonymous
        for link in node.private_links_active
        if link.key == view_only_link
    )


signals = blinker.Namespace()
contributor_added = signals.signal('contributor-added')
unreg_contributor_added = signals.signal('unreg-contributor-added')
write_permissions_revoked = signals.signal('write-permissions-revoked')


class MetaSchema(StoredObject):

    _id = fields.StringField(default=lambda: str(ObjectId()))
    name = fields.StringField()
    schema = fields.DictionaryField()
    category = fields.StringField()

    # Version of the Knockout metadata renderer to use (e.g. if data binds
    # change)
    metadata_version = fields.IntegerField()
    # Version of the schema to use (e.g. if questions, responses change)
    schema_version = fields.IntegerField()


def ensure_schemas(clear=True):
    """Import meta-data schemas from JSON to database, optionally clearing
    database first.

    :param clear: Clear schema database before import
    """
    if clear:
        try:
            MetaSchema.remove()
        except AttributeError:
            if not settings.DEBUG_MODE:
                raise
    for schema in OSF_META_SCHEMAS:
        try:
            MetaSchema.find_one(
                Q('name', 'eq', schema['name']) &
                Q('schema_version', 'eq', schema['schema_version'])
            )
        except:
            schema['name'] = schema['name'].replace(' ', '_')
            schema_obj = MetaSchema(**schema)
            schema_obj.save()


class MetaData(GuidStoredObject):

    _id = fields.StringField(primary=True)

    target = fields.AbstractForeignField(backref='metadata')
    data = fields.DictionaryField()

    date_created = fields.DateTimeField(auto_now_add=datetime.datetime.utcnow)
    date_modified = fields.DateTimeField(auto_now=datetime.datetime.utcnow)


def validate_comment_reports(value, *args, **kwargs):
    for key, val in value.iteritems():
        if not User.load(key):
            raise ValidationValueError('Keys must be user IDs')
        if not isinstance(val, dict):
            raise ValidationTypeError('Values must be dictionaries')
        if 'category' not in val or 'text' not in val:
            raise ValidationValueError(
                'Values must include `category` and `text` keys'
            )


class Comment(GuidStoredObject):

    OVERVIEW = "node"
    FILES = "files"
    WIKI = "wiki"

    _id = fields.StringField(primary=True)

    user = fields.ForeignField('user', required=True, backref='commented')
    node = fields.ForeignField('node', required=True, backref='comment_owner')
    target = fields.AbstractForeignField(required=True, backref='commented')
    # The file/wiki/overview's page that the comment is for
    root_target = fields.AbstractForeignField(backref='comment_target')

    date_created = fields.DateTimeField(auto_now_add=datetime.datetime.utcnow)
    date_modified = fields.DateTimeField(auto_now=datetime.datetime.utcnow)
    modified = fields.BooleanField()

    is_deleted = fields.BooleanField(default=False)
    # Whether the original file/wiki is deleted
    # NOTE Currently, is_hidden will set to True only when a wiki is deleted.
    is_hidden = fields.BooleanField(default=False)
    # The type root_target: node/files/wiki
    page = fields.StringField()
    content = fields.StringField()

    # Dictionary field mapping user IDs to dictionaries of report details:
    # {
    #   'icpnw': {'category': 'hate', 'message': 'offensive'},
    #   'cdi38': {'category': 'spam', 'message': 'godwins law'},
    # }
    reports = fields.DictionaryField(validate=validate_comment_reports)

    @property
    def url(self):
        return '/{}/'.format(self._id)

    @property
    def deep_url(self):
        return '/{0}discussions/{1}/'.format(self.node.deep_url, self._id)

    @classmethod
    def create(cls, auth, **kwargs):

        comment = cls(**kwargs)
        if isinstance(comment.target, Comment):
            comment.root_target = comment.target.root_target
        else:
            comment.root_target = comment.target
        comment.save()
{'fraction_non_alphanumeric': 0.08625742574257425, 'fraction_numerical': 0.0007920792079207921, 'mean_word_length': 3.6884515410323058, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2066972', 'n_tokens_mistral': 3572, 'n_tokens_neox': 3440, 'n_words': 997}