<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from urlparse import urlparse

from flask import current_app

from udata.forms import ModelForm, fields, validators
from udata.i18n import lazy_gettext as _

from udata.core.storages import resources
from udata.core.spatial.forms import SpatialCoverageField

from .models import (
    Dataset, Resource, License, Checksum, CommunityResource,
    UPDATE_FREQUENCIES, DEFAULT_FREQUENCY, RESOURCE_FILETYPES, CHECKSUM_TYPES,
    LEGACY_FREQUENCIES, RESOURCE_TYPES, RESOURCE_FILETYPE_FILE,
)

__all__ = ('DatasetForm', 'ResourceForm', 'CommunityResourceForm')


class ChecksumForm(ModelForm):
    model_class = Checksum
    choices = list(zip(CHECKSUM_TYPES, CHECKSUM_TYPES))
    type = fields.SelectField(choices=choices, default='sha1')
    value = fields.StringField()


def normalize_format(data):
    '''Normalize format field: strip and lowercase'''
    if data:
        return data.strip().lower()


def enforce_filetype_file(form, field):
    '''Only allowed domains in resource.url when filetype is file'''
    if field.data != RESOURCE_FILETYPE_FILE:
        return
    url = form._fields.get('url').data
    domain = urlparse(url).netloc
    allowed_domains = current_app.config.get(
        'RESOURCES_FILE_ALLOWED_DOMAINS', [])
    allowed_domains += [current_app.config.get('SERVER_NAME')]
    if current_app.config.get('CDN_DOMAIN'):
        allowed_domains.append(current_app.config['CDN_DOMAIN'])
    if '*' in allowed_domains:
        return
    if domain and domain not in allowed_domains:
        message = _('Domain "{domain}" not allowed for filetype "{filetype}"').format(
            domain=domain, filetype=RESOURCE_FILETYPE_FILE
        )
        raise validators.ValidationError(message)


class BaseResourceForm(ModelForm):
    title = fields.StringField(_('Title'), [validators.required()])
    description = fields.MarkdownField(_('Description'))
    filetype = fields.RadioField(
        _('File type'), [validators.required(), enforce_filetype_file],
        choices=RESOURCE_FILETYPES.items(), default='file',
        description=_('Whether the resource is an uploaded file, '
                      'a remote file or an API'))
    type = fields.RadioField(
        _('Type'), [validators.required()],
        choices=RESOURCE_TYPES.items(), default='other',
        description=_('Resource type (documentation, API...)'))
    url = fields.UploadableURLField(
        _('URL'), [validators.required()], storage=resources)
    format = fields.StringField(
        _('Format'),
        filters=[normalize_format],
    )
    checksum = fields.FormField(ChecksumForm)
    mime = fields.StringField(
        _('Mime type'),
        description=_('The mime type associated to the extension. '
                      '(ex: text/plain)'))
    filesize = fields.IntegerField(
        _('Size'), [validators.optional()],
        description=_('The file size in bytes'))
    published = fields.DateTimeField(
        _('Publication date'),
        description=_('The publication date of the resource'))
    extras = fields.ExtrasField()


class ResourceForm(BaseResourceForm):
    model_class = Resource

    id = fields.UUIDField()


class CommunityResourceForm(BaseResourceForm):
    model_class = CommunityResource

    dataset = fields.DatasetField(_('Related dataset'))<commit_msg>Added missing commit<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from urlparse import urlparse

from flask import current_app

from udata.forms import ModelForm, fields, validators
from udata.i18n import lazy_gettext as _

from udata.core.storages import resources
from udata.core.spatial.forms import SpatialCoverageField

from .models import (
    Dataset, Resource, License, Checksum, CommunityResource,
    UPDATE_FREQUENCIES, DEFAULT_FREQUENCY, RESOURCE_FILETYPES, CHECKSUM_TYPES,
    LEGACY_FREQUENCIES, RESOURCE_TYPES, RESOURCE_FILETYPE_FILE,
)

__all__ = ('DatasetForm', 'ResourceForm', 'CommunityResourceForm')


class ChecksumForm(ModelForm):
    model_class = Checksum
    choices = list(zip(CHECKSUM_TYPES, CHECKSUM_TYPES))
    type = fields.SelectField(choices=choices, default='sha1')
    value = fields.StringField()


def normalize_format(data):
    '''Normalize format field: strip and lowercase'''
    if data:
        return data.strip().lower()


def enforce_filetype_file(form, field):
    '''Only allowed domains in resource.url when filetype is file'''
    if form._fields.get('filetype').data != RESOURCE_FILETYPE_FILE:
        return
    domain = urlparse(field.data).netloc
    allowed_domains = current_app.config['RESOURCES_FILE_ALLOWED_DOMAINS']
    allowed_domains += [current_app.config.get('SERVER_NAME')]
    if current_app.config.get('CDN_DOMAIN'):
        allowed_domains.append(current_app.config['CDN_DOMAIN'])
    if '*' in allowed_domains:
        return
    if domain and domain not in allowed_domains:
        message = _('Domain "{domain}" not allowed for filetype "{filetype}"')
        raise validators.ValidationError(message.format(
            domain=domain, filetype=RESOURCE_FILETYPE_FILE
        ))


class BaseResourceForm(ModelForm):
    title = fields.StringField(_('Title'), [validators.required()])
    description = fields.MarkdownField(_('Description'))
    filetype = fields.RadioField(
        _('File type'), [validators.required()],
        choices=RESOURCE_FILETYPES.items(), default='file',
        description=_('Whether the resource is an uploaded file, '
                      'a remote file or an API'))
    type = fields.RadioField(
        _('Type'), [validators.required()],
        choices=RESOURCE_TYPES.items(), default='other',
        description=_('Resource type (documentation, API...)'))
    url = fields.UploadableURLField(
        _('URL'), [validators.required(), enforce_filetype_file],
        storage=resources)
    format = fields.StringField(
        _('Format'),
        filters=[normalize_format],
    )
    checksum = fields.FormField(ChecksumForm)
    mime = fields.StringField(
        _('Mime type'),
        description=_('The mime type associated to the extension. '
                      '(ex: text/plain)'))
    filesize = fields.IntegerField(
        _('Size'), [validators.optional()],
        description=_('The file size in bytes'))
    published = fields.DateTimeField(
        _('Publication date'),
        description=_('The publication date of the resource'))
    extras = fields.ExtrasField()


class ResourceForm(BaseResourceForm):
    model_class = Resource

    id = fields.UUIDField()


class CommunityResourceForm(BaseResourceForm):
    model_class = CommunityResource

    dataset = fields.DatasetField(_('Related dataset'))
{'fraction_non_alphanumeric': 0.11213153606873055, 'fraction_numerical': 0.0011850096282032291, 'mean_word_length': 3.751583391977481, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7121385', 'n_tokens_mistral': 1984, 'n_tokens_neox': 1943, 'n_words': 446}