<commit_before>import re

from urlparse import urlparse

from django.core.exceptions import ValidationError
from django.db import models

from monocle.providers import Provider, registry
from monocle.settings import settings


RESOURCE_CHOICES = [(x, x.capitalize()) for x in settings.RESOURCE_TYPES]


class ThirdPartyProvider(models.Model, Provider):
    """
    Database-backed third-party provider configuration
    """
    name = models.CharField(max_length=50, blank=True)
    api_endpoint = models.URLField(verify_exists=False)
    resource_type = models.CharField(choices=RESOURCE_CHOICES, max_length=10)
    is_active = models.BooleanField(default=True)
    expose = models.BooleanField(default=False,
                                 help_text="Expose this resource to external requests")

    class Meta:
        ordering = ('api_endpoint', 'resource_type')

    @property
    def url_schemes(self):
        return list(self._schemes.all())

    def clean(self):
        """Ensures the API endpoint is valid"""
        if not self.api_endpoint:
            raise ValidationError('API Endpoint is required')

        if self.api_endpoint.lower().startswith('https'):
            raise ValidationError('API Endpoint cannot be a HTTPS endpoint')

    def __unicode__(self):
        return self.name or self.api_endpoint


class URLScheme(models.Model):
    scheme = models.CharField(max_length=255, unique=True,
                              help_text="Wildcard URL pattern: http://*.flickr.com/photos/*")
    provider = models.ForeignKey('ThirdPartyProvider', related_name='_schemes')

    def clean(self):
        """
        Validate the URL scheme according to these examples

        OK: http://www.flickr.com/photos/*
        OK: http://www.flickr.com/photos/*/foo
        OK: http://*.flickr.com/photos/*
        NOT OK: https://www.flickr.com/photos/*
        NOT OK: http://*.com/photos/*
        NOT OK: *://www.flickr.com/photos/*
        """
        if not self.scheme:
            raise ValidationError('URL Scheme is required')

        parts = urlparse(self.scheme.lower())

        if not parts.scheme or parts.scheme == 'https':
            raise ValidationError('URL Scheme cannot be a wildcard and must not be HTTPS')

        if re.match(r'\*\.?(\w{3}|(\w{2}\.)?\w{2})$', parts.netloc):
            raise ValidationError('URL Scheme is too agressive')

        if not self.provider:
            raise ValidationError('This URL Scheme must belong to a provider')

    def __unicode__(self):
        return self.scheme


def _update_provider(sender, instance, created, **kwargs):
    """Post-save signal callback"""
    registry.update(instance)


def _unregister_provider(sender, instance, **kwargs):
    """Post-delete signal callback"""
    registry.unregister(instance)


# Connect signals
models.signals.post_save.connect(_update_provider, sender=ThirdPartyProvider)
models.signals.post_delete.connect(_unregister_provider, sender=ThirdPartyProvider)

# Register providers
registry.ensure()<commit_msg>registry.ensure() fails when the database is out of sync<commit_after>import re

from urlparse import urlparse

from django.core.exceptions import ValidationError
from django.db import models

from monocle.providers import Provider, registry
from monocle.settings import settings


RESOURCE_CHOICES = [(x, x.capitalize()) for x in settings.RESOURCE_TYPES]


class ThirdPartyProvider(models.Model, Provider):
    """
    Database-backed third-party provider configuration
    """
    name = models.CharField(max_length=50, blank=True)
    api_endpoint = models.URLField(verify_exists=False)
    resource_type = models.CharField(choices=RESOURCE_CHOICES, max_length=10)
    is_active = models.BooleanField(default=True)
    expose = models.BooleanField(default=False,
                                 help_text="Expose this resource to external requests")

    class Meta:
        ordering = ('api_endpoint', 'resource_type')

    @property
    def url_schemes(self):
        return list(self._schemes.all())

    def clean(self):
        """Ensures the API endpoint is valid"""
        if not self.api_endpoint:
            raise ValidationError('API Endpoint is required')

        if self.api_endpoint.lower().startswith('https'):
            raise ValidationError('API Endpoint cannot be a HTTPS endpoint')

    def __unicode__(self):
        return self.name or self.api_endpoint


class URLScheme(models.Model):
    scheme = models.CharField(max_length=255, unique=True,
                              help_text="Wildcard URL pattern: http://*.flickr.com/photos/*")
    provider = models.ForeignKey('ThirdPartyProvider', related_name='_schemes')

    def clean(self):
        """
        Validate the URL scheme according to these examples

        OK: http://www.flickr.com/photos/*
        OK: http://www.flickr.com/photos/*/foo
        OK: http://*.flickr.com/photos/*
        NOT OK: https://www.flickr.com/photos/*
        NOT OK: http://*.com/photos/*
        NOT OK: *://www.flickr.com/photos/*
        """
        if not self.scheme:
            raise ValidationError('URL Scheme is required')

        parts = urlparse(self.scheme.lower())

        if not parts.scheme or parts.scheme == 'https':
            raise ValidationError('URL Scheme cannot be a wildcard and must not be HTTPS')

        if re.match(r'\*\.?(\w{3}|(\w{2}\.)?\w{2})$', parts.netloc):
            raise ValidationError('URL Scheme is too agressive')

        if not self.provider:
            raise ValidationError('This URL Scheme must belong to a provider')

    def __unicode__(self):
        return self.scheme


def _update_provider(sender, instance, created, **kwargs):
    """Post-save signal callback"""
    registry.update(instance)


def _unregister_provider(sender, instance, **kwargs):
    """Post-delete signal callback"""
    registry.unregister(instance)


# Connect signals
models.signals.post_save.connect(_update_provider, sender=ThirdPartyProvider)
models.signals.post_delete.connect(_unregister_provider, sender=ThirdPartyProvider)

# Register providers - but only if the model is available
try:
    registry.ensure()
except:
    pass
{'fraction_non_alphanumeric': 0.1117952041477641, 'fraction_numerical': 0.0032404406999351912, 'mean_word_length': 3.0880794701986756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6297748', 'n_tokens_mistral': 1802, 'n_tokens_neox': 1685, 'n_words': 447}