<commit_before>import decimal
import logging
import sys
import json
import os
import re
import types
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.template.defaultfilters import linebreaks
from django.utils.html import strip_tags, urlize
from django.utils.encoding import smart_str
from django.template import defaultfilters
from micawber.contrib.mcdjango import providers
from micawber.exceptions import ProviderException
from micawber.parsers import standalone_url_re, full_handler
from rest_framework import serializers
from sorl.thumbnail.shortcuts import get_thumbnail
from django.core.urlresolvers import reverse


logger = logging.getLogger(__name__)


# TODO Think about adding a feature to set thumbnail quality based on country.
#      This would be useful for countries with slow internet connections.
class SorlImageField(serializers.ImageField):
    def __init__(self, source, geometry_string, **kwargs):
        self.crop = kwargs.pop('crop', 'center')
        self.colorspace = kwargs.pop('colorspace', 'RGB')
        self.geometry_string = geometry_string
        super(SorlImageField, self).__init__(source, **kwargs)

    def to_native(self, value):
        if not value:
            return ""

        if not value.name:
            return ""


        if not os.path.exists(value.path):
            return ""

        # The get_thumbnail() helper doesn't respect the THUMBNAIL_DEBUG setting
        # so we need to deal with exceptions like is done in the template tag.
        thumbnail = ""
        try:
            thumbnail = unicode(get_thumbnail(value, self.geometry_string, crop=self.crop, colorspace=self.colorspace))
        except Exception:
            if getattr(settings, 'THUMBNAIL_DEBUG', None):
                raise
            logger.error('Thumbnail failed:', exc_info=sys.exc_info())
            return ""
        request = self.context.get('request')
        relative_url = settings.MEDIA_URL + thumbnail
        if request:
            return request.build_absolute_uri(relative_url)
        return relative_url


class ContentTextField(serializers.CharField):
    """
    A serializer for content text such as text field found in Reaction and TextWallPost. This serializer creates
    clickable links for text urls and adds <br/> and/or <p></p> in-place of new line characters.
    """

    def to_native(self, value):
        # Convert model instance text -> text for reading.
        text = super(ContentTextField, self).to_native(value)
        # This is equivalent to the django template filter: '{{ value|urlize|linebreaks }}'. Note: Text from the
        # database is escaped again here (on read) just as a double check for HTML / JS injection.
        text = linebreaks(urlize(text, None, True, True))
        # This ensure links open in a new window (BB-136).
        return re.sub(r'<a ', '<a target="_blank" ', text)

    def from_native(self, value):
        # Convert text -> model instance text for writing.
        text = super(ContentTextField, self).from_native(value)
        # HTML tags are stripped and any HTML / JS that is left is escaped.
        return strip_tags(text)


class OEmbedField(serializers.Field):
    def __init__(self, source, maxwidth=None, maxheight=None, **kwargs):
        super(OEmbedField, self).__init__(source)
        self.params = getattr(settings, 'MICAWBER_DEFAULT_SETTINGS', {})
        self.params.update(kwargs)
        if maxwidth and maxheight:
            self.params['maxwidth'] = maxwidth
            self.params['maxheight'] = maxheight
        elif maxwidth:
            self.params['maxwidth'] = maxwidth
            self.params.pop('maxheight', None)

    def to_native(self, value):
        if not value or not standalone_url_re.match(value):
            return ""
        url = value.strip()
        try:
            response = providers.request(url, **self.params)
        except ProviderException:
            return ""
        else:
            return full_handler(url, response, **self.params)


#
# Serializers for django_polymorphic models. See WallPost Serializers for an example on how to use this.
#
class PolymorphicSerializerOptions(serializers.SerializerOptions):
    def __init__(self, meta):
        super(PolymorphicSerializerOptions, self).__init__(meta)
        self.child_models = getattr(meta, 'child_models', None)
<commit_msg>Enforce HTTPS on video links<commit_after>import decimal
import logging
import sys
import json
import os
import re
import types
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.template.defaultfilters import linebreaks
from django.utils.html import strip_tags, urlize
from django.utils.encoding import smart_str
from django.template import defaultfilters
from micawber.contrib.mcdjango import providers
from micawber.exceptions import ProviderException
from micawber.parsers import standalone_url_re, full_handler
from rest_framework import serializers
from sorl.thumbnail.shortcuts import get_thumbnail
from django.core.urlresolvers import reverse


logger = logging.getLogger(__name__)


# TODO Think about adding a feature to set thumbnail quality based on country.
#      This would be useful for countries with slow internet connections.
class SorlImageField(serializers.ImageField):
    def __init__(self, source, geometry_string, **kwargs):
        self.crop = kwargs.pop('crop', 'center')
        self.colorspace = kwargs.pop('colorspace', 'RGB')
        self.geometry_string = geometry_string
        super(SorlImageField, self).__init__(source, **kwargs)

    def to_native(self, value):
        if not value:
            return ""

        if not value.name:
            return ""


        if not os.path.exists(value.path):
            return ""

        # The get_thumbnail() helper doesn't respect the THUMBNAIL_DEBUG setting
        # so we need to deal with exceptions like is done in the template tag.
        thumbnail = ""
        try:
            thumbnail = unicode(get_thumbnail(value, self.geometry_string, crop=self.crop, colorspace=self.colorspace))
        except Exception:
            if getattr(settings, 'THUMBNAIL_DEBUG', None):
                raise
            logger.error('Thumbnail failed:', exc_info=sys.exc_info())
            return ""
        request = self.context.get('request')
        relative_url = settings.MEDIA_URL + thumbnail
        if request:
            return request.build_absolute_uri(relative_url)
        return relative_url


class ContentTextField(serializers.CharField):
    """
    A serializer for content text such as text field found in Reaction and TextWallPost. This serializer creates
    clickable links for text urls and adds <br/> and/or <p></p> in-place of new line characters.
    """

    def to_native(self, value):
        # Convert model instance text -> text for reading.
        text = super(ContentTextField, self).to_native(value)
        # This is equivalent to the django template filter: '{{ value|urlize|linebreaks }}'. Note: Text from the
        # database is escaped again here (on read) just as a double check for HTML / JS injection.
        text = linebreaks(urlize(text, None, True, True))
        # This ensure links open in a new window (BB-136).
        return re.sub(r'<a ', '<a target="_blank" ', text)

    def from_native(self, value):
        # Convert text -> model instance text for writing.
        text = super(ContentTextField, self).from_native(value)
        # HTML tags are stripped and any HTML / JS that is left is escaped.
        return strip_tags(text)


class OEmbedField(serializers.Field):
    def __init__(self, source, maxwidth=None, maxheight=None, **kwargs):
        super(OEmbedField, self).__init__(source)
        self.params = getattr(settings, 'MICAWBER_DEFAULT_SETTINGS', {})
        self.params.update(kwargs)
        # enforce HTTPS, see https://groups.google.com/forum/?fromgroups#!topic/youtube-api-gdata/S9Fa-Zw2Ma8
        self.params['scheme'] = 'https'
        if maxwidth and maxheight:
            self.params['maxwidth'] = maxwidth
            self.params['maxheight'] = maxheight
        elif maxwidth:
            self.params['maxwidth'] = maxwidth
            self.params.pop('maxheight', None)

    def to_native(self, value):
        if not value or not standalone_url_re.match(value):
            return ""
        url = value.strip()
        try:
            response = providers.request(url, **self.params)
        except ProviderException:
            return ""
        else:
            return full_handler(url, response, **self.params)


#
# Serializers for django_polymorphic models. See WallPost Serializers for an example on how to use this.
#
class PolymorphicSerializerOptions(serializers.SerializerOptions):
    def __init__(self, meta):
        super(PolymorphicSerializerOptions, self).__init__(meta)
        self.child_models = getattr(meta, 'child_models', None)

{'fraction_non_alphanumeric': 0.08590914090859092, 'fraction_numerical': 0.000989990100098999, 'mean_word_length': 3.0534997770842622, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5551275', 'n_tokens_mistral': 2476, 'n_tokens_neox': 2420, 'n_words': 766}