<commit_before>from django import forms
from django.core.files import File
from django.conf import settings

import widgets
import urllib2
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO


class FPFileField(forms.FileField):
    widget = widgets.FPFileWidget
    default_mimetypes = "*/*"

    def __init__(self, *args, **kwargs):
        """
        Initializes the Filepicker file field.
        Valid arguments:
        * apikey. This string is required if it isn't set as settings.FILEPICKER_API_KEY
        * mimetypes. Optional, the allowed mimetypes for files. Defaults to "*/*" (all files)
        """

        if 'apikey' in kwargs:
            self.apikey = kwargs.pop('apikey')
        elif hasattr(settings, 'FILEPICKER_API_KEY'):
            self.apikey = settings.FILEPICKER_API_KEY
        else:
            raise Exception("Cannot find filepicker.io api key." +
            " Be sure to either pass as the apikey argument when creating the FPFileField," +
            " or set it as settings.FILEPICKER_API_KEY. To get a key, go to https://filepicker.io")

        self.mimetypes = kwargs.pop('mimetypes', self.default_mimetypes)
        if not isinstance(self.mimetypes, basestring):
            #If mimetypes is an array, form a csv string
            try:
                self.mimetypes = ",".join(iter(self.mimetypes))
            except TypeError:
                self.mimetypes = str(self.mimetypes)

        super(FPFileField, self).__init__(*args, **kwargs)

    def widget_attrs(self, widget):
        attrs = {
                'data-fp-apikey': self.apikey,
                'data-fp-mimetypes': self.mimetypes,
                }

        if hasattr(settings, 'FILEPICKER_SERVICES'):
            attrs['data-fp-option-services'] = settings.FILEPICKER_SERVICES

        return attrs

    def to_python(self, data):
        """Takes the url in data and creates a File object"""
        if not data:
            return None

        url_fp = urllib2.urlopen(data)

        disposition = url_fp.info().getheader('Content-Disposition')
        if disposition:
            name = disposition.rpartition("filename=")[2].strip('" ')
        else:
            name = "fp-file"
        size = long(url_fp.info().getheader('Content-Length', 0))

        fp = File(StringIO(url_fp.read()), name=name)<commit_msg>make sure 'data' looks like a URL before fetching it; Otherwise, this might be a file we already have (and we're in the Admin.)<commit_after>from django import forms
from django.core.files import File
from django.conf import settings

import widgets
import urllib2
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO


class FPFileField(forms.FileField):
    widget = widgets.FPFileWidget
    default_mimetypes = "*/*"

    def __init__(self, *args, **kwargs):
        """
        Initializes the Filepicker file field.
        Valid arguments:
        * apikey. This string is required if it isn't set as settings.FILEPICKER_API_KEY
        * mimetypes. Optional, the allowed mimetypes for files. Defaults to "*/*" (all files)
        """

        if 'apikey' in kwargs:
            self.apikey = kwargs.pop('apikey')
        elif hasattr(settings, 'FILEPICKER_API_KEY'):
            self.apikey = settings.FILEPICKER_API_KEY
        else:
            raise Exception("Cannot find filepicker.io api key." +
            " Be sure to either pass as the apikey argument when creating the FPFileField," +
            " or set it as settings.FILEPICKER_API_KEY. To get a key, go to https://filepicker.io")

        self.mimetypes = kwargs.pop('mimetypes', self.default_mimetypes)
        if not isinstance(self.mimetypes, basestring):
            #If mimetypes is an array, form a csv string
            try:
                self.mimetypes = ",".join(iter(self.mimetypes))
            except TypeError:
                self.mimetypes = str(self.mimetypes)

        super(FPFileField, self).__init__(*args, **kwargs)

    def widget_attrs(self, widget):
        attrs = {
                'data-fp-apikey': self.apikey,
                'data-fp-mimetypes': self.mimetypes,
                }

        if hasattr(settings, 'FILEPICKER_SERVICES'):
            attrs['data-fp-option-services'] = settings.FILEPICKER_SERVICES

        return attrs

    def to_python(self, data):
        """Takes the url in data and creates a File object"""
        if not data or not data.startswith('http'):
            return None

        url_fp = urllib2.urlopen(data)

        disposition = url_fp.info().getheader('Content-Disposition')
        if disposition:
            name = disposition.rpartition("filename=")[2].strip('" ')
        else:
            name = "fp-file"
        size = long(url_fp.info().getheader('Content-Length', 0))

        fp = File(StringIO(url_fp.read()), name=name)
{'fraction_non_alphanumeric': 0.10087988541027215, 'fraction_numerical': 0.0016369961121342337, 'mean_word_length': 2.3502398903358466, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4301856', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1423, 'n_words': 393}