<commit_before>import hashlib
from itertools import islice
from couchdbkit.ext.django.schema import Document, IntegerProperty, DictProperty,\
    DocumentSchema, StringProperty, SchemaListProperty, ListProperty,\
    StringListProperty, DateTimeProperty, SchemaProperty, BooleanProperty
import json
from StringIO import StringIO
import couchexport
from couchexport.util import SerializableFunctionProperty,\
    get_schema_index_view_keys, force_tag_to_list, clear_attachments
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.mixins import UnicodeMixIn
from dimagi.utils.couch.database import get_db, iter_docs
from soil import DownloadBase
from couchdbkit.exceptions import ResourceNotFound
from couchexport.properties import TimeStampProperty, JsonProperty
from couchdbkit.consumer import Consumer
from dimagi.utils.logging import notify_exception

class Format(object):
    """
    Supported formats go here.
    """
    CSV = "csv"
    ZIP = "zip"
    XLS = "xls"
    XLS_2007 = "xlsx"
    HTML = "html"
    JSON = "json"
    
    FORMAT_DICT = {CSV: {"mimetype": "application/zip",
                         "extension": "zip",
                         "download": True},
                   ZIP: {"mimetype": "application/zip",
                         "extension": "zip",
                         "download": True},
                   XLS: {"mimetype": "application/vnd.ms-excel",
                         "extension": "xls",
                         "download": True},
                   XLS_2007: {"mimetype": "application/vnd.ms-excel",
                              "extension": "xlsx",
                              "download": True},
                   HTML: {"mimetype": "text/html",
                          "extension": "html",
                          "download": False},
                   JSON: {"mimetype": "application/json",
                          "extension": "json",
                          "download": False}}
    
    VALID_FORMATS = FORMAT_DICT.keys()
    
    def __init__(self, slug, mimetype, extension, download):
        self.slug = slug
        self.mimetype = mimetype
        self.extension = extension
        self.download = download
    
    @classmethod
    def from_format(cls, format):
        format = format.lower()
        if format not in cls.VALID_FORMATS:
            raise ValueError("Unsupported export format: %s!" % format)
        return cls(format, **cls.FORMAT_DICT[format])

class ExportSchema(Document, UnicodeMixIn):
    """
    An export schema that can store intermittent contents of the export so
    that the entire doc list doesn't have to be used to generate the export
    """
    index = JsonProperty()
    seq = StringProperty() # semi-deprecated
    schema = DictProperty()
    timestamp = TimeStampProperty()

    def __unicode__(self):
        return "%s: %s" % (json.dumps(self.index), self.seq)

    @property
    def is_bigcouch(self):<commit_msg>raise URLError instead of ValueError for unsupported report export format<commit_after>import hashlib
from itertools import islice
from urllib2 import URLError
from couchdbkit.ext.django.schema import Document, IntegerProperty, DictProperty,\
    DocumentSchema, StringProperty, SchemaListProperty, ListProperty,\
    StringListProperty, DateTimeProperty, SchemaProperty, BooleanProperty
import json
from StringIO import StringIO
import couchexport
from couchexport.util import SerializableFunctionProperty,\
    get_schema_index_view_keys, force_tag_to_list, clear_attachments
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.mixins import UnicodeMixIn
from dimagi.utils.couch.database import get_db, iter_docs
from soil import DownloadBase
from couchdbkit.exceptions import ResourceNotFound
from couchexport.properties import TimeStampProperty, JsonProperty
from couchdbkit.consumer import Consumer
from dimagi.utils.logging import notify_exception

class Format(object):
    """
    Supported formats go here.
    """
    CSV = "csv"
    ZIP = "zip"
    XLS = "xls"
    XLS_2007 = "xlsx"
    HTML = "html"
    JSON = "json"
    
    FORMAT_DICT = {CSV: {"mimetype": "application/zip",
                         "extension": "zip",
                         "download": True},
                   ZIP: {"mimetype": "application/zip",
                         "extension": "zip",
                         "download": True},
                   XLS: {"mimetype": "application/vnd.ms-excel",
                         "extension": "xls",
                         "download": True},
                   XLS_2007: {"mimetype": "application/vnd.ms-excel",
                              "extension": "xlsx",
                              "download": True},
                   HTML: {"mimetype": "text/html",
                          "extension": "html",
                          "download": False},
                   JSON: {"mimetype": "application/json",
                          "extension": "json",
                          "download": False}}
    
    VALID_FORMATS = FORMAT_DICT.keys()
    
    def __init__(self, slug, mimetype, extension, download):
        self.slug = slug
        self.mimetype = mimetype
        self.extension = extension
        self.download = download
    
    @classmethod
    def from_format(cls, format):
        format = format.lower()
        if format not in cls.VALID_FORMATS:
            raise URLError("Unsupported export format: %s!" % format)
        return cls(format, **cls.FORMAT_DICT[format])

class ExportSchema(Document, UnicodeMixIn):
    """
    An export schema that can store intermittent contents of the export so
    that the entire doc list doesn't have to be used to generate the export
    """
    index = JsonProperty()
    seq = StringProperty() # semi-deprecated
    schema = DictProperty()
    timestamp = TimeStampProperty()

    def __unicode__(self):
        return "%s: %s" % (json.dumps(self.index), self.seq)

    @property
    def is_bigcouch(self):
{'fraction_non_alphanumeric': 0.09496739675639525, 'fraction_numerical': 0.002842334057849858, 'mean_word_length': 2.2940528634361232, 'pattern_counts': {'":': 36, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6847898', 'n_tokens_mistral': 1651, 'n_tokens_neox': 1583, 'n_words': 440}