<commit_before>import csv
from xml.dom.minidom import Document
import hashlib

from django.http import HttpResponse, HttpResponseBadRequest
from django.template.defaultfilters import slugify
import simplejson as json


## Taken from http://djangosnippets.org/snippets/790/
def export_csv(qs, filename, fields=None):
    model = qs.model
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = ('attachment; filename=%s.csv'
                                       % slugify(filename))
    writer = csv.writer(response)
    # Write headers to CSV file
    if fields:
        headers = fields
    else:
        headers = []
        for field in model._meta.fields:
            headers.append(field.name)
        fields = headers
    _headers = []
    for header in headers:
        if header:
            _headers.append(header)
        else:
            _headers.append(unicode(model._meta.verbose_name))

    headers = _headers

    writer.writerow(headers)
    # Write data to CSV file
    for obj in qs:
        row = []
        for field in fields:
            if field == '':
                val = unicode(obj)
            else:
                val = getattr(obj, field)
                if callable(val):
                    val = val()
                elif getattr(val, 'all', None):
                    val = ', '.join([unicode(item) for item in val.all()])
                # work around csv unicode limitation
                elif type(val) == unicode:
                    val = val.encode("utf-8")
            row.append(val)
        writer.writerow(row)
    # Return CSV file to browser as download
    return response


def export_json(qs, filename, fields=None):
    model = qs.model

    if not fields:
        headers = []
        for field in model._meta.fields:
            headers.append(field.name)
        fields = headers
    objs = []
    for obj in qs:
        item = {}
        for field in fields:
            if field == '':
                field = unicode(obj._meta.verbose_name)
                val = unicode(obj)
            else:
                val = getattr(obj, field)
                if callable(val):
                    val = val()
                elif getattr(val, 'all', None):
                    val = [unicode(i) for i in val.all()]
                # work around csv unicode limitation
                elif type(val) == unicode:
                    val = val.encode("utf-8")
            item[field] = val
        objs.append(item)
    # Return JS file to browser as download<commit_msg>Error in export json and csv<commit_after>import csv
from xml.dom.minidom import Document
import hashlib

from django.http import HttpResponse, HttpResponseBadRequest
from django.template.defaultfilters import slugify
import simplejson as json


## Taken from http://djangosnippets.org/snippets/790/
def export_csv(qs, filename, fields=None):
    model = qs.model
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = ('attachment; filename=%s.csv'
                                       % slugify(filename))
    writer = csv.writer(response)
    # Write headers to CSV file
    if fields:
        headers = fields
    else:
        headers = []
        for field in model._meta.fields:
            headers.append(field.name)
        fields = headers
    _headers = []
    for header in headers:
        if header:
            _headers.append(header)
        else:
            _headers.append(unicode(model._meta.verbose_name))

    headers = _headers

    writer.writerow(headers)
    # Write data to CSV file
    for obj in qs:
        row = []
        for field in fields:
            if field == '':
                val = unicode(obj)
            else:
                val = getattr(obj, field)
                if getattr(val, 'all', None):
                    val = ', '.join([unicode(item) for item in val.all()])
                # work around csv unicode limitation
                elif type(val) == unicode:
                    val = val.encode("utf-8")
            row.append(val)
        writer.writerow(row)
    # Return CSV file to browser as download
    return response


def export_json(qs, filename, fields=None):
    model = qs.model

    if not fields:
        headers = []
        for field in model._meta.fields:
            headers.append(field.name)
        fields = headers
    objs = []
    for obj in qs:
        item = {}
        for field in fields:
            if field == '':
                field = unicode(obj._meta.verbose_name)
                val = unicode(obj)
            else:
                val = getattr(obj, field)
                if getattr(val, 'all', None):
                    val = [unicode(i) for i in val.all()]
                # work around csv unicode limitation
                elif type(val) == unicode:
                    val = val.encode("utf-8")
            item[field] = val
        objs.append(item)
    # Return JS file to browser as download
{'fraction_non_alphanumeric': 0.09445667400440264, 'fraction_numerical': 0.0020012007204322593, 'mean_word_length': 1.650053022269353, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1766341', 'n_tokens_mistral': 1401, 'n_tokens_neox': 1305, 'n_words': 390}