<commit_before>from django.http import HttpResponse
from django.contrib.gis.geos import GEOSGeometry
from django.contrib.gis.geos import WKTWriter
from django.utils import simplejson as json
import types

def geojson_to_csv(request):
    """
    This function should recieve a post request with
    a featurecollection.

    From this featurecollection this function will make a csv
    string and return it as a csv file for download.
    """

    json_string = request.raw_post_data

    feature_collection = json.loads(json_string)

    crs = getattr(feature_collection,
                  'crs',
                  {
                    'type': 'name',
                    'properties': {
                        'name': 'urn:ogc:def:crs:EPSG::4979'
                        }})

    features = feature_collection['features']

    csv_header_set = set()
    for feature in features:
        csv_header_set.update(create_csv_header_set(feature['properties']))

    csv_header_list = list(csv_header_set)
    csv_header_list.sort()
    csv_header_list.insert(0, 'wkt')
    csv_header_list.insert(0, 'id')

    #create the csv
    csv_string = u""

    #write the header
    for header in csv_header_list:
        csv_string = u"%s%s;" % (csv_string, header)

    csv_string = u"%s\n" % (csv_string,)
    csv_row = []
    csv_rows = []
    print csv_string

    for feature in features:

        geometry = feature['geometry']
        properties = feature['properties']

        id = None
        if feature.has_key('id'):
            id = feature['id']

        properties['id'] = id

        #make the geometry
        properties['wkt'] = GEOSGeometry(json.dumps(geometry)).wkt

        value_list = get_value_list(properties, csv_header_list)

        for value in value_list:

            #remove harmful characterers
            if type(value) == types.UnicodeType or type(value) == types.StringType:
                value = value.replace(";", "")
                value = value.replace("\n", " ")
                value = value.replace("\r", " ")

            csv_row.append("%s;" % value)

        csv_rows.append(''.join(csv_row))

    csv_string = "%s%s" % (csv_string, '\n'.join(csv_rows))

    return HttpResponse(csv_string,
                        content_type='text/csv')


def json_to_csv(request):
    """
    This function takes a array of json objects
    and returns a csv file for download.
    """

    json_string = request.raw_post_data

    json_array = json.loads(json_string)

    if type(json_array) == types.DictType:
        json_array = [json_array]

    #this part has to check for nested objects
    csv_header_set = set()
    for json_dict in json_array:
        csv_header_set.update(create_csv_header_set(json_dict))

    csv_header_list = list(csv_header_set)
    csv_header_list.sort()

    #create the csv
    csv_string = u""

    #write the header
    for header in csv_header_list:
        csv_string = u"%s%s;" % (csv_string, header)

    csv_string = u"%s\n" % (csv_string,)
    csv_row = []
    csv_rows = []

    for json_dict in json_array:
        value_list = get_value_list(json_dict, csv_header_list)

        for value in value_list:

            #remove harmful characterers
            if type(value) == types.UnicodeType or type(value) == types.StringType:
                value = value.replace(";", "")
                value = value.replace("\n", " ")
                value = value.replace("\r", " ")

            #modify value to json string
            value = json.dumps(value)

            csv_row.append("%s;" % value)

        csv_rows.append(''.join(csv_row))

    csv_string = "%s%s" % (csv_string, '\n'.join(csv_rows))

    return HttpResponse(csv_string,
                        mimetype='text/csv')

def create_csv_header_set(json_dict):

    csv_header_set = set()

    for key, item in json_dict.items():
        if type(item) == types.DictType:
            temp_set = create_csv_header_set(item)
            for header in temp_set:
                add_header = "%s.%s" % (key, header)
                csv_header_set.add(add_header)
        else:<commit_msg>fixed a bug with not emptying rows between rows<commit_after>from django.http import HttpResponse
from django.contrib.gis.geos import GEOSGeometry
from django.contrib.gis.geos import WKTWriter
from django.utils import simplejson as json
import types

def geojson_to_csv(request):
    """
    This function should recieve a post request with
    a featurecollection.

    From this featurecollection this function will make a csv
    string and return it as a csv file for download.
    """

    json_string = request.raw_post_data

    feature_collection = json.loads(json_string)

    crs = getattr(feature_collection,
                  'crs',
                  {
                    'type': 'name',
                    'properties': {
                        'name': 'urn:ogc:def:crs:EPSG::4979'
                        }})

    features = feature_collection['features']

    csv_header_set = set()
    for feature in features:
        csv_header_set.update(create_csv_header_set(feature['properties']))

    csv_header_list = list(csv_header_set)
    csv_header_list.sort()
    csv_header_list.insert(0, 'wkt')
    csv_header_list.insert(0, 'id')

    #create the csv
    csv_string = u""

    #write the header
    for header in csv_header_list:
        csv_string = u"%s%s;" % (csv_string, header)

    csv_string = u"%s\n" % (csv_string,)
    csv_row = []
    csv_rows = []
    print csv_string

    for feature in features:

        geometry = feature['geometry']
        properties = feature['properties']

        id = None
        if feature.has_key('id'):
            id = feature['id']

        properties['id'] = id

        #make the geometry
        properties['wkt'] = GEOSGeometry(json.dumps(geometry)).wkt

        value_list = get_value_list(properties, csv_header_list)

        for value in value_list:

            #remove harmful characterers
            if type(value) == types.UnicodeType or type(value) == types.StringType:
                value = value.replace(";", "")
                value = value.replace("\n", " ")
                value = value.replace("\r", " ")

            csv_row.append("%s;" % value)

        csv_rows.append(''.join(csv_row))
        csv_row = []

    csv_string = "%s%s" % (csv_string, '\n'.join(csv_rows))

    return HttpResponse(csv_string,
                        content_type='text/csv')


def json_to_csv(request):
    """
    This function takes a array of json objects
    and returns a csv file for download.
    """

    json_string = request.raw_post_data

    json_array = json.loads(json_string)

    if type(json_array) == types.DictType:
        json_array = [json_array]

    #this part has to check for nested objects
    csv_header_set = set()
    for json_dict in json_array:
        csv_header_set.update(create_csv_header_set(json_dict))

    csv_header_list = list(csv_header_set)
    csv_header_list.sort()

    #create the csv
    csv_string = u""

    #write the header
    for header in csv_header_list:
        csv_string = u"%s%s;" % (csv_string, header)

    csv_string = u"%s\n" % (csv_string,)
    csv_row = []
    csv_rows = []

    for json_dict in json_array:
        value_list = get_value_list(json_dict, csv_header_list)

        for value in value_list:

            #remove harmful characterers
            if type(value) == types.UnicodeType or type(value) == types.StringType:
                value = value.replace(";", "")
                value = value.replace("\n", " ")
                value = value.replace("\r", " ")

            #modify value to json string
            value = json.dumps(value)

            csv_row.append("%s;" % value)

        csv_rows.append(''.join(csv_row))
        csv_row = []

    csv_string = "%s%s" % (csv_string, '\n'.join(csv_rows))

    return HttpResponse(csv_string,
                        mimetype='text/csv')

def create_csv_header_set(json_dict):

    csv_header_set = set()

    for key, item in json_dict.items():
        if type(item) == types.DictType:
            temp_set = create_csv_header_set(item)
            for header in temp_set:
                add_header = "%s.%s" % (key, header)
                csv_header_set.add(add_header)
        else:
{'fraction_non_alphanumeric': 0.10865384615384616, 'fraction_numerical': 0.0014423076923076924, 'mean_word_length': 2.2593027810419115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2849124', 'n_tokens_mistral': 2668, 'n_tokens_neox': 2479, 'n_words': 583}