<commit_before># -*- coding: utf-8 -*-

import os
import zipfile
import tempfile
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from django.conf import settings
from django.utils.encoding import smart_str
from django.contrib.gis.db.models.fields import GeometryField
from django.contrib.gis.gdal import check_err, OGRGeomType

from osgeo import ogr, osr


def zip_shapes(shapes, delete=True):
    buffr = StringIO()
    zipf = zipfile.ZipFile(buffr, 'w', zipfile.ZIP_DEFLATED)

    for shp_name, shp_path in shapes:
        zip_shapefile_path(zipf, shp_path, shp_name, delete)

    zipf.close()
    buffr.flush()

    zip_stream = buffr.getvalue()
    buffr.close()

    return zip_stream


def zip_shapefile_path(zipf, shapefile_path, file_name, delete=True):
    files = ['shp', 'shx', 'prj', 'dbf']
    for item in files:
        filename = '%s.%s' % (shapefile_path.replace('.shp', ''), item)
        zipf.write(filename, arcname='%s.%s' % (file_name.replace('.shp', ''), item))
        if delete:
            os.remove(filename)


class ShapeCreator(object):

    def __init__(self):
        self.shapes = []  # list of pair (shp name, shp path)

    def add_shape(self, shp_name, shp_filepath):
        self.shapes.append((shp_name, shp_filepath))

    def add_shape_from_qs(self, qs, srid_out=None, shp_name='shp_download'):
        self.add_shape_from_model(qs, qs.model, srid_out, shp_name)

    def add_shape_from_model(self, iterable, model, srid_out=None, shp_name='shp_download'):
        fieldmap = fieldmap_from_model(model)
        get_geom, geom_type, srid = info_from_geo_field(geo_field_from_model(model))

        shp_filepath = shape_write(iterable, fieldmap, get_geom, geom_type, srid, srid_out)
        self.add_shape(shp_name, shp_filepath)

    def as_zip(self):
        return zip_shapes(self.shapes)


def shape_write(iterable, fieldmap, get_geom, geom_type, srid, srid_out=None):

    # normalize ?!
    fieldmap = dict((k[:10], v) for k, v in fieldmap.iteritems())

    # refactor
    tmp_file, layer, ds, native_srs, output_srs = create_shape_format_layer(fieldmap.keys(), geom_type, srid, srid_out)

    feature_def = layer.GetLayerDefn()

    transform = lambda ogr_geom: ogr_geom

    # omg.. will not work for 3D coords
    if native_srs != output_srs:
        ct = osr.CoordinateTransformation(native_srs, output_srs)

        def transform(ogr_geom):
            ogr_geom.Transform(ct)
            return ogr_geom

    for item in iterable:
        feat = ogr.Feature(feature_def)

        for fieldname, getter in fieldmap.iteritems():
            feat.SetField(fieldname, getter(item))

        geom = get_geom(item)
        if geom:
            ogr_geom = transform(ogr.CreateGeometryFromWkt(geom.wkt))
            check_err(feat.SetGeometry(ogr_geom))

        check_err(layer.CreateFeature(feat))

    ds.Destroy()

    return tmp_file.name


def create_shape_format_layer(fieldnames, geom_type, srid, srid_out=None):
    # Create temp file
    tmp = tempfile.NamedTemporaryFile(suffix='.shp', mode='w+b', dir=settings.TEMP_DIR)
    # we must close the file for GDAL to be able to open and write to it
    tmp.close()
    # create shape format

    dr = ogr.GetDriverByName('ESRI Shapefile')
    ds = dr.CreateDataSource(tmp.name)
    if ds is None:
        raise Exception('Could not create file!')

    ogr_type = OGRGeomType(geom_type).num

    native_srs = osr.SpatialReference()
    native_srs.ImportFromEPSG(srid)

    if srid_out:
        output_srs = osr.SpatialReference()
        output_srs.ImportFromEPSG(srid_out)
    else:
        output_srs = native_srs

    layer = ds.CreateLayer('lyr', srs=output_srs, geom_type=ogr_type)
    if layer is None:
        raise ValueError('Could not create layer (type=%s, srs=%s)' % (geom_type, output_srs))

    # Create other fields
    for fieldname in fieldnames:
        field_defn = ogr.FieldDefn(fieldname, ogr.OFTString)
        field_defn.SetWidth(255)
        if layer.CreateField(field_defn) != 0:
            raise Exception('Faild to create field')

    return tmp, layer, ds, native_srs, output_srs


def fieldmap_from_fields(model, fieldnames):
    from .serializers import plain_text
    return dict(
        (fname, lambda x, fname=fname: smart_str(plain_text(getattr(x, fname + '_csv_display',
                                                            getattr(x, fname + '_display', getattr(x, fname))))))
        for fname in fieldnames
    )


def fieldmap_from_model(model):
    """Extract all non geometry fields from model"""
    fields = model._meta.fields
    non_geometry_field_names = [f.name for f in fields if not isinstance(f, GeometryField)]

    return fieldmap_from_fields(model, non_geometry_field_names)


def geo_field_from_model(model, default_geo_field_name=None):<commit_msg>Make sure field value is string before serializing (fixes #706)<commit_after># -*- coding: utf-8 -*-

import os
import zipfile
import tempfile
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from django.conf import settings
from django.utils.encoding import smart_str
from django.contrib.gis.db.models.fields import GeometryField
from django.contrib.gis.gdal import check_err, OGRGeomType

from osgeo import ogr, osr


def zip_shapes(shapes, delete=True):
    buffr = StringIO()
    zipf = zipfile.ZipFile(buffr, 'w', zipfile.ZIP_DEFLATED)

    for shp_name, shp_path in shapes:
        zip_shapefile_path(zipf, shp_path, shp_name, delete)

    zipf.close()
    buffr.flush()

    zip_stream = buffr.getvalue()
    buffr.close()

    return zip_stream


def zip_shapefile_path(zipf, shapefile_path, file_name, delete=True):
    files = ['shp', 'shx', 'prj', 'dbf']
    for item in files:
        filename = '%s.%s' % (shapefile_path.replace('.shp', ''), item)
        zipf.write(filename, arcname='%s.%s' % (file_name.replace('.shp', ''), item))
        if delete:
            os.remove(filename)


class ShapeCreator(object):

    def __init__(self):
        self.shapes = []  # list of pair (shp name, shp path)

    def add_shape(self, shp_name, shp_filepath):
        self.shapes.append((shp_name, shp_filepath))

    def add_shape_from_qs(self, qs, srid_out=None, shp_name='shp_download'):
        self.add_shape_from_model(qs, qs.model, srid_out, shp_name)

    def add_shape_from_model(self, iterable, model, srid_out=None, shp_name='shp_download'):
        fieldmap = fieldmap_from_model(model)
        get_geom, geom_type, srid = info_from_geo_field(geo_field_from_model(model))

        shp_filepath = shape_write(iterable, fieldmap, get_geom, geom_type, srid, srid_out)
        self.add_shape(shp_name, shp_filepath)

    def as_zip(self):
        return zip_shapes(self.shapes)


def shape_write(iterable, fieldmap, get_geom, geom_type, srid, srid_out=None):

    # normalize ?!
    fieldmap = dict((k[:10], v) for k, v in fieldmap.iteritems())

    # refactor
    tmp_file, layer, ds, native_srs, output_srs = create_shape_format_layer(fieldmap.keys(), geom_type, srid, srid_out)

    feature_def = layer.GetLayerDefn()

    transform = lambda ogr_geom: ogr_geom

    # omg.. will not work for 3D coords
    if native_srs != output_srs:
        ct = osr.CoordinateTransformation(native_srs, output_srs)

        def transform(ogr_geom):
            ogr_geom.Transform(ct)
            return ogr_geom

    for item in iterable:
        feat = ogr.Feature(feature_def)

        for fieldname, getter in fieldmap.iteritems():
            feat.SetField(fieldname, getter(item))

        geom = get_geom(item)
        if geom:
            ogr_geom = transform(ogr.CreateGeometryFromWkt(geom.wkt))
            check_err(feat.SetGeometry(ogr_geom))

        check_err(layer.CreateFeature(feat))

    ds.Destroy()

    return tmp_file.name


def create_shape_format_layer(fieldnames, geom_type, srid, srid_out=None):
    # Create temp file
    tmp = tempfile.NamedTemporaryFile(suffix='.shp', mode='w+b', dir=settings.TEMP_DIR)
    # we must close the file for GDAL to be able to open and write to it
    tmp.close()
    # create shape format

    dr = ogr.GetDriverByName('ESRI Shapefile')
    ds = dr.CreateDataSource(tmp.name)
    if ds is None:
        raise Exception('Could not create file!')

    ogr_type = OGRGeomType(geom_type).num

    native_srs = osr.SpatialReference()
    native_srs.ImportFromEPSG(srid)

    if srid_out:
        output_srs = osr.SpatialReference()
        output_srs.ImportFromEPSG(srid_out)
    else:
        output_srs = native_srs

    layer = ds.CreateLayer('lyr', srs=output_srs, geom_type=ogr_type)
    if layer is None:
        raise ValueError('Could not create layer (type=%s, srs=%s)' % (geom_type, output_srs))

    # Create other fields
    for fieldname in fieldnames:
        field_defn = ogr.FieldDefn(fieldname, ogr.OFTString)
        field_defn.SetWidth(255)
        if layer.CreateField(field_defn) != 0:
            raise Exception('Faild to create field')

    return tmp, layer, ds, native_srs, output_srs


def fieldmap_from_fields(model, fieldnames):
    from .serializers import plain_text
    return dict(
        (fname, lambda x, fname=fname: smart_str(plain_text(unicode(getattr(x, fname + '_csv_display',
                                                                    getattr(x, fname + '_display', getattr(x, fname)))))))
        for fname in fieldnames
    )


def fieldmap_from_model(model):
    """Extract all non geometry fields from model"""
    fields = model._meta.fields
    non_geometry_field_names = [f.name for f in fields if not isinstance(f, GeometryField)]

    return fieldmap_from_fields(model, non_geometry_field_names)


def geo_field_from_model(model, default_geo_field_name=None):
{'fraction_non_alphanumeric': 0.10201575769978512, 'fraction_numerical': 0.0019441317916709302, 'mean_word_length': 3.283085013146363, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3279853', 'n_tokens_mistral': 3454, 'n_tokens_neox': 3305, 'n_words': 750}