<commit_before>import os
from io import StringIO
from unittest.mock import patch

from django import VERSION
from django.conf import settings
from django.contrib.gis import gdal
from django.contrib.gis.db.models import GeometryField
from django.http import HttpResponse
from django.test import TestCase
from django.test.utils import override_settings
from django.utils import translation

from mapentity.registry import app_settings
from mapentity.serializers import ZipShapeSerializer, CSVSerializer
from ..models import MushroomSpot, Tag, Supermarket


class CommonShapefileSerializerMixin(object):
    def getShapefileLayers(self):
        shapefiles = self.serializer.path_directory
        shapefiles = [shapefile for shapefile in os.listdir(shapefiles) if shapefile[-3:] == "shp"]
        datasources = [gdal.DataSource(os.path.join(self.serializer.path_directory, s)) for s in shapefiles]
        layers = [ds[0] for ds in datasources]
        return layers


class MushroomShapefileSerializerTest(CommonShapefileSerializerMixin, TestCase):
    def setUp(self):
        self.point1 = MushroomSpot.objects.create(serialized='SRID=%s;POINT(0 0)' % settings.SRID)
        self.point1.tags.add(Tag.objects.create(label="Tag1"))
        self.point1.tags.add(Tag.objects.create(label="Tag2"))
        self.line1 = MushroomSpot.objects.create(serialized='SRID=%s;LINESTRING(0 0, 10 0)' % settings.SRID)
        self.multipoint = MushroomSpot.objects.create(serialized='SRID=%s;MULTIPOINT((1 1), (2 2))' % settings.SRID)
        self.multiline = MushroomSpot.objects.create(serialized='SRID=%s;MULTILINESTRING((1 1, 2 2), '
                                                                '(3 3, 4 4))' % settings.SRID)
        self.polygon = MushroomSpot.objects.create(serialized='SRID=%s;POLYGON((1 1, 2 2, '
                                                              '1 2, 1 1))' % settings.SRID)
        self.multipolygon = MushroomSpot.objects.create(serialized='SRID=%s;MULTIPOLYGON(((1 1, 2 2, '
                                                                   '1 2, 1 1)))' % settings.SRID)
        self.geometrycollection = MushroomSpot.objects.create(
            serialized='SRID=%s;GEOMETRYCOLLECTION(POINT(0 0), POLYGON((1 1, 2 2, 1 2, 1 1))))' % settings.SRID)
        MushroomSpot.geomfield = GeometryField(name='geom', srid=settings.SRID)

        self.serializer = ZipShapeSerializer()
        response = HttpResponse()
        self.serializer.serialize(MushroomSpot.objects.all(), stream=response,
                                  fields=['id', 'name', 'number', 'size', 'boolean', 'tags'], delete=False)

    def test_serializer_no_geom(self):
        response = HttpResponse()
        MushroomSpot.objects.create()
        self.serializer.serialize(MushroomSpot.objects.all(), stream=response,
                                  fields=['id', 'name', 'number', 'size', 'boolean', 'tags'], delete=False)
        l_point, l_linestring, l_m_point, l_m_linestring, l_polygon, l_m_polygon = self.getShapefileLayers()
        self.assertEqual(len(l_point), 1)
        self.assertEqual(len(l_linestring), 1)
        self.assertEqual(len(l_m_point), 1)
        self.assertEqual(len(l_m_linestring), 1)
        self.assertEqual(len(l_polygon), 1)
        self.assertEqual(len(l_m_polygon), 1)

    def test_serializer_creates_one_layer_per_type(self):
        self.assertEqual(len(self.getShapefileLayers()), 6)

    def test_each_layer_has_records_by_type(self):
        l_point, l_linestring, l_m_point, l_m_linestring, l_polygon, l_m_polygon = self.getShapefileLayers()
        self.assertEqual(len(l_point), 2)
        self.assertEqual(len(l_linestring), 1)
        self.assertEqual(len(l_m_point), 1)
        self.assertEqual(len(l_m_linestring), 1)
        self.assertEqual(len(l_polygon), 2)
        self.assertEqual(len(l_m_polygon), 1)

    def test_each_layer_has_a_different_geometry_type(self):
        layer_types = [layer.geom_type.name for layer in self.getShapefileLayers()]

        self.assertCountEqual(layer_types, ['Polygon', 'LineString', 'LineString', 'Point', 'MultiPoint', 'Polygon'])

    def test_layer_has_right_projection(self):
        for layer in self.getShapefileLayers():
            self.assertIn(layer.srs.name, ('RGF_1993_Lambert_93', 'RGF93_Lambert_93', 'RGF93 / Lambert-93'))
            self.assertCountEqual(layer.fields, ['id', 'name', 'number', 'size', 'boolean', 'tags'])

    def test_geometries_come_from_records(self):
        layers = self.getShapefileLayers()
        geom_type_layer = {layer.name: layer for layer in layers}
        feature = geom_type_layer['Point'][0]
        self.assertEqual(str(feature['id']), str(self.point1.pk))
        self.assertTrue(feature.geom.geos.equals(self.point1.geom))

        feature = geom_type_layer['MultiPoint'][0]
        self.assertEqual(str(feature['id']), str(self.multipoint.pk))
        self.assertTrue(feature.geom.geos.equals(self.multipoint.geom))

        feature = geom_type_layer['LineString'][0]
        self.assertEqual(str(feature['id']), str(self.line1.pk))
        self.assertTrue(feature.geom.geos.equals(self.line1.geom))

        feature = geom_type_layer['MultiLineString'][0]
        self.assertEqual(str(feature['id']), str(self.multiline.pk))
        self.assertTrue(feature.geom.geos.equals(self.multiline.geom))

        feature = geom_type_layer['Polygon'][0]
        self.assertEqual(str(feature['id']), str(self.polygon.pk))
        self.assertTrue(feature.geom.geos.equals(self.polygon.geom))

        feature = geom_type_layer['MultiPolygon'][0]
        self.assertEqual(str(feature['id']), str(self.multipolygon.pk))
        self.assertTrue(feature.geom.geos.equals(self.multipolygon.geom))

    def test_attributes(self):
        l_point, l_linestring, l_m_point, l_m_linestring, l_polygon, l_m_polygon = self.getShapefileLayers()
        feature = l_point[0]
        self.assertEqual(feature['name'].value, self.point1.name)


class NoGeomShapefileSerializerTest(CommonShapefileSerializerMixin, TestCase):
    def setUp(self):
        self.market = Tag.objects.create(label="Label")

    def test_multiple_geoms(self):
        self.serializer = ZipShapeSerializer()<commit_msg>Fix tests shapefiles unconsistent<commit_after>import os
from io import StringIO
from unittest.mock import patch

from django import VERSION
from django.conf import settings
from django.contrib.gis import gdal
from django.contrib.gis.db.models import GeometryField
from django.http import HttpResponse
from django.test import TestCase
from django.test.utils import override_settings
from django.utils import translation

from mapentity.registry import app_settings
from mapentity.serializers import ZipShapeSerializer, CSVSerializer
from ..models import MushroomSpot, Tag, Supermarket


class CommonShapefileSerializerMixin(object):
    def getShapefileLayers(self):
        shapefiles = self.serializer.path_directory
        shapefiles = [shapefile for shapefile in os.listdir(shapefiles) if shapefile[-3:] == "shp"]
        layers = {
            s: gdal.DataSource(os.path.join(self.serializer.path_directory, s))[0] for s in shapefiles
        }
        return layers


class MushroomShapefileSerializerTest(CommonShapefileSerializerMixin, TestCase):
    def setUp(self):
        self.point1 = MushroomSpot.objects.create(serialized='SRID=%s;POINT(0 0)' % settings.SRID)
        self.point1.tags.add(Tag.objects.create(label="Tag1"))
        self.point1.tags.add(Tag.objects.create(label="Tag2"))
        self.line1 = MushroomSpot.objects.create(serialized='SRID=%s;LINESTRING(0 0, 10 0)' % settings.SRID)
        self.multipoint = MushroomSpot.objects.create(serialized='SRID=%s;MULTIPOINT((1 1), (2 2))' % settings.SRID)
        self.multiline = MushroomSpot.objects.create(serialized='SRID=%s;MULTILINESTRING((1 1, 2 2), '
                                                                '(3 3, 4 4))' % settings.SRID)
        self.polygon = MushroomSpot.objects.create(serialized='SRID=%s;POLYGON((1 1, 2 2, '
                                                              '1 2, 1 1))' % settings.SRID)
        self.multipolygon = MushroomSpot.objects.create(serialized='SRID=%s;MULTIPOLYGON(((1 1, 2 2, '
                                                                   '1 2, 1 1)))' % settings.SRID)
        self.geometrycollection = MushroomSpot.objects.create(
            serialized='SRID=%s;GEOMETRYCOLLECTION(POINT(0 0), POLYGON((1 1, 2 2, 1 2, 1 1))))' % settings.SRID)
        MushroomSpot.geomfield = GeometryField(name='geom', srid=settings.SRID)

        self.serializer = ZipShapeSerializer()
        response = HttpResponse()
        self.serializer.serialize(MushroomSpot.objects.all(), stream=response,
                                  fields=['id', 'name', 'number', 'size', 'boolean', 'tags'], delete=False)

    def test_serializer_no_geom(self):
        response = HttpResponse()
        MushroomSpot.objects.create()
        self.serializer.serialize(MushroomSpot.objects.all(), stream=response,
                                  fields=['id', 'name', 'number', 'size', 'boolean', 'tags'], delete=False)
        layers = self.getShapefileLayers()
        self.assertEqual(len(layers['Point.shp']), 1)
        self.assertEqual(len(layers['LineString.shp']), 1)
        self.assertEqual(len(layers['Polygon.shp']), 1)
        self.assertEqual(len(layers['MultiPoint.shp']), 2)
        self.assertEqual(len(layers['MultiLineString.shp']), 1)
        self.assertEqual(len(layers['MultiPolygon.shp']), 2)

    def test_serializer_creates_one_layer_per_type(self):
        self.assertEqual(len(self.getShapefileLayers()), 6)

    def test_each_layer_has_records_by_type(self):
        layers = self.getShapefileLayers()
        self.assertEqual(len(layers['Point.shp']), 1)
        self.assertEqual(len(layers['LineString.shp']), 1)
        self.assertEqual(len(layers['Polygon.shp']), 1)
        self.assertEqual(len(layers['MultiPoint.shp']), 2)
        self.assertEqual(len(layers['MultiLineString.shp']), 1)
        self.assertEqual(len(layers['MultiPolygon.shp']), 2)

    def test_each_layer_has_a_different_geometry_type(self):
        layer_types = [layer.geom_type.name for layer in self.getShapefileLayers().values()]

        self.assertCountEqual(layer_types, ['Polygon', 'LineString', 'LineString', 'Point', 'MultiPoint', 'Polygon'])

    def test_layer_has_right_projection(self):
        for layer in self.getShapefileLayers().values():
            self.assertIn(layer.srs.name, ('RGF_1993_Lambert_93', 'RGF93_Lambert_93', 'RGF93 / Lambert-93'))
            self.assertCountEqual(layer.fields, ['id', 'name', 'number', 'size', 'boolean', 'tags'])

    def test_geometries_come_from_records(self):
        layers = self.getShapefileLayers().values()
        geom_type_layer = {layer.name: layer for layer in layers}
        feature = geom_type_layer['Point'][0]
        self.assertEqual(str(feature['id']), str(self.point1.pk))
        self.assertTrue(feature.geom.geos.equals(self.point1.geom))

        feature = geom_type_layer['MultiPoint'][0]
        self.assertEqual(str(feature['id']), str(self.multipoint.pk))
        self.assertTrue(feature.geom.geos.equals(self.multipoint.geom))

        feature = geom_type_layer['LineString'][0]
        self.assertEqual(str(feature['id']), str(self.line1.pk))
        self.assertTrue(feature.geom.geos.equals(self.line1.geom))

        feature = geom_type_layer['MultiLineString'][0]
        self.assertEqual(str(feature['id']), str(self.multiline.pk))
        self.assertTrue(feature.geom.geos.equals(self.multiline.geom))

        feature = geom_type_layer['Polygon'][0]
        self.assertEqual(str(feature['id']), str(self.polygon.pk))
        self.assertTrue(feature.geom.geos.equals(self.polygon.geom))

        feature = geom_type_layer['MultiPolygon'][0]
        self.assertEqual(str(feature['id']), str(self.multipolygon.pk))
        self.assertTrue(feature.geom.geos.equals(self.multipolygon.geom))

    def test_attributes(self):
        l_point = self.getShapefileLayers()['Point.shp']
        feature = l_point[0]
        self.assertEqual(feature['name'].value, self.point1.name)


class NoGeomShapefileSerializerTest(CommonShapefileSerializerMixin, TestCase):
    def setUp(self):
        self.market = Tag.objects.create(label="Label")

    def test_multiple_geoms(self):
        self.serializer = ZipShapeSerializer()
{'fraction_non_alphanumeric': 0.1295328455415293, 'fraction_numerical': 0.014794564605612286, 'mean_word_length': 3.3842086711314767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3204394', 'n_tokens_mistral': 4148, 'n_tokens_neox': 3850, 'n_words': 632}