<commit_before>import os
import unittest
from unittest import skipUnless

from django.contrib.gis.gdal import HAS_GDAL

from ..test_data import TEST_DATA, TestDS, get_ds_file

if HAS_GDAL:
    from django.contrib.gis.gdal import DataSource, Envelope, OGRGeometry, GDALException, OGRIndexError, GDAL_VERSION
    from django.contrib.gis.gdal.field import OFTReal, OFTInteger, OFTString

    # List of acceptable data sources.
    ds_list = (
        TestDS(
            'test_point', nfeat=5, nfld=3, geom='POINT', gtype=1, driver='ESRI Shapefile',
            fields={'dbl': OFTReal, 'int': OFTInteger, 'str': OFTString},
            extent=(-1.35011, 0.166623, -0.524093, 0.824508),  # Got extent from QGIS
            srs_wkt=(
                'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_1984",'
                '6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",'
                '0.017453292519943295]]'
            ),
            field_values={
                'dbl': [float(i) for i in range(1, 6)],
                'int': list(range(1, 6)),
                'str': [str(i) for i in range(1, 6)],
            },
            fids=range(5)
        ),
        TestDS(
            'test_vrt', ext='vrt', nfeat=3, nfld=3, geom='POINT', gtype='Point25D',
            driver='OGR_VRT' if GDAL_VERSION >= (2, 0) else 'VRT',
            fields={
                'POINT_X': OFTString,
                'POINT_Y': OFTString,
                'NUM': OFTString,
            },  # VRT uses CSV, which all types are OFTString.
            extent=(1.0, 2.0, 100.0, 523.5),  # Min/Max from CSV
            field_values={
                'POINT_X': ['1.0', '5.0', '100.0'],
                'POINT_Y': ['2.0', '23.0', '523.5'],
                'NUM': ['5', '17', '23'],
            },
            fids=range(1, 4)
        ),
        TestDS(
            'test_poly', nfeat=3, nfld=3, geom='POLYGON', gtype=3,
            driver='ESRI Shapefile',
            fields={'float': OFTReal, 'int': OFTInteger, 'str': OFTString},
            extent=(-1.01513, -0.558245, 0.161876, 0.839637),  # Got extent from QGIS
            srs_wkt=(
                'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_1984",'
                '6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",'
                '0.017453292519943295]]'
            ),
        )
    )

bad_ds = (TestDS('foo'),)


@skipUnless(HAS_GDAL, "GDAL is required")
class DataSourceTest(unittest.TestCase):

    def test01_valid_shp(self):
        "Testing valid SHP Data Source files."

        for source in ds_list:
            # Loading up the data source
            ds = DataSource(source.ds)

            # Making sure the layer count is what's expected (only 1 layer in a SHP file)
            self.assertEqual(1, len(ds))

            # Making sure GetName works
            self.assertEqual(source.ds, ds.name)

            # Making sure the driver name matches up
            self.assertEqual(source.driver, str(ds.driver))

            # Making sure indexing works
            try:
                ds[len(ds)]
            except OGRIndexError:
                pass
            else:
                self.fail('Expected an IndexError!')

    def test02_invalid_shp(self):
        "Testing invalid SHP files for the Data Source."
        for source in bad_ds:
            with self.assertRaises(GDALException):
                DataSource(source.ds)

    def test03a_layers(self):
        "Testing Data Source Layers."
        for source in ds_list:
            ds = DataSource(source.ds)

            # Incrementing through each layer, this tests DataSource.__iter__
            for layer in ds:
                # Making sure we get the number of features we expect
                self.assertEqual(len(layer), source.nfeat)

                # Making sure we get the number of fields we expect
                self.assertEqual(source.nfld, layer.num_fields)
                self.assertEqual(source.nfld, len(layer.fields))

                # Testing the layer's extent (an Envelope), and its properties
                if source.driver == 'VRT' and (GDAL_VERSION >= (1, 7, 0) and GDAL_VERSION < (1, 7, 3)):
                    # There's a known GDAL regression with retrieving the extent
                    # of a VRT layer in versions 1.7.0-1.7.2:
                    #  http://trac.osgeo.org/gdal/ticket/3783
                    pass
                else:
                    self.assertIsInstance(layer.extent, Envelope)
                    self.assertAlmostEqual(source.extent[0], layer.extent.min_x, 5)
                    self.assertAlmostEqual(source.extent[1], layer.extent.min_y, 5)
                    self.assertAlmostEqual(source.extent[2], layer.extent.max_x, 5)
                    self.assertAlmostEqual(source.extent[3], layer.extent.max_y, 5)

                # Now checking the field names.
                flds = layer.fields
                for f in flds:
                    self.assertIn(f, source.fields)

                # Negative FIDs are not allowed.
                with self.assertRaises(OGRIndexError):
                    layer.__getitem__(-1)
                with self.assertRaises(OGRIndexError):
                    layer.__getitem__(50000)

                if hasattr(source, 'field_values'):
                    fld_names = source.field_values.keys()

                    # Testing `Layer.get_fields` (which uses Layer.__iter__)
                    for fld_name in fld_names:
                        self.assertEqual(source.field_values[fld_name], layer.get_fields(fld_name))

                    # Testing `Layer.__getitem__`.<commit_msg>Removed a test workaround for some old GDAL 1.7.x versions.<commit_after>import os
import unittest
from unittest import skipUnless

from django.contrib.gis.gdal import HAS_GDAL

from ..test_data import TEST_DATA, TestDS, get_ds_file

if HAS_GDAL:
    from django.contrib.gis.gdal import DataSource, Envelope, OGRGeometry, GDALException, OGRIndexError, GDAL_VERSION
    from django.contrib.gis.gdal.field import OFTReal, OFTInteger, OFTString

    # List of acceptable data sources.
    ds_list = (
        TestDS(
            'test_point', nfeat=5, nfld=3, geom='POINT', gtype=1, driver='ESRI Shapefile',
            fields={'dbl': OFTReal, 'int': OFTInteger, 'str': OFTString},
            extent=(-1.35011, 0.166623, -0.524093, 0.824508),  # Got extent from QGIS
            srs_wkt=(
                'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_1984",'
                '6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",'
                '0.017453292519943295]]'
            ),
            field_values={
                'dbl': [float(i) for i in range(1, 6)],
                'int': list(range(1, 6)),
                'str': [str(i) for i in range(1, 6)],
            },
            fids=range(5)
        ),
        TestDS(
            'test_vrt', ext='vrt', nfeat=3, nfld=3, geom='POINT', gtype='Point25D',
            driver='OGR_VRT' if GDAL_VERSION >= (2, 0) else 'VRT',
            fields={
                'POINT_X': OFTString,
                'POINT_Y': OFTString,
                'NUM': OFTString,
            },  # VRT uses CSV, which all types are OFTString.
            extent=(1.0, 2.0, 100.0, 523.5),  # Min/Max from CSV
            field_values={
                'POINT_X': ['1.0', '5.0', '100.0'],
                'POINT_Y': ['2.0', '23.0', '523.5'],
                'NUM': ['5', '17', '23'],
            },
            fids=range(1, 4)
        ),
        TestDS(
            'test_poly', nfeat=3, nfld=3, geom='POLYGON', gtype=3,
            driver='ESRI Shapefile',
            fields={'float': OFTReal, 'int': OFTInteger, 'str': OFTString},
            extent=(-1.01513, -0.558245, 0.161876, 0.839637),  # Got extent from QGIS
            srs_wkt=(
                'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_1984",'
                '6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",'
                '0.017453292519943295]]'
            ),
        )
    )

bad_ds = (TestDS('foo'),)


@skipUnless(HAS_GDAL, "GDAL is required")
class DataSourceTest(unittest.TestCase):

    def test01_valid_shp(self):
        "Testing valid SHP Data Source files."

        for source in ds_list:
            # Loading up the data source
            ds = DataSource(source.ds)

            # Making sure the layer count is what's expected (only 1 layer in a SHP file)
            self.assertEqual(1, len(ds))

            # Making sure GetName works
            self.assertEqual(source.ds, ds.name)

            # Making sure the driver name matches up
            self.assertEqual(source.driver, str(ds.driver))

            # Making sure indexing works
            try:
                ds[len(ds)]
            except OGRIndexError:
                pass
            else:
                self.fail('Expected an IndexError!')

    def test02_invalid_shp(self):
        "Testing invalid SHP files for the Data Source."
        for source in bad_ds:
            with self.assertRaises(GDALException):
                DataSource(source.ds)

    def test03a_layers(self):
        "Testing Data Source Layers."
        for source in ds_list:
            ds = DataSource(source.ds)

            # Incrementing through each layer, this tests DataSource.__iter__
            for layer in ds:
                # Making sure we get the number of features we expect
                self.assertEqual(len(layer), source.nfeat)

                # Making sure we get the number of fields we expect
                self.assertEqual(source.nfld, layer.num_fields)
                self.assertEqual(source.nfld, len(layer.fields))

                # Testing the layer's extent (an Envelope), and its properties
                self.assertIsInstance(layer.extent, Envelope)
                self.assertAlmostEqual(source.extent[0], layer.extent.min_x, 5)
                self.assertAlmostEqual(source.extent[1], layer.extent.min_y, 5)
                self.assertAlmostEqual(source.extent[2], layer.extent.max_x, 5)
                self.assertAlmostEqual(source.extent[3], layer.extent.max_y, 5)

                # Now checking the field names.
                flds = layer.fields
                for f in flds:
                    self.assertIn(f, source.fields)

                # Negative FIDs are not allowed.
                with self.assertRaises(OGRIndexError):
                    layer.__getitem__(-1)
                with self.assertRaises(OGRIndexError):
                    layer.__getitem__(50000)

                if hasattr(source, 'field_values'):
                    fld_names = source.field_values.keys()

                    # Testing `Layer.get_fields` (which uses Layer.__iter__)
                    for fld_name in fld_names:
                        self.assertEqual(source.field_values[fld_name], layer.get_fields(fld_name))

                    # Testing `Layer.__getitem__`.
{'fraction_non_alphanumeric': 0.11290322580645161, 'fraction_numerical': 0.0441042464005832, 'mean_word_length': 1.7876555753111507, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2256090', 'n_tokens_mistral': 3723, 'n_tokens_neox': 3273, 'n_words': 772}