<commit_before>from collections import defaultdict
import logging
import psycopg2.extras
import time
from datapunt_geosearch.config import (
    DATAPUNT_API_URL,
    DSN_VARIOUS_SMALL_DATASETS,
)
from datapunt_geosearch.db import dbconnection
from datapunt_geosearch.exceptions import DataSourceException

_logger = logging.getLogger(__name__)


class DatasetRegistry:
    """
    Dataset Registry.
    """

    INITIALIZE_DELAY = 600  # 10 minutes

    def __init__(self, delay=None):
        # Datasets is a dictionary of DSN => Datasets.
        self.datasets = defaultdict(list)
        self.providers = dict()
        self.static_dataset_names = []
        self._datasets_initialized = None
        if delay is not None:
            self.INITIALIZE_DELAY = delay

    def register_dataset(self, dsn_name, dataset_class):
        self.datasets[dsn_name].append(dataset_class)

        for key in dataset_class.metadata["datasets"].keys():
            self.providers[key] = dataset_class
            for item in dataset_class.metadata["datasets"][key].keys():
                if (
                    item in self.providers.keys()
                    and self.providers[item] != dataset_class
                ):
                    _logger.debug(
                        "Provider for {} already defined {} and will be overwritten by {}.".format(
                            item, self.providers[item], dataset_class
                        )
                    )
                self.providers[item] = dataset_class

    def register_external_dataset(self, name, base_url, path, field_mapping=None):
        from datapunt_geosearch.datasource import ExternalDataSource
        class_name = "{}ExternalDataSource".format(name.upper())

        meta = {
            "base_url": base_url,
            "datasets": {
                name: {
                    name: path
                }
            }
        }
        if field_mapping is not None:
            meta["field_mapping"] = field_mapping

        dataset_class = type(
            class_name,
            (ExternalDataSource,),
            dict(metadata=meta)
        )

        self.register_dataset("EXT_{}".format(name.upper()), dataset_class)
        return dataset_class

    def get_all_datasets(self):
        self.init_datasets()
        return self.providers

    def get_all_dataset_names(self):
        return self.get_all_datasets().keys()

    def get_by_name(self, name):
        return self.get_all_datasets().get(name)

    def filter_datasets(self, names=None, scopes=None):
        return set(
            [
                dataset
                for name, dataset in self.get_all_datasets().items()
                if name in names and dataset.check_scopes(scopes=scopes)
            ]
        )

    def init_dataset(self, row, class_name, dsn_name, scopes=None):
        """
        Initialize dataset class and register it in registry based on row data

        Args:
          row (dict): Dictionary with keys:
            - schema,
            - name,
            - name_field,
            - table_name,
            - geometry_field,
            - geometry_type,
            - id_field
            - dataset_name
          class_name (str): Name for the new class
          dsn_name: DSN name for namespacing
          scopes: Optional comma separated list of Authentication scopes for dataset.

        Returns:
          DataSourceBase subclass for given dataset.
        """
        from datapunt_geosearch.datasource import DataSourceBase

        if row.get("schema") is None:
            row["schema"] = "public"
        schema_table = row["schema"] + "." + row["table_name"]

        if row["geometry_type"] and row["geometry_type"].upper() in ["POLYGON"]:
            operator = "contains"
        else:
            operator = "within"
        name = row["name"]
        name_field = row["name_field"]
        dataset_name = row["dataset_name"]
        geometry_field = row["geometry_field"]
        id_field = row["id_field"]

        dataset_class = type(
            class_name,
            (DataSourceBase,),
            {
                "metadata": {
                    "geofield": geometry_field,<commit_msg>Also use contains for MULTIPOLYGONs (#18)<commit_after>from collections import defaultdict
import logging
import psycopg2.extras
import time
from datapunt_geosearch.config import (
    DATAPUNT_API_URL,
    DSN_VARIOUS_SMALL_DATASETS,
)
from datapunt_geosearch.db import dbconnection
from datapunt_geosearch.exceptions import DataSourceException

_logger = logging.getLogger(__name__)


class DatasetRegistry:
    """
    Dataset Registry.
    """

    INITIALIZE_DELAY = 600  # 10 minutes

    def __init__(self, delay=None):
        # Datasets is a dictionary of DSN => Datasets.
        self.datasets = defaultdict(list)
        self.providers = dict()
        self.static_dataset_names = []
        self._datasets_initialized = None
        if delay is not None:
            self.INITIALIZE_DELAY = delay

    def register_dataset(self, dsn_name, dataset_class):
        self.datasets[dsn_name].append(dataset_class)

        for key in dataset_class.metadata["datasets"].keys():
            self.providers[key] = dataset_class
            for item in dataset_class.metadata["datasets"][key].keys():
                if (
                    item in self.providers.keys()
                    and self.providers[item] != dataset_class
                ):
                    _logger.debug(
                        "Provider for {} already defined {} and will be overwritten by {}.".format(
                            item, self.providers[item], dataset_class
                        )
                    )
                self.providers[item] = dataset_class

    def register_external_dataset(self, name, base_url, path, field_mapping=None):
        from datapunt_geosearch.datasource import ExternalDataSource
        class_name = "{}ExternalDataSource".format(name.upper())

        meta = {
            "base_url": base_url,
            "datasets": {
                name: {
                    name: path
                }
            }
        }
        if field_mapping is not None:
            meta["field_mapping"] = field_mapping

        dataset_class = type(
            class_name,
            (ExternalDataSource,),
            dict(metadata=meta)
        )

        self.register_dataset("EXT_{}".format(name.upper()), dataset_class)
        return dataset_class

    def get_all_datasets(self):
        self.init_datasets()
        return self.providers

    def get_all_dataset_names(self):
        return self.get_all_datasets().keys()

    def get_by_name(self, name):
        return self.get_all_datasets().get(name)

    def filter_datasets(self, names=None, scopes=None):
        return set(
            [
                dataset
                for name, dataset in self.get_all_datasets().items()
                if name in names and dataset.check_scopes(scopes=scopes)
            ]
        )

    def init_dataset(self, row, class_name, dsn_name, scopes=None):
        """
        Initialize dataset class and register it in registry based on row data

        Args:
          row (dict): Dictionary with keys:
            - schema,
            - name,
            - name_field,
            - table_name,
            - geometry_field,
            - geometry_type,
            - id_field
            - dataset_name
          class_name (str): Name for the new class
          dsn_name: DSN name for namespacing
          scopes: Optional comma separated list of Authentication scopes for dataset.

        Returns:
          DataSourceBase subclass for given dataset.
        """
        from datapunt_geosearch.datasource import DataSourceBase

        if row.get("schema") is None:
            row["schema"] = "public"
        schema_table = row["schema"] + "." + row["table_name"]

        if row["geometry_type"] and row["geometry_type"].upper() in ["POLYGON", "MULTIPOLYGON"]:
            operator = "contains"
        else:
            operator = "within"
        name = row["name"]
        name_field = row["name_field"]
        dataset_name = row["dataset_name"]
        geometry_field = row["geometry_field"]
        id_field = row["id_field"]

        dataset_class = type(
            class_name,
            (DataSourceBase,),
            {
                "metadata": {
                    "geofield": geometry_field,
{'fraction_non_alphanumeric': 0.0880547686496695, 'fraction_numerical': 0.0016525023607176581, 'mean_word_length': 1.8692854724009482, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6192588', 'n_tokens_mistral': 2365, 'n_tokens_neox': 2261, 'n_words': 531}