<commit_before>from __future__ import absolute_import

from datetime import datetime
from mapproxy.script.conf.app import config_command
from mapproxy.seed.seeder import seed
from mapproxy.seed.config import SeedingConfiguration, SeedConfigurationError
from mapproxy.seed.spec import validate_seed_conf
from mapproxy.config.loader import ProxyConfiguration, ConfigurationError, validate_references
from mapproxy.config.spec import validate_options

from mapproxy.config.config import load_config, base_config, load_default_config
from mapproxy.seed import seeder
from mapproxy.seed.util import ProgressLog
from django.conf import settings
import yaml
from django.core.files.temp import NamedTemporaryFile
import logging
from django.db import connections
import requests
from pysqlite2 import dbapi2 as sqlite3
from .geopackage import (get_tile_table_names, get_zoom_levels_table,
                         get_table_tile_matrix_information, set_gpkg_contents_bounds)

logger = logging.getLogger(__name__)

class CustomLogger(ProgressLog):

    def __init__(self, task_uid=None, *args, **kwargs):

        self.task_uid = task_uid
        super(CustomLogger, self).__init__(*args, **kwargs)
        # Log mapproxy status but allow a setting to reduce database writes.
        self.log_step_step = 1
        self.log_step_counter = self.log_step_step

    def log_step(self, progress):
        from ..tasks.export_tasks import update_progress
        if self.task_uid:
            if self.log_step_counter == 0:
                update_progress(self.task_uid, progress=progress.progress * 100)
                self.log_step_counter = self.log_step_step
            self.log_step_counter -= 1
        super(CustomLogger, self).log_step(progress)


class ExternalRasterServiceToGeopackage(object):
    """
    Convert a External service to a geopackage.
    """

    def __init__(self, config=None, gpkgfile=None, bbox=None, service_url=None, layer=None, debug=None, name=None,
                 level_from=None, level_to=None, service_type=None, task_uid=None, selection=None):
        """
        Initialize the ExternalServiceToGeopackage utility.

        Args:
            gpkgfile: where to write the gpkg output
            debug: turn debugging on / off
        """
        self.gpkgfile = gpkgfile
        self.bbox = bbox
        self.service_url = service_url
        self.debug = debug
        self.name = name
        self.level_from = level_from
        self.level_to = level_to
        self.layer = layer
        self.config = config
        self.service_type = service_type
        self.task_uid = task_uid
        self.selection = selection

    def build_config(self):
        pass

    def get_check_config(self):
        """
        Create a MapProxy configuration object and verifies its validity
        """
        if self.config:
            conf_dict = yaml.load(self.config)
        else:
            conf_dict = create_conf_from_url(self.service_url)

        if not conf_dict.get('grids'):
            conf_dict['grids'] = {'geodetic': {'srs': 'EPSG:4326',
                                               'tile_size': [256, 256],
                                               'origin': 'nw'},
                                  'webmercator': {'srs': 'EPSG:3857',
                                                  'tile_size': [256, 256],
                                                  'origin': 'nw'}}

        # If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
        conf_dict['caches'] = conf_dict.get('caches', {})
        try:
            conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile
        except KeyError:
            conf_dict['caches']['cache'] = get_cache_template(["{0}_{1}".format(self.layer, self.service_type)],
                                                              [grids for grids in conf_dict.get('grids')],
                                                              self.gpkgfile, table_name=self.layer)<commit_msg>Remove MapProxy autoconfig from external service setup<commit_after>from __future__ import absolute_import

from datetime import datetime
from mapproxy.script.conf.app import config_command
from mapproxy.seed.seeder import seed
from mapproxy.seed.config import SeedingConfiguration, SeedConfigurationError
from mapproxy.seed.spec import validate_seed_conf
from mapproxy.config.loader import ProxyConfiguration, ConfigurationError, validate_references
from mapproxy.config.spec import validate_options

from mapproxy.config.config import load_config, base_config, load_default_config
from mapproxy.seed import seeder
from mapproxy.seed.util import ProgressLog
from django.conf import settings
import yaml
from django.core.files.temp import NamedTemporaryFile
import logging
from django.db import connections
import requests
from pysqlite2 import dbapi2 as sqlite3
from .geopackage import (get_tile_table_names, get_zoom_levels_table,
                         get_table_tile_matrix_information, set_gpkg_contents_bounds)

logger = logging.getLogger(__name__)

class CustomLogger(ProgressLog):

    def __init__(self, task_uid=None, *args, **kwargs):

        self.task_uid = task_uid
        super(CustomLogger, self).__init__(*args, **kwargs)
        # Log mapproxy status but allow a setting to reduce database writes.
        self.log_step_step = 1
        self.log_step_counter = self.log_step_step

    def log_step(self, progress):
        from ..tasks.export_tasks import update_progress
        if self.task_uid:
            if self.log_step_counter == 0:
                update_progress(self.task_uid, progress=progress.progress * 100)
                self.log_step_counter = self.log_step_step
            self.log_step_counter -= 1
        super(CustomLogger, self).log_step(progress)


class ExternalRasterServiceToGeopackage(object):
    """
    Convert a External service to a geopackage.
    """

    def __init__(self, config=None, gpkgfile=None, bbox=None, service_url=None, layer=None, debug=None, name=None,
                 level_from=None, level_to=None, service_type=None, task_uid=None, selection=None):
        """
        Initialize the ExternalServiceToGeopackage utility.

        Args:
            gpkgfile: where to write the gpkg output
            debug: turn debugging on / off
        """
        self.gpkgfile = gpkgfile
        self.bbox = bbox
        self.service_url = service_url
        self.debug = debug
        self.name = name
        self.level_from = level_from
        self.level_to = level_to
        self.layer = layer
        self.config = config
        self.service_type = service_type
        self.task_uid = task_uid
        self.selection = selection

    def build_config(self):
        pass

    def get_check_config(self):
        """
        Create a MapProxy configuration object and verifies its validity
        """
        if self.config:
            conf_dict = yaml.load(self.config)
        else:
            raise ConfigurationError("Configuration is required for raster data providers")

        if not conf_dict.get('grids'):
            conf_dict['grids'] = {'geodetic': {'srs': 'EPSG:4326',
                                               'tile_size': [256, 256],
                                               'origin': 'nw'},
                                  'webmercator': {'srs': 'EPSG:3857',
                                                  'tile_size': [256, 256],
                                                  'origin': 'nw'}}

        # If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
        conf_dict['caches'] = conf_dict.get('caches', {})
        try:
            conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile
        except KeyError:
            conf_dict['caches']['cache'] = get_cache_template(["{0}_{1}".format(self.layer, self.service_type)],
                                                              [grids for grids in conf_dict.get('grids')],
                                                              self.gpkgfile, table_name=self.layer)
{'fraction_non_alphanumeric': 0.0823327615780446, 'fraction_numerical': 0.007596177407498162, 'mean_word_length': 2.2822677925211097, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6947008', 'n_tokens_mistral': 2225, 'n_tokens_neox': 2141, 'n_words': 551}