<commit_before># This file is part of the MapProxy project.
# Copyright (C) 2011 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import with_statement

import os
import sys
import time
import operator

from mapproxy.config import abspath
from mapproxy.config.loader import ConfigurationError
from mapproxy.config.coverage import load_coverage
from mapproxy.srs import SRS
from mapproxy.util.py import memoize
from mapproxy.util.times import timestamp_from_isodate, timestamp_before
from mapproxy.util.coverage import MultiCoverage, BBOXCoverage, GeomCoverage
from mapproxy.util.yaml import load_yaml_file, YAMLError
from mapproxy.seed.util import bidict
from mapproxy.seed.seeder import SeedTask, CleanupTask
from mapproxy.seed.spec import validate_seed_conf


class SeedConfigurationError(ConfigurationError):
    pass


import logging
log = logging.getLogger('mapproxy.seed.config')

def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf):
    try:
        conf = load_yaml_file(seed_conf_filename)
    except YAMLError, ex:
        raise SeedConfigurationError(ex)

    if 'views' in conf:
        # TODO: deprecate old config
        seed_conf = LegacySeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
    else:
        errors, informal_only = validate_seed_conf(conf)
        for error in errors:
            log.warn(error)
        if not informal_only:
            raise SeedConfigurationError('invalid configuration')
        seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
    return seed_conf

class LegacySeedingConfiguration(object):
    """
    Read old seed.yaml configuration (with seed and views).
    """
    def __init__(self, seed_conf, mapproxy_conf):
        self.conf = seed_conf
        self.mapproxy_conf = mapproxy_conf
        self.grids = bidict((name, grid_conf.tile_grid()) for name, grid_conf in self.mapproxy_conf.grids.iteritems())
        self.seed_tasks = []
        self.cleanup_tasks = []
        self._init_tasks()

    def _init_tasks(self):
        for cache_name, options in self.conf['seeds'].iteritems():
            remove_before = None
            if 'remove_before' in options:
                remove_before = before_timestamp_from_options(options['remove_before'])
            try:
                caches = self.mapproxy_conf.caches[cache_name].caches()
            except KeyError:
                print >>sys.stderr, 'error: cache %s not found. available caches: %s' % (
                    cache_name, ','.join(self.mapproxy_conf.caches.keys()))
                return
            caches = dict((grid, tile_mgr) for grid, extent, tile_mgr in caches)
            for view in options['views']:
                view_conf = self.conf['views'][view]
                coverage = load_coverage(view_conf)

                cache_srs = view_conf.get('srs', None)
                if cache_srs is not None:
                    cache_srs = [SRS(s) for s in cache_srs]

                level = view_conf.get('level', None)
                assert len(level) == 2

                for grid, tile_mgr in caches.iteritems():
                    if cache_srs and grid.srs not in cache_srs: continue
                    md = dict(name=view, cache_name=cache_name, grid_name=self.grids[grid])
                    levels = range(level[0], level[1]+1)
                    if coverage:
                        if isinstance(coverage, GeomCoverage) and coverage.geom.is_empty:
                            continue
                        seed_coverage = coverage.transform_to(grid.srs)
                    else:
                        seed_coverage = BBOXCoverage(grid.bbox, grid.srs)

                    self.seed_tasks.append(SeedTask(md, tile_mgr, levels, remove_before, seed_coverage))

                    if remove_before:
                        levels = range(grid.levels)
                        complete_extent = bool(coverage)
                        self.cleanup_tasks.append(CleanupTask(md, tile_mgr, levels, remove_before,
                            seed_coverage, complete_extent=complete_extent))

    def seed_tasks_names(self):
        return self.conf['seeds'].keys()

    def cleanup_tasks_names(self):
        return self.conf['seeds'].keys()

    def seeds(self, names=None):
        if names is None:
            return self.seed_tasks
        else:
            return [t for t in self.seed_tasks if t.md['name'] in names]

    def cleanups(self, names=None):
        if names is None:
            return self.cleanup_tasks
        else:
            return [t for t in self.cleanup_tasks if t.md['name'] in names]

class SeedingConfiguration(object):
    def __init__(self, seed_conf, mapproxy_conf):
        self.conf = seed_conf
        self.mapproxy_conf = mapproxy_conf
        self.grids = bidict((name, grid_conf.tile_grid()) for name, grid_conf in self.mapproxy_conf.grids.iteritems())

    @memoize
    def coverage(self, name):
        coverage_conf = self.conf.get('coverages', {}).get(name)
        if coverage_conf is None:
            raise ValueError('no coverage %s configured' % name)

        return load_coverage(coverage_conf)

    def cache(self, cache_name):
        cache = {}
        if cache_name not in self.mapproxy_conf.caches:
            raise SeedConfigurationError('cache %s not found. available caches: %s' % (
                cache_name, ','.join(self.mapproxy_conf.caches.keys())))
        for tile_grid, extent, tile_mgr in self.mapproxy_conf.caches[cache_name].caches():
            grid_name = self.grids[tile_grid]
            cache[grid_name] = tile_mgr
        return cache

    def seed_tasks_names(self):
        seeds = self.conf.get('seeds', {})
        if seeds:
            return seeds.keys()<commit_msg>print available coverages when referencing missing coverage in seed conf<commit_after># This file is part of the MapProxy project.
# Copyright (C) 2011 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import with_statement

import os
import sys
import time
import operator

from mapproxy.config import abspath
from mapproxy.config.loader import ConfigurationError
from mapproxy.config.coverage import load_coverage
from mapproxy.srs import SRS
from mapproxy.util.py import memoize
from mapproxy.util.times import timestamp_from_isodate, timestamp_before
from mapproxy.util.coverage import MultiCoverage, BBOXCoverage, GeomCoverage
from mapproxy.util.yaml import load_yaml_file, YAMLError
from mapproxy.seed.util import bidict
from mapproxy.seed.seeder import SeedTask, CleanupTask
from mapproxy.seed.spec import validate_seed_conf


class SeedConfigurationError(ConfigurationError):
    pass


import logging
log = logging.getLogger('mapproxy.seed.config')

def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf):
    try:
        conf = load_yaml_file(seed_conf_filename)
    except YAMLError, ex:
        raise SeedConfigurationError(ex)

    if 'views' in conf:
        # TODO: deprecate old config
        seed_conf = LegacySeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
    else:
        errors, informal_only = validate_seed_conf(conf)
        for error in errors:
            log.warn(error)
        if not informal_only:
            raise SeedConfigurationError('invalid configuration')
        seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
    return seed_conf

class LegacySeedingConfiguration(object):
    """
    Read old seed.yaml configuration (with seed and views).
    """
    def __init__(self, seed_conf, mapproxy_conf):
        self.conf = seed_conf
        self.mapproxy_conf = mapproxy_conf
        self.grids = bidict((name, grid_conf.tile_grid()) for name, grid_conf in self.mapproxy_conf.grids.iteritems())
        self.seed_tasks = []
        self.cleanup_tasks = []
        self._init_tasks()

    def _init_tasks(self):
        for cache_name, options in self.conf['seeds'].iteritems():
            remove_before = None
            if 'remove_before' in options:
                remove_before = before_timestamp_from_options(options['remove_before'])
            try:
                caches = self.mapproxy_conf.caches[cache_name].caches()
            except KeyError:
                print >>sys.stderr, 'error: cache %s not found. available caches: %s' % (
                    cache_name, ','.join(self.mapproxy_conf.caches.keys()))
                return
            caches = dict((grid, tile_mgr) for grid, extent, tile_mgr in caches)
            for view in options['views']:
                view_conf = self.conf['views'][view]
                coverage = load_coverage(view_conf)

                cache_srs = view_conf.get('srs', None)
                if cache_srs is not None:
                    cache_srs = [SRS(s) for s in cache_srs]

                level = view_conf.get('level', None)
                assert len(level) == 2

                for grid, tile_mgr in caches.iteritems():
                    if cache_srs and grid.srs not in cache_srs: continue
                    md = dict(name=view, cache_name=cache_name, grid_name=self.grids[grid])
                    levels = range(level[0], level[1]+1)
                    if coverage:
                        if isinstance(coverage, GeomCoverage) and coverage.geom.is_empty:
                            continue
                        seed_coverage = coverage.transform_to(grid.srs)
                    else:
                        seed_coverage = BBOXCoverage(grid.bbox, grid.srs)

                    self.seed_tasks.append(SeedTask(md, tile_mgr, levels, remove_before, seed_coverage))

                    if remove_before:
                        levels = range(grid.levels)
                        complete_extent = bool(coverage)
                        self.cleanup_tasks.append(CleanupTask(md, tile_mgr, levels, remove_before,
                            seed_coverage, complete_extent=complete_extent))

    def seed_tasks_names(self):
        return self.conf['seeds'].keys()

    def cleanup_tasks_names(self):
        return self.conf['seeds'].keys()

    def seeds(self, names=None):
        if names is None:
            return self.seed_tasks
        else:
            return [t for t in self.seed_tasks if t.md['name'] in names]

    def cleanups(self, names=None):
        if names is None:
            return self.cleanup_tasks
        else:
            return [t for t in self.cleanup_tasks if t.md['name'] in names]

class SeedingConfiguration(object):
    def __init__(self, seed_conf, mapproxy_conf):
        self.conf = seed_conf
        self.mapproxy_conf = mapproxy_conf
        self.grids = bidict((name, grid_conf.tile_grid()) for name, grid_conf in self.mapproxy_conf.grids.iteritems())

    @memoize
    def coverage(self, name):
        coverage_conf = (self.conf.get('coverages') or {}).get(name)
        if coverage_conf is None:
            raise SeedConfigurationError('coverage %s not found. available coverages: %s' % (
                name, ','.join((self.conf.get('coverages') or {}).keys())))

        return load_coverage(coverage_conf)

    def cache(self, cache_name):
        cache = {}
        if cache_name not in self.mapproxy_conf.caches:
            raise SeedConfigurationError('cache %s not found. available caches: %s' % (
                cache_name, ','.join(self.mapproxy_conf.caches.keys())))
        for tile_grid, extent, tile_mgr in self.mapproxy_conf.caches[cache_name].caches():
            grid_name = self.grids[tile_grid]
            cache[grid_name] = tile_mgr
        return cache

    def seed_tasks_names(self):
        seeds = self.conf.get('seeds', {})
        if seeds:
            return seeds.keys()
{'fraction_non_alphanumeric': 0.0862936183590066, 'fraction_numerical': 0.001886199308393587, 'mean_word_length': 2.504544202698981, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3675386', 'n_tokens_mistral': 3667, 'n_tokens_neox': 3636, 'n_words': 967}