<commit_before># This file is part of the GBI project.
# Copyright (C) 2012 Omniscale GmbH & Co. KG <http://omniscale.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import time
import sys

import requests
from cStringIO import StringIO

from mapproxy.cache.couchdb import CouchDBCache, CouchDBMDTemplate
from mapproxy.cache.mbtiles import MBTilesCache
from mapproxy.cache.tile import TileManager
from mapproxy.client.http import HTTPClient, HTTPClientError, log_request
from mapproxy.client.tile import TileClient, TileURLTemplate
from mapproxy.grid import tile_grid
from mapproxy.image import ImageSource
from mapproxy.image.opts import ImageOptions
from mapproxy.layer import CacheMapLayer
from mapproxy.seed.seeder import SeedTask, SeedProgress as SeedProgress_
from mapproxy.source import DummySource
from mapproxy.source.tile import TiledSource
from mapproxy.tilefilter import watermark_filter
from mapproxy.util import reraise_exception
from mapproxy.util.coverage import BBOXCoverage

from .coverage import coverage_from_geojson, coverage_intersection

DEFAULT_GRID = tile_grid(3857, origin='nw', name='GoogleMapsCompatible')

class SeedProgress(SeedProgress_):
    def __init__(self, terminate_event, *args, **kw):
        SeedProgress_.__init__(self, *args, **kw)
        self.terminate_event = terminate_event

    def running(self):
        return not self.terminate_event.is_set()

class ProgressLog(object):
    def __init__(self, task, app_state, task_cls):
        self.app_state = app_state
        self.task_id = task.id
        self.previous_tiles = task.tiles or 0
        self.task_cls = task_cls

    def log_step(self, progress):
        pass

    def log_progress(self, progress, level, bbox, tiles):
        session = self.app_state.user_db_session()
        query = session.query(self.task_cls)
        task = query.filter(self.task_cls.id == self.task_id).one()

        task.seed_progress = progress_identifier(progress.current_progress_identifier())
        task.progress = progress.progress
        task.tiles = self.previous_tiles + tiles

        session.commit()

def progress_identifier(level_progresses):
    """
    >>> progress_identifier([(0, 1)])
    '0-1'
    >>> progress_identifier([(0, 1), (2, 4)])
    '0-1|2-4'
    """
    return '|'.join('%d-%d' % lvl for lvl in level_progresses)

def parse_progress_identifier(identifier):
    """
    >>> parse_progress_identifier('')
    None
    >>> parse_progress_identifier('0-1')
    [(0, 1)]
    >>> parse_progress_identifier('0-1|2-4')
    [(0, 1), (2, 4)]
    """
    if not identifier:
        return None
    levels = []
    for level in identifier.split('|'):
        level = level.split('-')
        levels.append((int(level[0]), int(level[1])))
    return levels

class RequestsHTTPClient(object):
    """
    MapProxy compatible HTTPClient implementation that uses the
    session function of the requests lib to get keep-alive connections.
    Only supports GET requests, so it might only be used for readonly CouchDB access.
    """
    def __init__(self, url=None, username=None, password=<PASSWORD>, insecure=False,
                 ssl_ca_certs=None, timeout=None, headers=None):
        self.req_session = requests.Session(timeout=timeout)

    def open(self, url, data=None):

        code = None
        result = None
        try:
            start_time = time.time()
            result = self.req_session.get(url)
        except (requests.exceptions.RequestException, Exception), e:
            reraise_exception(HTTPClientError('Internal HTTP error "%s": %r'
                                              % (url, e)), sys.exc_info())
        else:
            code = result.status_code
            if not (200 <= code < 300):
                reraise_exception(HTTPClientError('HTTP Error "%s": %d'
                % (url, code), response_code=code), sys.exc_info())

            if code == 204:
                raise HTTPClientError('HTTP Error "204 No Content"', response_code=204)
            return result
        finally:
            log_request(url, code, result, duration=time.time()-start_time, method='GET')

    def open_image(self, url, data=None):
        resp = self.open(url, data=data)
        if 'content-type' in resp.headers:
            if not resp.headers['content-type'].lower().startswith('image'):
                raise HTTPClientError('response is not an image: (%s)' % (resp.content))
        return ImageSource(StringIO(resp.content))


def create_source(raster_source, app_state):
    url = raster_source.url
    username = raster_source.username
    password = <PASSWORD>_source.password

    http_client = HTTPClient(url, username, password)
                            # , insecure=insecure,
                             # ssl_ca_certs=ssl_ca_certs, timeout=timeout,
                             # headers=headers)

    grid = DEFAULT_GRID
    image_opts = None
    coverage = coverage_from_geojson(raster_source.download_coverage)
    format = raster_source.format

    url_template = TileURLTemplate(
        '%s/%s/%s-%%(z)s-%%(x)s-%%(y)s/tile' % (url, raster_source.layer, raster_source.matrix_set),
        format=format)
    client = TileClient(url_template, http_client=http_client, grid=grid)

    if app_state.tilebox.is_running():
        port = app_state.config.get('tilebox', 'port')
        url_template = TileURLTemplate(
            'http://127.0.0.1:%d/%s/%s-%%(z)s-%%(x)s-%%(y)s/tile' % (port, raster_source.layer, raster_source.matrix_set),
            format=format)
        tilebox_client =  TileClient(url_template, http_client=RequestsHTTPClient(), grid=grid)
        client = FallbackTileClient(tilebox_client, client)<commit_msg>make sure WMTS url does not contain two slashes<commit_after># This file is part of the GBI project.
# Copyright (C) 2012 Omniscale GmbH & Co. KG <http://omniscale.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import time
import sys

import requests
from cStringIO import StringIO

from mapproxy.cache.couchdb import CouchDBCache, CouchDBMDTemplate
from mapproxy.cache.mbtiles import MBTilesCache
from mapproxy.cache.tile import TileManager
from mapproxy.client.http import HTTPClient, HTTPClientError, log_request
from mapproxy.client.tile import TileClient, TileURLTemplate
from mapproxy.grid import tile_grid
from mapproxy.image import ImageSource
from mapproxy.image.opts import ImageOptions
from mapproxy.layer import CacheMapLayer
from mapproxy.seed.seeder import SeedTask, SeedProgress as SeedProgress_
from mapproxy.source import DummySource
from mapproxy.source.tile import TiledSource
from mapproxy.tilefilter import watermark_filter
from mapproxy.util import reraise_exception
from mapproxy.util.coverage import BBOXCoverage

from .coverage import coverage_from_geojson, coverage_intersection

DEFAULT_GRID = tile_grid(3857, origin='nw', name='GoogleMapsCompatible')

class SeedProgress(SeedProgress_):
    def __init__(self, terminate_event, *args, **kw):
        SeedProgress_.__init__(self, *args, **kw)
        self.terminate_event = terminate_event

    def running(self):
        return not self.terminate_event.is_set()

class ProgressLog(object):
    def __init__(self, task, app_state, task_cls):
        self.app_state = app_state
        self.task_id = task.id
        self.previous_tiles = task.tiles or 0
        self.task_cls = task_cls

    def log_step(self, progress):
        pass

    def log_progress(self, progress, level, bbox, tiles):
        session = self.app_state.user_db_session()
        query = session.query(self.task_cls)
        task = query.filter(self.task_cls.id == self.task_id).one()

        task.seed_progress = progress_identifier(progress.current_progress_identifier())
        task.progress = progress.progress
        task.tiles = self.previous_tiles + tiles

        session.commit()

def progress_identifier(level_progresses):
    """
    >>> progress_identifier([(0, 1)])
    '0-1'
    >>> progress_identifier([(0, 1), (2, 4)])
    '0-1|2-4'
    """
    return '|'.join('%d-%d' % lvl for lvl in level_progresses)

def parse_progress_identifier(identifier):
    """
    >>> parse_progress_identifier('')
    None
    >>> parse_progress_identifier('0-1')
    [(0, 1)]
    >>> parse_progress_identifier('0-1|2-4')
    [(0, 1), (2, 4)]
    """
    if not identifier:
        return None
    levels = []
    for level in identifier.split('|'):
        level = level.split('-')
        levels.append((int(level[0]), int(level[1])))
    return levels

class RequestsHTTPClient(object):
    """
    MapProxy compatible HTTPClient implementation that uses the
    session function of the requests lib to get keep-alive connections.
    Only supports GET requests, so it might only be used for readonly CouchDB access.
    """
    def __init__(self, url=None, username=None, password=<PASSWORD>, insecure=False,
                 ssl_ca_certs=None, timeout=None, headers=None):
        self.req_session = requests.Session(timeout=timeout)

    def open(self, url, data=None):

        code = None
        result = None
        try:
            start_time = time.time()
            result = self.req_session.get(url)
        except (requests.exceptions.RequestException, Exception), e:
            reraise_exception(HTTPClientError('Internal HTTP error "%s": %r'
                                              % (url, e)), sys.exc_info())
        else:
            code = result.status_code
            if not (200 <= code < 300):
                reraise_exception(HTTPClientError('HTTP Error "%s": %d'
                % (url, code), response_code=code), sys.exc_info())

            if code == 204:
                raise HTTPClientError('HTTP Error "204 No Content"', response_code=204)
            return result
        finally:
            log_request(url, code, result, duration=time.time()-start_time, method='GET')

    def open_image(self, url, data=None):
        resp = self.open(url, data=data)
        if 'content-type' in resp.headers:
            if not resp.headers['content-type'].lower().startswith('image'):
                raise HTTPClientError('response is not an image: (%s)' % (resp.content))
        return ImageSource(StringIO(resp.content))


def create_source(raster_source, app_state):
    url = raster_source.url
    username = raster_source.username
    password = <PASSWORD>

    http_client = HTTPClient(url, username, password)
                            # , insecure=insecure,
                             # ssl_ca_certs=ssl_ca_certs, timeout=timeout,
                             # headers=headers)

    grid = DEFAULT_GRID
    image_opts = None
    coverage = coverage_from_geojson(raster_source.download_coverage)
    format = raster_source.format

    url_template = TileURLTemplate(
        '%s/%s/%s-%%(z)s-%%(x)s-%%(y)s/tile' % (url.rstrip('/'), raster_source.layer, raster_source.matrix_set),
        format=format)
    client = TileClient(url_template, http_client=http_client, grid=grid)

    if app_state.tilebox.is_running():
        port = app_state.config.get('tilebox', 'port')
        url_template = TileURLTemplate(
            'http://127.0.0.1:%d/%s/%s-%%(z)s-%%(x)s-%%(y)s/tile' % (port, raster_source.layer, raster_source.matrix_set),
            format=format)
        tilebox_client =  TileClient(url_template, http_client=RequestsHTTPClient(), grid=grid)
        client = FallbackTileClient(tilebox_client, client)
{'fraction_non_alphanumeric': 0.10622798591098304, 'fraction_numerical': 0.009606147934678195, 'mean_word_length': 3.153257978723404, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3196556', 'n_tokens_mistral': 3764, 'n_tokens_neox': 3672, 'n_words': 949}