<commit_before>#!/usr/bin/env python3

import argparse
import requests
import sys
import yaml
import shutil
import time
import re
from lxml import etree
from pathlib import Path
from decimal import Decimal

MAPFILE_TEMPLATE = 'template.map'
APACHE_CONFIG = '/etc/httpd/conf.d/oe2_wms.conf'
APACHE_CONFIG_TEMPLATE = """<Directory {redirect_endpoint}>
        SetHandler fcgid-script
        Options ExecCGI
        SetEnv MS_MAPFILE {mapfile_location}
</Directory>
"""
DIMENSION_TEMPLATE = """"wms_timeextent" "{periods}"
                "wms_timeitem" "TIME"
                "wms_timedefault" "{default}"
                "wms_timeformat" "YYYY-MM-DD, YYYY-MM-DDTHH:MM:SSZ"
"""
STYLE_TEMPLATE = """"wms_style" "default"
                "wms_style_default_legendurl_width" "{width}"
                "wms_style_default_legendurl_height" "{height}"
                "wms_style_default_legendurl_format" "image/png"
                "wms_style_default_legendurl_href" "{href}"
"""
VALIDATION_TEMPLATE = """
        VALIDATION
            "time"                  "^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z|[0-9]{4}-[0-9]{2}-[0-9]{2})|(default)$"
            "default_time"          "{default}"
            {shapefile_validation}
        END
"""
STR_BYTES_REMOVAL = re.compile(r"^b'(.*)'$")


def get_map_bounds(bbox, epsg, scale_denominator, tilesize, matrix_width, matrix_height):
    upper_left_x, lower_right_y, lower_right_x, upper_left_y = bbox
    pixelsize = Decimal(str(0.00028))
    if epsg == "EPSG:4326":
        units = Decimal(str(111319.490793274)) # convert to meters
    else:
        units = Decimal(str(1))
    size = round((tilesize*2)*(scale_denominator/units)*(pixelsize/2),8)
    lower_right_y = str((-1*size*matrix_height) + Decimal(upper_left_y))
    lower_right_x = str(Decimal(upper_left_x) + size*matrix_width)
    return [upper_left_x, lower_right_y, lower_right_x, upper_left_y]

def get_tile_level(tms, tilematrixsets):
    tile_level = 0
    for tilematrixset in tilematrixsets:
        if tilematrixset.findtext('{*}Identifier') == tms:
            tile_level = len(tilematrixset.findall('{*}TileMatrix')) - 1
    return str(tile_level)

def strip_trailing_slash(string):
    if string.endswith('/'):
        string = string[:-1]
    return string

def strip_decode_bytes_format(string):
    # interpret any bytes in the string
    string = string.encode('latin-1').decode('unicode_escape').encode('latin-1').decode('utf-8')
    match = STR_BYTES_REMOVAL.match(string)
    if match:
        return match.group(match.lastindex)
    return string

def bulk_replace(source_str, replace_list):
    out_str = source_str
    for item in replace_list:
        out_str = out_str.replace(item[0], str(item[1]))
    return out_str

def get_layer_config(layer_config_path):
    with layer_config_path.open() as f:
        config = yaml.safe_load(f.read())
    return {'path': str(layer_config_path), 'config': config}

def get_layer_configs(endpoint_config):
    try:
        layer_source = Path(endpoint_config['layer_config_source'])
    except KeyError:
        print("\nERROR: Must specify 'layer_config_source'!")
        sys.exit()

    # Find all source configs - traversing down a single directory level
    if not layer_source.exists():
        print(f"ERROR: Can't find specified layer config location: {layer_source}")
        sys.exit()
    if layer_source.is_file():
        return [get_layer_config(layer_source)]
    elif layer_source.is_dir():
        return [
            get_layer_config(filepath) for filepath in layer_source.iterdir()
            if filepath.is_file() and filepath.name.endswith('.yaml')
        ]

def get_gc(url):
    # Get source GetCapabilities
    print('Fetching ' + url)
    attempt = 1
    retries = 10
    duration = 30 # in seconds
    try:
        r = requests.get(url)
        if r.status_code != 200:
            print("Can't get GetCapabilities file from url " + gc_url)
            sys.exit()

        return r.content
    except:
        while attempt < retries:
            time.sleep(duration)
            attempt = attempt + 1
            print("Failed attempt " + str(attempt) + " to connect to " + gc_url)
            try:
                r = requests.get(gc_url)
                if r.status_code == 200:
                    break
            except Exception as e:
                print("ERROR:", e)
        if attempt == retries:
            print("Can't get GetCapabilities file from url " + gc_url)
            sys.exit()

# Parse arguments
parser = argparse.ArgumentParser(description='Make WMS endpoint.')
parser.add_argument('endpoint_config', type=str, help='an endpoint config YAML file')
parser.add_argument('--shapefile_bucket', dest='shapefile_bucket', type=str, default='', help='S3 bucket used for shapefiles')
args = parser.parse_args()
endpoint_config = yaml.safe_load(Path(args.endpoint_config).read_text())
print('Using endpoint config ' + args.endpoint_config)
if args.shapefile_bucket != '':
    print('Using shapefile bucket ' + args.shapefile_bucket)
    shapefile_bucket = '/vsis3/' + args.shapefile_bucket
else:
    shapefile_bucket = ''
outfilename = Path(endpoint_config['mapserver']['mapfile_location'])
header = Path(endpoint_config['mapserver']['mapfile_header'])
redirect_endpoint = Path(strip_trailing_slash(endpoint_config['mapserver']['redirect_endpoint']))
epsg_code = endpoint_config['epsg_code']

# Get layer configs
layer_configs = get_layer_configs(endpoint_config)

# Get source GetCapabilities
gc_url = endpoint_config['mapserver']['source_wmts_gc_uri']
# Replace matching host names with local Docker host IP http://172.17.0.1 so that connections stay local
if endpoint_config['mapserver'].get('replace_with_local'):
    replace_with_local = endpoint_config['mapserver']['replace_with_local']
    gc_url = gc_url.replace(replace_with_local, 'http://172.17.0.1:8080')

gc_content = get_gc(gc_url)

# Read GetCapabilities
root = etree.fromstring(gc_content)
layers = root.find('{*}Contents').findall('{*}Layer')
layer_strings = []

tilematrixsets = root.find('{*}Contents').findall('{*}TileMatrixSet')
tm = tilematrixsets[0].findall('{*}TileMatrix')
scale_denominator = Decimal(tm[0].findtext('{*}ScaleDenominator'))
tile_width = int(tm[0].findtext('{*}TileWidth'))
tile_height = int(tm[0].findtext('{*}TileHeight'))
matrix_width = int(tm[0].findtext('{*}MatrixWidth'))
matrix_height = int(tm[0].findtext('{*}MatrixHeight'))

for layer in layers:
    web_mercator = None
    layer_name = layer.findtext('{*}Identifier')
    tms = layer.find('{*}TileMatrixSetLink').findtext('{*}TileMatrixSet')
    bbox = layer.find('{*}BoundingBox')
    if bbox is None:
        bbox = layer.find('{*}WGS84BoundingBox')

    upper_left_x = bbox.findtext('{*}LowerCorner').split(' ')[0]
    upper_left_y = bbox.findtext('{*}UpperCorner').split(' ')[1]
    lower_right_x = bbox.findtext('{*}UpperCorner').split(' ')[0]
    lower_right_y = bbox.findtext('{*}LowerCorner').split(' ')[1]
    
    bounds = get_map_bounds([upper_left_x, lower_right_y, lower_right_x, upper_left_y], epsg_code, scale_denominator, tile_width, matrix_width, matrix_height)

    resource_url = layer.findall('{*}ResourceURL')[-1] # get last if multiple found
    bands_count = 4 if resource_url.get('format') == 'image/png' else 3
    template_string = resource_url.get('template')
    # Replace matching host names with local Docker host IP http://172.17.0.1 so that connections stay local
    if endpoint_config['mapserver'].get('replace_with_local'):<commit_msg>Use first ResourceURL when configuring WMS layer template<commit_after>#!/usr/bin/env python3

import argparse
import requests
import sys
import yaml
import shutil
import time
import re
from lxml import etree
from pathlib import Path
from decimal import Decimal

MAPFILE_TEMPLATE = 'template.map'
APACHE_CONFIG = '/etc/httpd/conf.d/oe2_wms.conf'
APACHE_CONFIG_TEMPLATE = """<Directory {redirect_endpoint}>
        SetHandler fcgid-script
        Options ExecCGI
        SetEnv MS_MAPFILE {mapfile_location}
</Directory>
"""
DIMENSION_TEMPLATE = """"wms_timeextent" "{periods}"
                "wms_timeitem" "TIME"
                "wms_timedefault" "{default}"
                "wms_timeformat" "YYYY-MM-DD, YYYY-MM-DDTHH:MM:SSZ"
"""
STYLE_TEMPLATE = """"wms_style" "default"
                "wms_style_default_legendurl_width" "{width}"
                "wms_style_default_legendurl_height" "{height}"
                "wms_style_default_legendurl_format" "image/png"
                "wms_style_default_legendurl_href" "{href}"
"""
VALIDATION_TEMPLATE = """
        VALIDATION
            "time"                  "^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z|[0-9]{4}-[0-9]{2}-[0-9]{2})|(default)$"
            "default_time"          "{default}"
            {shapefile_validation}
        END
"""
STR_BYTES_REMOVAL = re.compile(r"^b'(.*)'$")


def get_map_bounds(bbox, epsg, scale_denominator, tilesize, matrix_width, matrix_height):
    upper_left_x, lower_right_y, lower_right_x, upper_left_y = bbox
    pixelsize = Decimal(str(0.00028))
    if epsg == "EPSG:4326":
        units = Decimal(str(111319.490793274)) # convert to meters
    else:
        units = Decimal(str(1))
    size = round((tilesize*2)*(scale_denominator/units)*(pixelsize/2),8)
    lower_right_y = str((-1*size*matrix_height) + Decimal(upper_left_y))
    lower_right_x = str(Decimal(upper_left_x) + size*matrix_width)
    return [upper_left_x, lower_right_y, lower_right_x, upper_left_y]

def get_tile_level(tms, tilematrixsets):
    tile_level = 0
    for tilematrixset in tilematrixsets:
        if tilematrixset.findtext('{*}Identifier') == tms:
            tile_level = len(tilematrixset.findall('{*}TileMatrix')) - 1
    return str(tile_level)

def strip_trailing_slash(string):
    if string.endswith('/'):
        string = string[:-1]
    return string

def strip_decode_bytes_format(string):
    # interpret any bytes in the string
    string = string.encode('latin-1').decode('unicode_escape').encode('latin-1').decode('utf-8')
    match = STR_BYTES_REMOVAL.match(string)
    if match:
        return match.group(match.lastindex)
    return string

def bulk_replace(source_str, replace_list):
    out_str = source_str
    for item in replace_list:
        out_str = out_str.replace(item[0], str(item[1]))
    return out_str

def get_layer_config(layer_config_path):
    with layer_config_path.open() as f:
        config = yaml.safe_load(f.read())
    return {'path': str(layer_config_path), 'config': config}

def get_layer_configs(endpoint_config):
    try:
        layer_source = Path(endpoint_config['layer_config_source'])
    except KeyError:
        print("\nERROR: Must specify 'layer_config_source'!")
        sys.exit()

    # Find all source configs - traversing down a single directory level
    if not layer_source.exists():
        print(f"ERROR: Can't find specified layer config location: {layer_source}")
        sys.exit()
    if layer_source.is_file():
        return [get_layer_config(layer_source)]
    elif layer_source.is_dir():
        return [
            get_layer_config(filepath) for filepath in layer_source.iterdir()
            if filepath.is_file() and filepath.name.endswith('.yaml')
        ]

def get_gc(url):
    # Get source GetCapabilities
    print('Fetching ' + url)
    attempt = 1
    retries = 10
    duration = 30 # in seconds
    try:
        r = requests.get(url)
        if r.status_code != 200:
            print("Can't get GetCapabilities file from url " + gc_url)
            sys.exit()

        return r.content
    except:
        while attempt < retries:
            time.sleep(duration)
            attempt = attempt + 1
            print("Failed attempt " + str(attempt) + " to connect to " + gc_url)
            try:
                r = requests.get(gc_url)
                if r.status_code == 200:
                    break
            except Exception as e:
                print("ERROR:", e)
        if attempt == retries:
            print("Can't get GetCapabilities file from url " + gc_url)
            sys.exit()

# Parse arguments
parser = argparse.ArgumentParser(description='Make WMS endpoint.')
parser.add_argument('endpoint_config', type=str, help='an endpoint config YAML file')
parser.add_argument('--shapefile_bucket', dest='shapefile_bucket', type=str, default='', help='S3 bucket used for shapefiles')
args = parser.parse_args()
endpoint_config = yaml.safe_load(Path(args.endpoint_config).read_text())
print('Using endpoint config ' + args.endpoint_config)
if args.shapefile_bucket != '':
    print('Using shapefile bucket ' + args.shapefile_bucket)
    shapefile_bucket = '/vsis3/' + args.shapefile_bucket
else:
    shapefile_bucket = ''
outfilename = Path(endpoint_config['mapserver']['mapfile_location'])
header = Path(endpoint_config['mapserver']['mapfile_header'])
redirect_endpoint = Path(strip_trailing_slash(endpoint_config['mapserver']['redirect_endpoint']))
epsg_code = endpoint_config['epsg_code']

# Get layer configs
layer_configs = get_layer_configs(endpoint_config)

# Get source GetCapabilities
gc_url = endpoint_config['mapserver']['source_wmts_gc_uri']
# Replace matching host names with local Docker host IP http://172.17.0.1 so that connections stay local
if endpoint_config['mapserver'].get('replace_with_local'):
    replace_with_local = endpoint_config['mapserver']['replace_with_local']
    gc_url = gc_url.replace(replace_with_local, 'http://172.17.0.1:8080')

gc_content = get_gc(gc_url)

# Read GetCapabilities
root = etree.fromstring(gc_content)
layers = root.find('{*}Contents').findall('{*}Layer')
layer_strings = []

tilematrixsets = root.find('{*}Contents').findall('{*}TileMatrixSet')
tm = tilematrixsets[0].findall('{*}TileMatrix')
scale_denominator = Decimal(tm[0].findtext('{*}ScaleDenominator'))
tile_width = int(tm[0].findtext('{*}TileWidth'))
tile_height = int(tm[0].findtext('{*}TileHeight'))
matrix_width = int(tm[0].findtext('{*}MatrixWidth'))
matrix_height = int(tm[0].findtext('{*}MatrixHeight'))

for layer in layers:
    web_mercator = None
    layer_name = layer.findtext('{*}Identifier')
    tms = layer.find('{*}TileMatrixSetLink').findtext('{*}TileMatrixSet')
    bbox = layer.find('{*}BoundingBox')
    if bbox is None:
        bbox = layer.find('{*}WGS84BoundingBox')

    upper_left_x = bbox.findtext('{*}LowerCorner').split(' ')[0]
    upper_left_y = bbox.findtext('{*}UpperCorner').split(' ')[1]
    lower_right_x = bbox.findtext('{*}UpperCorner').split(' ')[0]
    lower_right_y = bbox.findtext('{*}LowerCorner').split(' ')[1]
    
    bounds = get_map_bounds([upper_left_x, lower_right_y, lower_right_x, upper_left_y], epsg_code, scale_denominator, tile_width, matrix_width, matrix_height)

    resource_url = layer.findall('{*}ResourceURL')[0] # get first if multiple found
    bands_count = 4 if resource_url.get('format') == 'image/png' else 3
    template_string = resource_url.get('template')
    # Replace matching host names with local Docker host IP http://172.17.0.1 so that connections stay local
    if endpoint_config['mapserver'].get('replace_with_local'):
{'fraction_non_alphanumeric': 0.13442991148104108, 'fraction_numerical': 0.01598625974369137, 'mean_word_length': 3.7772167876301674, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2351616', 'n_tokens_mistral': 5123, 'n_tokens_neox': 4922, 'n_words': 964}