<commit_before>import json
import logging
import os
import time
import urllib.parse
import urllib.request
from datetime import datetime
from zipfile import ZipFile

import requests

import bgt_setup
import fme.comparison as fme_comparison
import fme.fme_server as fme_server
import fme.fme_utils as fme_utils
import fme.sql_utils as fme_sql_utils
from fme.transform_db import start_transformation_db
from fme.transform_dgn import start_transformation_dgn, upload_dgn_files
from fme.transform_gebieden import start_transformation_gebieden
from fme.transform_nlcs import (
    start_transformation_nlcs_chunk, upload_nlcs_lijnen_files, upload_nlcs_vlakken_files)
from fme.transform_shapes import start_transformation_shapes
from fme.transform_stand_ligplaatsen import start_transformation_stand_ligplaatsen
from objectstore.objectstore import ObjectStore

logging.basicConfig(level=logging.INFO)
log = logging.getLogger(__name__)


def resolve_chunk_coordinates():
    """
    calls `00_kaartbladen_coordinatenbepaler.fmw` on FME server
    output csv is used to split dgnNCS job in smaller chunks

    :return: dict with 'jobid' and 'urltransform'
    """
    log.info("Starting transformation -:> Resolve Chunk Coordinates")

    return fme_utils.run_transformation_job(
        'BGT-DGN',
        '00_kaartbladen_coordinatenbepaler.fmw',
        {
            "subsection": "REST_SERVICE",
            "FMEDirectives": {},
            "NMDirectives": {"successTopics": [], "failureTopics": []},
            "TMDirectives": {"tag": "linux", "description": "Bepaal coordinaten"},
            "publishedParameters": [
                {"name": "SourceDataset_POSTGIS_3", "value": "bgt"},
                {"name": "DestDataset_CSV", "value": "$(FME_SHAREDRESOURCE_DATA)BGT_uitwissel"}]})


def retrieve_chunk_coordinates():
    """
    Retrieve the chunck coordinates stored in
    $(FME_SHAREDRESOURCE_DATA)BGT_uitwissel/Kaartbladen_coordinaten.csv
    and return an iterator with the coordinates

    :return:
    """
    content = fme_utils.download("BGT_uitwissel/Kaartbladen_coordinaten.csv")

    coords = [row.split(',') for row in content.split('\n')[1:-1]]
    return coords


def upload_pdok_zip_to_objectstore():
    """
    Upload the PDOK/bgt source zip to the objecstore
    :return:
    """
    store = ObjectStore('BGT')
    log.info("Upload BGT source zip")
    content = open('extract_bgt.zip', 'rb').read()

    timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
    filename = 'bron-gml/extract_bgt-{}.zip'.format(timestamp)
    store.put_to_objectstore(filename, content, 'application/octet-stream')
    log.info("Uploaded {} to objectstore BGT/bron-gml".format(filename))


def upload_over_onderbouw_backup():
    """
    Upload Over- en Onderbouw data

    :return:
    """

    # 1. fetch latest `GBKA_OVERBOUW.dat from objectstore
    log.info("ZIP and upload DGNv8 lijnen products to BGT objectstore")

    store = ObjectStore('BGT')
    headers = {
        'Content-Type': "application/json",
        'Authorization': 'fmetoken token={FME_API}'.format(FME_API=bgt_setup.FME_API),
    }

    # determine the latest upload filename
    latest_upload = sorted(
        [c['name'] for c in store._get_full_container_list([]) if c['name'].endswith('_GBKA_OVERBOUW.zip')])[-1]

    if not os.path.exists('/tmp/data'):
        os.makedirs('/tmp/data')

    # download the latest `GBKA_OVERBOUW.dat` zipfile and temporary store it
    with open("/tmp/data/GBKA_OVERBOUW.dat.zip", 'wb') as f:
        f.write(store.get_store_object(latest_upload))

    # unzip the GBKA_OVERBOUW.dat.zip file
    with ZipFile('/tmp/data/GBKA_OVERBOUW.dat.zip', 'r') as myzip:
        myzip.extractall('/tmp/data/')

    # get the data and process
    data = open("/tmp/data/Alle_Producten/GBKA_A/Objecten/WKT/GBKA_OVERBOUW.dat").read()
    # cut column [0:4], create sql insert statements and exec.
    db = create_fme_sql_connection()
    for line in str(data).split(os.linesep):
        fields = line.split('|')[1:]
        if len(fields) > 0:
            if int(fields[2]) > 0:
                # overbouw
                sql = "insert into imgeo.\"CFT_Overbouw\" (guid, relatievehoogteligging, bestandsnaam, geometrie) " \
                      "values ({}, {}, 'CFT_Overbouw', ST_GeomFromText('{}', 28992));".format(
                    fields[0].replace('$$', ''), int(fields[2]), fields[3])
            else:
                # onderbouw
                sql = "insert into imgeo.\"CFT_Onderbouw\" (guid, relatievehoogteligging, bestandsnaam, geometrie) " \
                      "values ({}, {}, 'CFT_Onderbouw', ST_GeomFromText('{}', 28992));".format(
                    fields[0].replace('$$', ''), int(fields[2]), fields[3])
            db.run_sql(sql)


def unzip_pdok_file():
    """
    Unzip the extract_bgt.zip file into the `/tmp/data` directory
    :return:
    """<commit_msg>add quotes<commit_after>import json
import logging
import os
import time
import urllib.parse
import urllib.request
from datetime import datetime
from zipfile import ZipFile

import requests

import bgt_setup
import fme.comparison as fme_comparison
import fme.fme_server as fme_server
import fme.fme_utils as fme_utils
import fme.sql_utils as fme_sql_utils
from fme.transform_db import start_transformation_db
from fme.transform_dgn import start_transformation_dgn, upload_dgn_files
from fme.transform_gebieden import start_transformation_gebieden
from fme.transform_nlcs import (
    start_transformation_nlcs_chunk, upload_nlcs_lijnen_files, upload_nlcs_vlakken_files)
from fme.transform_shapes import start_transformation_shapes
from fme.transform_stand_ligplaatsen import start_transformation_stand_ligplaatsen
from objectstore.objectstore import ObjectStore

logging.basicConfig(level=logging.INFO)
log = logging.getLogger(__name__)


def resolve_chunk_coordinates():
    """
    calls `00_kaartbladen_coordinatenbepaler.fmw` on FME server
    output csv is used to split dgnNCS job in smaller chunks

    :return: dict with 'jobid' and 'urltransform'
    """
    log.info("Starting transformation -:> Resolve Chunk Coordinates")

    return fme_utils.run_transformation_job(
        'BGT-DGN',
        '00_kaartbladen_coordinatenbepaler.fmw',
        {
            "subsection": "REST_SERVICE",
            "FMEDirectives": {},
            "NMDirectives": {"successTopics": [], "failureTopics": []},
            "TMDirectives": {"tag": "linux", "description": "Bepaal coordinaten"},
            "publishedParameters": [
                {"name": "SourceDataset_POSTGIS_3", "value": "bgt"},
                {"name": "DestDataset_CSV", "value": "$(FME_SHAREDRESOURCE_DATA)BGT_uitwissel"}]})


def retrieve_chunk_coordinates():
    """
    Retrieve the chunck coordinates stored in
    $(FME_SHAREDRESOURCE_DATA)BGT_uitwissel/Kaartbladen_coordinaten.csv
    and return an iterator with the coordinates

    :return:
    """
    content = fme_utils.download("BGT_uitwissel/Kaartbladen_coordinaten.csv")

    coords = [row.split(',') for row in content.split('\n')[1:-1]]
    return coords


def upload_pdok_zip_to_objectstore():
    """
    Upload the PDOK/bgt source zip to the objecstore
    :return:
    """
    store = ObjectStore('BGT')
    log.info("Upload BGT source zip")
    content = open('extract_bgt.zip', 'rb').read()

    timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
    filename = 'bron-gml/extract_bgt-{}.zip'.format(timestamp)
    store.put_to_objectstore(filename, content, 'application/octet-stream')
    log.info("Uploaded {} to objectstore BGT/bron-gml".format(filename))


def upload_over_onderbouw_backup():
    """
    Upload Over- en Onderbouw data

    :return:
    """

    # 1. fetch latest `GBKA_OVERBOUW.dat from objectstore
    log.info("ZIP and upload DGNv8 lijnen products to BGT objectstore")

    store = ObjectStore('BGT')
    headers = {
        'Content-Type': "application/json",
        'Authorization': 'fmetoken token={FME_API}'.format(FME_API=bgt_setup.FME_API),
    }

    # determine the latest upload filename
    latest_upload = sorted(
        [c['name'] for c in store._get_full_container_list([]) if c['name'].endswith('_GBKA_OVERBOUW.zip')])[-1]

    if not os.path.exists('/tmp/data'):
        os.makedirs('/tmp/data')

    # download the latest `GBKA_OVERBOUW.dat` zipfile and temporary store it
    with open("/tmp/data/GBKA_OVERBOUW.dat.zip", 'wb') as f:
        f.write(store.get_store_object(latest_upload))

    # unzip the GBKA_OVERBOUW.dat.zip file
    with ZipFile('/tmp/data/GBKA_OVERBOUW.dat.zip', 'r') as myzip:
        myzip.extractall('/tmp/data/')

    # get the data and process
    data = open("/tmp/data/Alle_Producten/GBKA_A/Objecten/WKT/GBKA_OVERBOUW.dat").read()
    # cut column [0:4], create sql insert statements and exec.
    db = create_fme_sql_connection()
    for line in str(data).split(os.linesep):
        fields = line.split('|')[1:]
        if len(fields) > 0:
            if int(fields[2]) > 0:
                # overbouw
                sql = "insert into imgeo.\"CFT_Overbouw\" (guid, relatievehoogteligging, bestandsnaam, geometrie) " \
                      "values ('{}', {}, 'CFT_Overbouw', ST_GeomFromText('{}', 28992));".format(
                    fields[0].replace('$$', ''), int(fields[2]), fields[3])
            else:
                # onderbouw
                sql = "insert into imgeo.\"CFT_Onderbouw\" (guid, relatievehoogteligging, bestandsnaam, geometrie) " \
                      "values ('{}', {}, 'CFT_Onderbouw', ST_GeomFromText('{}', 28992));".format(
                    fields[0].replace('$$', ''), int(fields[2]), fields[3])
            db.run_sql(sql)


def unzip_pdok_file():
    """
    Unzip the extract_bgt.zip file into the `/tmp/data` directory
    :return:
    """
{'fraction_non_alphanumeric': 0.12470612286619646, 'fraction_numerical': 0.00654196054380047, 'mean_word_length': 3.4151624548736463, 'pattern_counts': {'":': 26, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3047987', 'n_tokens_mistral': 3368, 'n_tokens_neox': 3245, 'n_words': 671}