<commit_before># Line smoothing for WKT geometries

import math


# Visvalingam algorithm
# See https://en.wikipedia.org/wiki/Visvalingam–Whyatt_algorithm
def visvalingam(wkt, epsilon):
    line = geometryparser(wkt)
    run = True

    # Line with less than 3 vertices cannot be simplified:
    if (len(line) < 3):
        return None

    while run is True:
        min_area = float("inf")         # Placeholder
        min_area_index = None           # Placeholder

        # Loop through all vertices:
        for i in range(len(line) - 2):  # -2 due to need of +2 vertices to create triangle
            
            vertex_index = i + 1        # Vertex "responsible for area" - middle vertex of three
            area = get_area(line[i], line[i+1], line[i+2])

            if (area < min_area):
                min_area = area
                min_area_index = i + 1  # Vertex to possibly be removed

        if (min_area < epsilon and min_area_index is not None and len(line) >= 3):
            line = pop_vertex(line, min_area_index)
        else:
            run = False

    return geometryparser_wkt(line)     # Return WKT


# Get triangle area (used in Visvalingam algorithm)
def get_area(p1, p2, p3):
    return math.fabs(0.5 * (((p2[0] - p1[0]) * (p3[1] - p1[1])) - ((p3[0] - p1[0]) * (p2[1] - p1[1]))))


# Remove vertex from line
#   Removes a certain vertex from line. The vertex to be removed is indicated by index number.
#   - Inputs: a line and vertex index number
#   - Output: line with vertex removed
def pop_vertex(line, index):
    ret = []
    for i in range(len(line)):
        if (i == index):
            continue
        else:
            ret.append(line[i])
    return tuple(ret)


# Douglas-Peucker algorithm
# TODO
def douglas_peucker(wkt, epsilon):
    # TODO
    pass


# Parses WKT geometries to point tuples ((x,y), (x,y), (x,y), ...)
def geometryparser(wkt):
    ret = None
    if (wkt[:10].upper() == "LINESTRING"):  # Only linestrings allowed
        tmp_array = []
        ret = wkt[10:].strip("() ").replace("(", "").replace(")", "")
        for i in ret.split(","):
            tmp = i.strip().split(" ")
            tmp_array.append((float(tmp[0]), float(tmp[1])))
        ret = tuple(tmp_array)
    else:                                   # Other geometry types are not allowed, terminate
        print("Invalid geometry type. Exiting.")
        exit()
    return ret


# Parses WKT strings from point tuples
#   - Input: tuple of vertices ((x,y), (x,y), (x,y), ...)
#   - Output: WKT Linestring, See https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry 
def geometryparser_wkt(vertices):
    wkt = "LINESTRING ("                    # No need for other geometry types
    for i in range(len(vertices)):
        wkt += str(vertices[i][0]) + " " + str(vertices[i][1]) + ", "
    wkt = wkt[:-2]
    wkt += ")"
    return wkt


#<commit_msg>Visvalingam line simplification, minor edits<commit_after># Line smoothing for WKT geometries

import math


# Visvalingam algorithm
# See https://en.wikipedia.org/wiki/Visvalingam–Whyatt_algorithm
def visvalingam(wkt, epsilon):
    line = geometryparser(wkt)
    run = True

    # Line with less than 3 vertices cannot be simplified:
    if (len(line) < 3):
        return None

    while run is True:
        min_area = float("inf")         # Placeholder
        min_area_index = None           # Placeholder

        # Loop through all vertices:
        for i in range(len(line) - 2):  # -2 due to need of +2 vertices to create triangle
            vertex_index = i + 1        # Vertex "responsible for area" - middle vertex of three
            area = get_area(line[i], line[i+1], line[i+2])
            if (area < min_area):
                min_area = area
                min_area_index = i + 1  # Vertex to possibly be removed
        if (min_area < epsilon and min_area_index is not None and len(line) >= 3):
            line = pop_vertex(line, min_area_index)
        else:
            run = False

    return geometryparser_wkt(line)     # Return WKT


# Get triangle area (used in Visvalingam algorithm)
def get_area(p1, p2, p3):
    return math.fabs(0.5 * (((p2[0] - p1[0]) * (p3[1] - p1[1])) - ((p3[0] - p1[0]) * (p2[1] - p1[1]))))


# Remove vertex from line
#   Removes a certain vertex from line. The vertex to be removed is indicated by index number.
#   - Inputs: a line and vertex index number
#   - Output: line with vertex removed
def pop_vertex(line, index):
    ret = []
    for i in range(len(line)):
        if (i == index):
            continue
        else:
            ret.append(line[i])
    return tuple(ret)


# Douglas-Peucker algorithm
def douglas_peucker(wkt, epsilon):
    # TODO
    pass


# Parses WKT geometries to point tuples ((x,y), (x,y), (x,y), ...)
def geometryparser(wkt):
    ret = None
    if (wkt[:10].upper() == "LINESTRING"):  # Only linestrings allowed (needed)
        tmp_array = []
        ret = wkt[10:].strip("() ").replace("(", "").replace(")", "")
        for i in ret.split(","):
            tmp = i.strip().split(" ")
            tmp_array.append((float(tmp[0]), float(tmp[1])))
        ret = tuple(tmp_array)
    else:                                   # Other geometry types are not allowed, terminate
        print("Invalid geometry type. Exiting.")
        exit()
    return ret


# Parses WKT strings from point tuples
#   - Input: tuple of vertices ((x,y), (x,y), (x,y), ...)
#   - Output: WKT Linestring, See https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry 
def geometryparser_wkt(vertices):
    wkt = "LINESTRING ("                    # No need for other geometry types
    for i in range(len(vertices)):
        wkt += str(vertices[i][0]) + " " + str(vertices[i][1]) + ", "
    wkt = wkt[:-2]
    wkt += ")"
    return wkt


#
{'fraction_non_alphanumeric': 0.13019580900034353, 'fraction_numerical': 0.013740982480247337, 'mean_word_length': 2.295415959252971, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6492225', 'n_tokens_mistral': 1920, 'n_tokens_neox': 1842, 'n_words': 560}