<jupyter_start><jupyter_text>Find generalization parameter and eliminate invalid polygonsWe try out some values of `tolerance` (in degrees), parameter for generalizing the majority-filtered and polygonized model output, so that the shapes are simpler.<jupyter_code># polygonized model output where majority filter with radius 3 was applied
polygons_path = '.../Data/WCSColombia/delivered/20200715/results_coarse_baseline_201920_polygons_filter3/results_coarse_baseline_201920_filter3.shp'
%%time

polygons = gpd.read_file(polygons_path)
len(polygons)
polygons.head(10)<jupyter_output><empty_output><jupyter_text>Tolerance value<jupyter_code>tolerance = 0.0003
polygon = polygons.iloc[-1000].geometry
polygon
polygon.area

poly_simp = polygon.simplify(tolerance, preserve_topology=False)
poly_simp
poly_simp.area

poly_simp = polygon.simplify(tolerance, preserve_topology=True)
poly_simp
poly_simp.area
polygon = polygons.iloc[-100].geometry
polygon
polygon.area

poly_simp = polygon.simplify(tolerance, preserve_topology=False)
poly_simp
poly_simp.area

poly_simp = polygon.simplify(tolerance, preserve_topology=True)
poly_simp
poly_simp.area
polygon = polygons.iloc[0].geometry
polygon
polygon.area

poly_simp = polygon.simplify(tolerance, preserve_topology=False)
poly_simp
poly_simp.area

poly_simp = polygon.simplify(tolerance, preserve_topology=True)
poly_simp
poly_simp.area
polygon = polygons.iloc[100000].geometry
polygon
polygon.area

poly_simp = polygon.simplify(tolerance, preserve_topology=False)
poly_simp
poly_simp.area

poly_simp = polygon.simplify(tolerance, preserve_topology=True)
poly_simp
poly_simp.area
polygon = polygons.iloc[200000].geometry
polygon
polygon.area

poly_simp = polygon.simplify(tolerance, preserve_topology=False)
poly_simp
poly_simp.area

poly_simp = polygon.simplify(tolerance, preserve_topology=True)
poly_simp
poly_simp.area
polygon = polygons.iloc[310000].geometry
polygon
polygon.area

poly_simp = polygon.simplify(tolerance, preserve_topology=False)
poly_simp
poly_simp.area

poly_simp = polygon.simplify(tolerance, preserve_topology=True)
poly_simp
poly_simp.area
polygon = polygons.iloc[330002].geometry
polygon
polygon.area

poly_simp = polygon.simplify(tolerance, preserve_topology=False)
poly_simp
poly_simp.area

poly_simp = polygon.simplify(tolerance, preserve_topology=True)
poly_simp
poly_simp.area<jupyter_output><empty_output><jupyter_text>Eliminate invalid geometries<jupyter_code>invalid_geometries = []

for i, row in tqdm(polygons.iterrows()):
    if not row.geometry.is_valid:
        invalid_geometries.append(row)
len(invalid_geometries)
invalid_geometries[1000].geometry
%%time

for i_row, row in tqdm(polygons.iterrows()):
    polygons.at[i_row, 'geometry'] = row.geometry.buffer(0)
still_invalid_geometries = []

for i, row in tqdm(polygons.iterrows()):
    if not row.geometry.is_valid:
        still_invalid_geometries.append(row)
len(still_invalid_geometries)
polygons.to_file('.../Data/WCSColombia/delivered/20200715/results_coarse_baseline_201920_polygons_filter3/valid/results_coarse_baseline_201920_polygons_filter3_valid.shp')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08881789137380192, 'fraction_numerical': 0.02843450479233227, 'mean_word_length': 7.997126436781609, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '629239', 'license': 'MIT', 'n_tokens_mistral': 1255, 'n_tokens_neox': 1129, 'n_words': 182, 'path': 'data/find_generalization_tolerance_and_make_valid.ipynb', 'repo_name': 'microsoft/landcover-orinoquia', 'size': '3130'}