<jupyter_start><jupyter_text>Parameters<jupyter_code>dbug = False<jupyter_output><empty_output><jupyter_text>Constants<jupyter_code>sep = '=' * 80
n_iris = 48592 # Found in EDA.ipynb<jupyter_output><empty_output><jupyter_text>Helpers<jupyter_code>def safe_wkt_loads(x):
    if isinstance(x, str):
        try:
            y = wkt.loads(x)
        except:
            print("WARNING:", x)
            y = Polygon([])
        return y
    else:
        print("WARNING: not a string:", x)
        return Polygon([])<jupyter_output><empty_output><jupyter_text>Version X<jupyter_code># Loading data
if dbug:
    dfx = pd.read_csv('CONTOURS-IRIS-X_dbug.csv')
else:
    dfx = pd.read_csv('CONTOURS-IRIS-X.csv')    
gdfx = gpd.GeoDataFrame(
    dfx, geometry=dfx.geometry.apply(wkt.loads)
)
del dfx
# Transformation
for i in ['geometry_nlarge', 'geometry_large']:
    print(sep)
    print("Apply safe WKT loads to", i, '...')
    gdfx[i] = gdfx[i].apply(safe_wkt_loads)
    gdfx[i] = gdfx[i].astype('geometry')
print(sep)
print("Counting number of IRIS in X ...")
gdfx['n_iris_nlarge'] = gdfx.iris_nlarge.str.replace('[','', regex=False).str.replace(']','', regex=False).str.split(',').apply(len)
# Checking number of rows
n_iris_gdfx = len(gdfx)
print(f"Number of IRIS in gdfx {n_iris_gdfx}")
if not dbug:
    assert(n_iris_gdfx==n_iris)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15808823529411764, 'fraction_numerical': 0.005147058823529412, 'mean_word_length': 4.234615384615385, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '194839', 'license': 'MIT', 'n_tokens_mistral': 568, 'n_tokens_neox': 561, 'n_words': 90, 'path': '.dataset/IGN-contours-iris/EDA-x.ipynb', 'repo_name': 'ucsky/opendata', 'size': '1360'}