<jupyter_start><jupyter_text>clean up holes coordinates<jupyter_code>na_values = ['', '*']             
hole_df = pd.read_csv(hole_path, na_values=na_values)
print_df(hole_df)<jupyter_output>(3821, 22)<jupyter_text>remove bad rows<jupyter_code>hole_df = hole_df[hole_df['Exp'] != 'TEST']
hole_df = hole_df[hole_df['Exp'] != '999']
hole_df.dropna(axis='index', how='all',  inplace=True, subset=['Latitude', 'Longitude'])<jupyter_output><empty_output><jupyter_text>fix typos<jupyter_code>hole_df.loc[2935, 'Latitude'] = '38 49.7822 S'
hole_df.loc[2964, 'Latitude'] = '38 8.1532 S'<jupyter_output><empty_output><jupyter_text>add decimal coordinates<jupyter_code>hole_df['Latitude_decimal'] = hole_df['Latitude'].apply(ddm2dec)
hole_df['Longitude_decimal'] = hole_df['Longitude'].apply(ddm2dec)
print_df(hole_df)
hole_df.to_csv(hole_path, index=False)<jupyter_output><empty_output><jupyter_text>create normalize taxa<jupyter_code>all_taxa_df = pd.read_csv(approved_taxa_path, dtype=str)

print_df(all_taxa_df, 2)
del all_taxa_df['taxon_group']
del all_taxa_df['notes']
del all_taxa_df['name comment field']
del all_taxa_df['Comment']


all_taxa_df.drop_duplicates(inplace=True)
print_df(all_taxa_df, 2)
taxa_df = pd.read_csv(taxa_draft_path, dtype=str, usecols=['verbatim_name'])
print_df(taxa_df)
merged_df = taxa_df.merge(all_taxa_df)
print_df(merged_df)
add_normalized_name_column(merged_df)
add_normalized_name_column(merged_df, include_descriptor=False, 
                           include_modifier=False, col_name='basic_name')

print_df(merged_df)
taxa_df = merged_df[['verbatim_name', 'normalized_name', 'basic_name']]
taxa_df
taxa_df.to_csv(noramalized_taxa_path, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13909994155464642, 'fraction_numerical': 0.02104032729398013, 'mean_word_length': 10.04516129032258, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '18702', 'license': 'MIT', 'n_tokens_mistral': 739, 'n_tokens_neox': 700, 'n_words': 79, 'path': 'data_cleaning/notebooks/prepare_files.ipynb', 'repo_name': 'wykhuh/ec_workshops_py', 'size': '1711'}