<jupyter_start><jupyter_text><jupyter_code>prop = 0.2
angle = 0
data = {'prop' : [prop] ,'angle' : [angle]}
test_df = pd.DataFrame(data, index=None) 
clustering(df1,1,4, False, test_df)
optimal_k(df2,2)
#너무 큰 비율->잘못된 detection이므로 제거
idx_nm = df2[df2['prop']>15].index
df2 = df2.drop(idx_nm)
df2.head()
df2['prop'] = (df2['prop']-df2['prop'].mean())/(df2['prop'].max()-df2['prop'].min())
df2['angle'] = (df2['angle']-df2['angle'].mean())/(df2['angle'].max()-df2['angle'].min())
clustering(df2,2,3)

df3 = pd.read_csv('/content/namsantower_df.csv')
df3
optimal_k(df3,3)
df3['prop'] = (df3['prop']-df3['prop'].mean())/(df3['prop'].max()-df3['prop'].min())
df3['angle'] = (df3['angle']-df3['angle'].mean())/(df3['angle'].max()-df3['angle'].min())
#너무 큰 비율->잘못된 detection이므로 제거
idx_nm = df3[df3['prop']>10].index
df3 = df3.drop(idx_nm)
clustering(df3,3,3)

optimal_k(df4,4)
#너무 큰 비율->잘못된 detection이므로 제거
idx_nm = df4[df4['prop']>20].index
df4 = df4.drop(idx_nm)
df4['prop'] = (df4['prop']-df4['prop'].mean())/(df4['prop'].max()-df4['prop'].min())
df4['angle'] = (df4['angle']-df4['angle'].mean())/(df4['angle'].max()-df4['angle'].min())
clustering(df4,4,5,False,test_df)

optimal_k(df5,5)
#너무 큰 비율->잘못된 detection이므로 제거
idx_nm = df5[df5['prop']>10].index
df5 = df5.drop(idx_nm)
df5['prop'] = (df5['prop']-df5['prop'].mean())/(df5['prop'].max()-df5['prop'].min())
df5['angle'] = (df5['angle']-df5['angle'].mean())/(df5['angle'].max()-df5['angle'].min())
clustering(df5,5,4)
df1.to_csv('lottetower_df.csv', index=False)
df2.to_csv('merrygoround_df.csv', index=False)
df3.to_csv('namsantower_df.csv', index=False)
df4.to_csv('myeongdongcathedral_df.csv', index=False)
df5.to_csv('lottecastle_df.csv', index=False)
df1.drop[:,1]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2780320366132723, 'fraction_numerical': 0.057208237986270026, 'mean_word_length': 7.969230769230769, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '336690', 'license': 'MIT', 'n_tokens_mistral': 852, 'n_tokens_neox': 891, 'n_words': 90, 'path': 'src/data_classification.ipynb', 'repo_name': 'PIAIAU/photo_guide', 'size': '1748'}