<jupyter_start><jupyter_text>data vernacular names, explorationby Christian,- those data './vernacular_name.csv' were genererated by notebook vernacular_name_spark.ipyn, thank's to Tim<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt 
# load vernacular names dataset
input_filename = './vernacular_name.csv'

df = pd.read_csv(input_filename, sep=',')
print(input_filename, ', loaded')
print(df.info())
df.dropna(subset = ["vernacular_name"], inplace=True)
display(df.head())
df['len_vernacular_name'] = df['vernacular_name'].apply(lambda x: len(str(x)))
print(df['len_vernacular_name'].describe())

display(df[df['len_vernacular_name']==2].head())
display(df[df['len_vernacular_name']==3].head())
display(df[df['len_vernacular_name']==4].head())

# we keep alphabetic name with more than 3 cars

df['isapha'] = df['vernacular_name'].apply(lambda x: x.isalpha())
df = df[df['isapha']==True]
df = df[df['len_vernacular_name']>3]
df = df.drop(['_c0', 'isapha', 'len_vernacular_name'], axis=1)

df.reset_index(drop=True, inplace=True)
df.info()
df['nbmots'] = df['name'].apply(lambda x: len(x.split(' ')))  
display(df[df['nbmots']>2]['name'])<jupyter_output><empty_output><jupyter_text>- some names , are not names<jupyter_code>df = df[df['nbmots']==1]
df.reset_index(drop=True, inplace=True)
print('nb of names:', len(df['name'].unique()))
display(df['name'])
dg = df.groupby('vernacular_name')['name'].count().sort_values(ascending=False)
print('nb of distinct venacular names', len(dg))
display(dg)
ax = dg.head(20).plot(kind='bar', title='vernacular names distribution')
ax.set(xlabel='vernacular name', ylabel='count species')
plt.show()
dg.describe()<jupyter_output><empty_output><jupyter_text>- what would it be, on top of this list without the beetle<jupyter_code>display(dg[1:19])
ax = dg[1:19].plot(kind='bar', title='vernacular names distribution')
ax.set(xlabel='vernacular name', ylabel='count species')
plt.show()<jupyter_output><empty_output><jupyter_text>- from this dataset we will have to filter few names as "family, specie, order"<jupyter_code>filtre = ['family', 'species', 'order', 'rank'] 
df = df[~df['vernacular_name'].isin(filtre)]
df.reset_index(drop=True, inplace=True)
df.info()
dg = df.groupby('vernacular_name')['name'].count().sort_values(ascending=False)
print('nb of distinct vernacular names', len(dg))
ax = dg.head(20).plot(kind='bar', title='vernacular names distribution')
ax.set(xlabel='vernacular name', ylabel='count species')
plt.show()
display(dg[1:19])
ax = dg[1:19].plot(kind='bar', title='vernacular names distribution')
ax.set(xlabel='vernacular name', ylabel='count species')
plt.show()
file_name = './vernacular_name_filtered.csv'

print('nb of distinct vernacular names:', len(dg))
print('nb of estimed species names:', len(df['name'].unique()))
print('size of vernacular_name file:', len(df))

df.to_csv(file_name, sep=';', index=False)
print('filtered vernacular names:', file_name, ', saved')<jupyter_output>nb of distinct vernacular names: 1184
nb of estimed species names: 9509
size of vernacular_name file: 9510
filtered vernacular names: ./vernacular_name_filtered.csv , saved
{'fraction_non_alphanumeric': 0.17736808645899554, 'fraction_numerical': 0.011760966306420852, 'mean_word_length': 8.507552870090635, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '287385', 'license': 'MIT', 'n_tokens_mistral': 1155, 'n_tokens_neox': 1112, 'n_words': 232, 'path': 'notebooks/vernacular_names_exploration.ipynb', 'repo_name': 'timtonthat/batch8_ceebios', 'size': '3146'}