<jupyter_start><jupyter_text>df_Homer_verses.to_csv('data/df_Homer_verses.csv')df_Corpus_WO_verses.to_csv('data/df_Corpus_WO_verses.csv')<jupyter_code>df_Homer_verses = pd.read_csv('data/df_Homer_verses.csv').drop(columns = 'Unnamed: 0')
df_Corpus_WO_verses = pd.read_csv('data/df_Corpus_WO_verses.csv').drop(columns = 'Unnamed: 0')
df_Homer_typology = df_Homer_verses['code'].value_counts()
df_Homer_typology = df_Homer_typology[df_Homer_typology > 2]
df_Homer_typology = np.asarray(df_Homer_typology.index).tolist()

df_Corpus_typology = df_Corpus_WO_verses['code'].value_counts()
df_Corpus_typology = df_Corpus_typology[df_Corpus_typology > 2]
df_Corpus_typology = np.asarray(df_Corpus_typology.index).tolist()
df_Homer_verses = df_Homer_verses[df_Homer_verses['code'].isin(df_Homer_typology)]
df_Corpus_WO_verses = df_Corpus_WO_verses[df_Corpus_WO_verses['code'].isin(df_Corpus_typology)]
df_Homer_verses['class'] = 1
df_Corpus_WO_verses['class'] = 0
dataset = pd.concat([df_Homer_verses, df_Corpus_WO_verses])
classes = np.asarray(dataset['class'])
dataset = dataset.drop(columns=['verse', 'class'])
dataset_ = pd.get_dummies(dataset['code'])
X, y = shuffle(np.asarray(dataset_), classes, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
lda = LinearDiscriminantAnalysis(n_components = 1)
lda.fit(X_train, y_train)
lda.score(X_test, y_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13477653631284917, 'fraction_numerical': 0.008379888268156424, 'mean_word_length': 11.247863247863247, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '178594', 'license': 'MIT', 'n_tokens_mistral': 621, 'n_tokens_neox': 606, 'n_words': 60, 'path': 'Typology.ipynb', 'repo_name': 'RPetitpierre/Homeric_epics_quantitative_analysis', 'size': '1432'}