<jupyter_start><jupyter_text>Predict Tinnitus Diagnostic Based on NPL of hybrid patient data_find relevant hidden features based on standard questionnaires and free-text description_ Load configuration<jupyter_code>import os
PATH = os.getcwd() 
import sys
sys.path.append(PATH + '/../')
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import LatentDirichletAllocation
import numpy as np
import pandas as pd
from utils.config import Config
import pyLDAvis
import pyLDAvis.sklearn as sklearnvis
import pickle
pyLDAvis.enable_notebook()

print("Config loaded")<jupyter_output>Config loaded<jupyter_text>Load and visualize csv data with panda<jupyter_code># read entire data set
df = pd.read_csv(Config.csv_files[0], sep=';', encoding='ISO-8859-1')

# load additional info
df_add = pd.read_csv(Config.additional_text[0], sep=';', encoding='ISO-8859-1')

# merge courrier and info
df["courrier_add"] = df["courrier"].astype(str) + df_add["0"]
print("done")
courriers=df["courrier_add"]# this is what we should spare
courriers_cleaned=[courrier for courrier in courriers if isinstance(courrier,str)]
stop_words = Config.stop_words
print(stop_words)
tf_vectorizer = CountVectorizer(encoding='ISO-8859-1',
                                strip_accents = None,
                                stop_words = stop_words,
                                lowercase = True,
                                token_pattern = r'\<KEY>',
                                max_df = 0.5, 
                                min_df = 1)
dtm_tf = tf_vectorizer.fit_transform(courriers_cleaned)
print(dtm_tf.shape)
pickle.dump(tf_vectorizer, open('../data/tf_vectorizer.pk', 'wb'))
pickle.dump(dtm_tf, open('../data/dtm_tf.pk', 'wb'))
tfidf_vectorizer = TfidfVectorizer(**tf_vectorizer.get_params())
dtm_tfidf = tfidf_vectorizer.fit_transform(courriers_cleaned)
print(dtm_tfidf.shape)
pickle.dump(tfidf_vectorizer, open('../data/tfidf_vectorizer.pk', 'wb'))
pickle.dump(dtm_tfidf, open('../data/dtm_tfidf.pk', 'wb'))

# for TF DTM
n_components = 10
lda_tf = LatentDirichletAllocation(n_components=n_components, random_state=42)
lda_tf.fit(dtm_tf)
pickle.dump(lda_tf, open('../data/lda_tf.pk', 'wb'))


# FROM HERE YOU CAN WORK WITHOUT TRAINING (JUST LOADING)
lda_tf = pickle.load( open('../data/lda_tf.pk', "rb" ) )
tf_vectorizer = pickle.load( open('../data/tf_vectorizer.pk', "rb" ) )
dtm_tf = pickle.load( open('../data/dtm_tf.pk', "rb" ) )
vis_data = sklearnvis.prepare(lda_tf, dtm_tf, tf_vectorizer)
pyLDAvis.display(vis_data)
# for TFIDF DTM
lda_tfidf = LatentDirichletAllocation(n_components=n_components, random_state=0)
lda_tfidf.fit(dtm_tfidf)
pickle.dump(lda_tf, open('../data/lda_tfidf.pk', 'wb'))
lda_tfidf = pickle.load( open('../data/lda_tfidf.pk', "rb" ) )
tfidf_vectorizer = pickle.load( open('../data/tfidf_vectorizer.pk', "rb" ) )
dtm_tfidf = pickle.load( open('../data/dtm_tfidf.pk', "rb" ) )
vis_data = sklearnvis.prepare(lda_tfidf, dtm_tfidf, tfidf_vectorizer)
pyLDAvis.display(vis_data)
def predict(sentence):
    new_vector =tf_vectorizer.transform([sentence])
    return lda_tf.transform(new_vector)

print(predict("j'étais à un concert et c'est survenu brusquement"))##categorie 1
print(predict("J'entends l'acouphène qui suit le rythme de mon coeur"))##categorie 2
print(predict("J'ai des problèmes d'équilibre et des vertiges"))##categorie 3
print(predict("Je suis appareillé et j'ai du mal à entendre"))##categorie 4
print(predict("J'ai eu beaucoup de stresse"))##categorie 5
print(predict("j'ai des problème de congestion nasal et les oreilles bouchées"))##categorie 7<jupyter_output>[[0.02500783 0.02500283 0.02500326 0.50391711 0.025      0.02500097
  0.02500144 0.02500495 0.29605957 0.02500203]]
[[0.02       0.02       0.02000152 0.02000026 0.02       0.02000481
  0.29175214 0.02000136 0.0200008  0.54823912]]
[[0.05000759 0.05000098 0.05001145 0.05001756 0.05000007 0.05001363
  0.05000258 0.05001278 0.05000382 0.54992955]]
[[0.69993204 0.03333333 0.03333674 0.03333888 0.03333333 0.03333914
  0.03335682 0.03334695 0.03334478 0.03333797]]
[[0.03334089 0.03333334 0.03333777 0.03333494 0.36666796 0.03334291
  0.36661175 0.03335329 0.03334116 0.03333599]]
[[0.02500099 0.025      0.02500113 0.02500242 0.025      0.77499034
  0.02500219 0.02500036 0.02500184 0.02500074]]
{'fraction_non_alphanumeric': 0.12779700115340253, 'fraction_numerical': 0.12410611303344868, 'mean_word_length': 4.4267834793491865, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '664089', 'license': 'MIT', 'n_tokens_mistral': 2011, 'n_tokens_neox': 1647, 'n_words': 325, 'path': 'notebooks/sklearn.ipynb', 'repo_name': 'lkorczowski/cleandata', 'size': '4345'}