Add

tabs/exploration_tab.py

@@ -94,169 +94,169 @@ full_txt_fr = load_data(dataPath+'/small_vocab_fr')
 if not st.session_state.reCalcule:
     full_txt_en, full_txt_split_en, full_txt_lem_en, full_txt_wo_stopword_en, full_df_count_word_en = load_all_preprocessed_data('en')
     full_txt_fr, full_txt_split_fr, full_txt_lem_fr, full_txt_wo_stopword_fr, full_df_count_word_fr = load_all_preprocessed_data('fr')
+else:
 
+    def remove_stopwords(text, lang): 
+        stop_words = set(stopwords.words(lang))
+        # stop_words will contain  set all english stopwords
+        filtered_sentence = []   
+        for word in text.split(): 
+            if word not in stop_words: 
+                filtered_sentence.append(word) 
+        return " ".join(filtered_sentence)
 
-def remove_stopwords(text, lang): 
-    stop_words = set(stopwords.words(lang))
-    # stop_words will contain  set all english stopwords
-    filtered_sentence = []   
-    for word in text.split(): 
-        if word not in stop_words: 
-            filtered_sentence.append(word) 
-    return " ".join(filtered_sentence)
+    def clean_undesirable_from_text(sentence, lang):
 
-def clean_undesirable_from_text(sentence, lang):
-
-    # Removing URLs 
-    sentence  = re.sub(r"https?://\S+|www\.\S+", "", sentence )
+        # Removing URLs 
+        sentence  = re.sub(r"https?://\S+|www\.\S+", "", sentence )
     
-    # Removing Punctuations (we keep the . character)
-    REPLACEMENTS = [("..", "."),
-                    (",", ""),
-                    (";", ""),
-                    (":", ""),
-                    ("?", ""),
-                    ('"', ""),
-                    ("-", " "),
-                    ("it's", "it is"),
-                    ("isn't","is not"),
-                    ("'", " ")
-                   ]
-    for old, new in REPLACEMENTS:
-        sentence = sentence.replace(old, new)
+        # Removing Punctuations (we keep the . character)
+        REPLACEMENTS = [("..", "."),
+                        (",", ""),
+                        (";", ""),
+                        (":", ""),
+                        ("?", ""),
+                        ('"', ""),
+                        ("-", " "),
+                        ("it's", "it is"),
+                        ("isn't","is not"),
+                        ("'", " ")
+                        ]
+        for old, new in REPLACEMENTS:
+            sentence = sentence.replace(old, new)
     
-    # Removing Digits
-    sentence= re.sub(r'[0-9]','',sentence)
+        # Removing Digits
+        sentence= re.sub(r'[0-9]','',sentence)
     
-    # Removing Additional Spaces
-    sentence = re.sub(' +', ' ', sentence)
+        # Removing Additional Spaces
+        sentence = re.sub(' +', ' ', sentence)
 
-    return sentence
+        return sentence
 
-def clean_untranslated_sentence(data1, data2):
-    i=0
-    while i<len(data1):
-        if data1[i]==data2[i]:
-            data1.pop(i)
-            data2.pop(i)
-        else: i+=1
-    return data1,data2
+    def clean_untranslated_sentence(data1, data2):
+        i=0
+        while i<len(data1):
+            if data1[i]==data2[i]:
+                data1.pop(i)
+                data2.pop(i)
+            else: i+=1
+        return data1,data2
 
-import spacy
+    import spacy
 
-nlp_en = spacy.load('en_core_web_sm')
-nlp_fr = spacy.load('fr_core_news_sm')
+    nlp_en = spacy.load('en_core_web_sm')
+    nlp_fr = spacy.load('fr_core_news_sm')
 
 
-def lemmatize(sentence,lang):
-    # Create a Doc object
-    if lang=='en':
-        nlp=nlp_en
-    elif lang=='fr':
-        nlp=nlp_fr
-    else: return
-    doc = nlp(sentence)
+    def lemmatize(sentence,lang):
+        # Create a Doc object
+        if lang=='en':
+            nlp=nlp_en
+        elif lang=='fr':
+            nlp=nlp_fr
+        else: return
+        doc = nlp(sentence)
 
-    # Create list of tokens from given string
-    tokens = [] 
-    for token in doc:
-        tokens.append(token)
+        # Create list of tokens from given string
+        tokens = [] 
+        for token in doc:
+            tokens.append(token)
 
-    lemmatized_sentence = " ".join([token.lemma_ for token in doc])
+        lemmatized_sentence = " ".join([token.lemma_ for token in doc])
  
-    return lemmatized_sentence
+        return lemmatized_sentence
 
 
-def preprocess_txt (data, lang): 
+    def preprocess_txt (data, lang): 
   
-    word_count = collections.Counter()
-    word_lem_count = collections.Counter()
-    word_wosw_count = collections.Counter()
-    corpus = []
-    data_split = []
-    sentence_length = []
-    data_split_wo_stopwords = []
-    data_length_wo_stopwords = []
-    data_lem = []
-    data_lem_length = []
+        word_count = collections.Counter()
+        word_lem_count = collections.Counter()
+        word_wosw_count = collections.Counter()
+        corpus = []
+        data_split = []
+        sentence_length = []
+        data_split_wo_stopwords = []
+        data_length_wo_stopwords = []
+        data_lem = []
+        data_lem_length = []
     
-    txt_en_one_string= ". ".join([s for s in data])
-    txt_en_one_string = txt_en_one_string.replace('..', '.')
-    txt_en_one_string = " "+clean_undesirable_from_text(txt_en_one_string, 'lang')
-    data = txt_en_one_string.split('.')
-    if data[-1]=="":
-        data.pop(-1)
-    for i in range(len(data)): # On enleve les ' ' qui commencent et finissent les phrases 
-        if data[i][0] == ' ':
-            data[i]=data[i][1:]
-        if data[i][-1] == ' ':
-            data[i]=data[i][:-1]
-    nb_phrases = len(data)
+        txt_en_one_string= ". ".join([s for s in data])
+        txt_en_one_string = txt_en_one_string.replace('..', '.')
+        txt_en_one_string = " "+clean_undesirable_from_text(txt_en_one_string, 'lang')
+        data = txt_en_one_string.split('.')
+        if data[-1]=="":
+            data.pop(-1)
+        for i in range(len(data)): # On enleve les ' ' qui commencent et finissent les phrases 
+            if data[i][0] == ' ':
+                data[i]=data[i][1:]
+            if data[i][-1] == ' ':
+                data[i]=data[i][:-1]
+        nb_phrases = len(data)
     
-    # Création d'un tableau de mots (sentence_split)
-    for i,sentence in enumerate(data):
-        sentence_split = word_tokenize(sentence)
-        word_count.update(sentence_split)
-        data_split.append(sentence_split)
-        sentence_length.append(len(sentence_split))
-
-    # La lemmatisation et le nettoyage des stopword va se faire en batch pour des raisons de vitesse
-    # (au lieu de le faire phrase par phrase)
-    # Ces 2 processus nécéssitent de connaitre la langue du corpus
-    if lang == 'en': l='english'
-    elif lang=='fr': l='french'
-    else: l="unknown"
-
-    if l!="unknown":
-        # Lemmatisation en 12 lots (On ne peut lemmatiser + de 1 M de caractères à la fois)
-        data_lemmatized=""
-        if lemmatize_to_do:
-            n_batch = 12
-            batch_size = round((nb_phrases/ n_batch)+0.5)
-            for i in range(n_batch):
-                to_lem = ".".join([s for s in data[i*batch_size:(i+1)*batch_size]])
-                data_lemmatized = data_lemmatized+"."+lemmatize(to_lem,lang).lower()
-
-            data_lem_for_sw = data_lemmatized[1:]  
-            data_lemmatized = data_lem_for_sw.split('.')
-            for i in range(nb_phrases):
-                data_lem.append(data_lemmatized[i].split())
-                data_lem_length.append(len(data_lemmatized[i].split()))
-                word_lem_count.update(data_lem[-1])
-                               
-        # Elimination des StopWords en un lot
-        # On élimine les Stopwords des phrases lémmatisés, si cette phase a eu lieu
-        # (wosw signifie "WithOut Stop Words")
-        if stopwords_to_do:
+        # Création d'un tableau de mots (sentence_split)
+        for i,sentence in enumerate(data):
+            sentence_split = word_tokenize(sentence)
+            word_count.update(sentence_split)
+            data_split.append(sentence_split)
+            sentence_length.append(len(sentence_split))
+
+        # La lemmatisation et le nettoyage des stopword va se faire en batch pour des raisons de vitesse
+        # (au lieu de le faire phrase par phrase)
+        # Ces 2 processus nécéssitent de connaitre la langue du corpus
+        if lang == 'en': l='english'
+        elif lang=='fr': l='french'
+        else: l="unknown"
+
+        if l!="unknown":
+            # Lemmatisation en 12 lots (On ne peut lemmatiser + de 1 M de caractères à la fois)
+            data_lemmatized=""
             if lemmatize_to_do:
-                data_wosw = remove_stopwords(data_lem_for_sw,l)
-            else:
-                data_wosw = remove_stopwords(txt_en_one_string,l)
+                n_batch = 12
+                batch_size = round((nb_phrases/ n_batch)+0.5)
+                for i in range(n_batch):
+                    to_lem = ".".join([s for s in data[i*batch_size:(i+1)*batch_size]])
+                    data_lemmatized = data_lemmatized+"."+lemmatize(to_lem,lang).lower()
+
+                data_lem_for_sw = data_lemmatized[1:]  
+                data_lemmatized = data_lem_for_sw.split('.')
+                for i in range(nb_phrases):
+                    data_lem.append(data_lemmatized[i].split())
+                    data_lem_length.append(len(data_lemmatized[i].split()))
+                    word_lem_count.update(data_lem[-1])
                                
-            data_wosw = data_wosw.split('.')
-            for i in range(nb_phrases):
-                data_split_wo_stopwords.append(data_wosw[i].split())
-                data_length_wo_stopwords.append(len(data_wosw[i].split()))
-                word_wosw_count.update(data_split_wo_stopwords[-1])
+            # Elimination des StopWords en un lot
+            # On élimine les Stopwords des phrases lémmatisés, si cette phase a eu lieu
+            # (wosw signifie "WithOut Stop Words")
+            if stopwords_to_do:
+                if lemmatize_to_do:
+                    data_wosw = remove_stopwords(data_lem_for_sw,l)
+                else:
+                    data_wosw = remove_stopwords(txt_en_one_string,l)
+                               
+                data_wosw = data_wosw.split('.')
+                for i in range(nb_phrases):
+                    data_split_wo_stopwords.append(data_wosw[i].split())
+                    data_length_wo_stopwords.append(len(data_wosw[i].split()))
+                    word_wosw_count.update(data_split_wo_stopwords[-1])
 
-    corpus = list(word_count.keys())
+        corpus = list(word_count.keys())
    
-    # Création d'un DataFrame txt_n_unique_val :
-    #      colonnes = mots
-    #      lignes = phases
-    #      valeur de la cellule = nombre d'occurence du mot dans la phrase
+        # Création d'un DataFrame txt_n_unique_val :
+        #      colonnes = mots
+        #      lignes = phases
+        #      valeur de la cellule = nombre d'occurence du mot dans la phrase
     
-    ## BOW
-    from sklearn.feature_extraction.text import CountVectorizer
-    count_vectorizer = CountVectorizer(analyzer="word", ngram_range=(1, 1), token_pattern=r"[^' ']+" )
+        ## BOW
+        from sklearn.feature_extraction.text import CountVectorizer
+        count_vectorizer = CountVectorizer(analyzer="word", ngram_range=(1, 1), token_pattern=r"[^' ']+" )
     
-    # Calcul du nombre d'apparition de chaque mot dans la phrases
-    countvectors = count_vectorizer.fit_transform(data)
-    corpus = count_vectorizer.get_feature_names_out()
+        # Calcul du nombre d'apparition de chaque mot dans la phrases
+        countvectors = count_vectorizer.fit_transform(data)
+        corpus = count_vectorizer.get_feature_names_out()
 
-    txt_n_unique_val=  pd.DataFrame(columns=corpus,index=range(nb_phrases), data=countvectors.todense()).astype(float)     
+        txt_n_unique_val=  pd.DataFrame(columns=corpus,index=range(nb_phrases), data=countvectors.todense()).astype(float)     
     
-    return data, corpus, data_split, data_lemmatized, data_wosw, txt_n_unique_val, sentence_length, data_length_wo_stopwords, data_lem_length      
+        return data, corpus, data_split, data_lemmatized, data_wosw, txt_n_unique_val, sentence_length, data_length_wo_stopwords, data_lem_length      
  
 
 def count_world(data):