<jupyter_start><jupyter_text>Read in<jupyter_code>articles_en = [open(f"C:/Users/mgoedicke/Desktop/Python/Notebooks/Untitled Folder/1971_9_en/{file}","r",encoding = "utf8").read()
                  for file in os.listdir("C:/Users/mgoedicke/Desktop/Python/Notebooks/Untitled Folder/1971_9_en/")
                  if file.endswith(".txt")
            ]

articles_fr = [open(f"C:/Users/mgoedicke/Desktop/Python/Notebooks/Untitled Folder/1971_9_fr/{file}","r",encoding = "utf8").read()
                  for file in os.listdir("C:/Users/mgoedicke/Desktop/Python/Notebooks/Untitled Folder/1971_9_fr/")
                  if file.endswith(".txt")
            ]


all_issues_en = [open(f"C:/Users/mgoedicke/Desktop/Python/Notebooks/Whole_Korpus/{file}",mode="r",encoding = "utf8").read()
                for file in os.listdir("C:/Users/mgoedicke/Desktop/Python/Notebooks/Whole_Korpus")
                if file.endswith("en.xml")
            ]
all_issues_en = " ".join(all_issues_en)

all_issues_fr = [open(f"C:/Users/mgoedicke/Desktop/Python/Notebooks/Whole_Korpus/{file}",mode="r",encoding = "utf8").read()
                for file in os.listdir("C:/Users/mgoedicke/Desktop/Python/Notebooks/Whole_Korpus")
                if file.endswith("fr.xml")
            ]
all_issues_fr = " ".join(all_issues_fr)


words_to_delete = ["<div>","</div>","\n","\t"]
for word_to_delete in words_to_delete:
    articles_en = [article.replace(word_to_delete,"") for article in articles_en]
    articles_fr = [article.replace(word_to_delete,"") for article in articles_fr]
for word_to_delete in words_to_delete:
    all_issues_en = all_issues_en.replace(word_to_delete,"")
    all_issues_fr = all_issues_fr.replace(word_to_delete,"")<jupyter_output><empty_output><jupyter_text>Helper methods for word importances<jupyter_code>from nltk.stem import SnowballStemmer

def get_clean_text(text, language):
    to_delete = ['"',",",".","?","!","“","”","'","(",")",":","‘","%","-","+","/","—"]
    for item in to_delete:
        text = text.replace(item,"")
    
    to_replace_with_whitespace = ["´","’"]
    for item in to_replace_with_whitespace:
        text = text.replace(item," ")
        
    stemmer = SnowballStemmer(language)
    text = " ".join([stemmer.stem(word) for word in text.split()])
    return text

def get_tfidf_score(word,document,reference_documents):
    tf = document.count(word) / len(document.split(" "))
    idf = math.log ( len(reference_documents) 
                    / (1 + len([reference_document for reference_document in reference_documents if word in reference_document]))
                   )    
    return tf * idf

def get_word_importances(text, reference_texts, cutoff = 10):
    #set of tfidf-scores per word 
    text_word_importances=set([(word,get_tfidf_score(word ,text ,reference_texts)) 
                               for word in text.split(" ") 
                               if word != ""
                              ]
                             )

    #sort
    text_word_importances = sorted(text_word_importances, key = lambda x: x[1], reverse = True) 
    
    #only the words, not the scores, of the top 10 words per article
    text_word_importances =  [item[0] for item in text_word_importances[:10]]
    
    return text_word_importances<jupyter_output><empty_output><jupyter_text>Compute word importances (commented take longer)<jupyter_code>articles_en2 = [get_clean_text(article, "english") for article in articles_en ]
articles_fr2 = [get_clean_text(article, "french") for article in articles_fr ]

word_importances_reference_articles_en = [get_word_importances(article, articles_en2) for article in articles_en2]
word_importances_reference_articles_fr = [get_word_importances(article, articles_fr2) for article in articles_fr2]

#issues_en2 = [get_clean_text(issue, "english") for issue in all_issues_en ]
#issues_fr2 = [get_clean_text(issue, "french") for issue in all_issues_fr ]

#word_importances_reference_issues_en = [get_word_importances(article, issues_en2) for article in articles_en2]
#word_importances_reference_issues_fr = [get_word_importances(article, issues_fr2) for article in articles_en2]<jupyter_output><empty_output><jupyter_text>Create DataFrame<jupyter_code>column_values_en = list(zip(articles_en
                            ,word_importances_reference_articles_en
                           # ,word_importances_reference_issues_en
                        )
                    )
column_values_fr = list(zip(articles_fr
                            ,word_importances_reference_articles_fr
                           # ,word_importances_reference_issues_fr
                        )
                    )

df_en = pd.DataFrame(column_values_en, columns = ["text","word_importances_compare_issue"])#,"word_importances_compare_all_issues"])
df_fr = pd.DataFrame(column_values_fr, columns = ["text","word_importances_compare_issue"])#,"word_importances_compare_all_issues"])<jupyter_output><empty_output><jupyter_text>Helper methods syntactic feature generation<jupyter_code>def get_occurring_interesting_numbers(text):
    pattern = "\d+[\%\€\$£¥]?"
    return list(set(re.findall(pattern,text)))

def get_occuring_elements(text,reference):
    return list(set([element for element in reference if element in text]))
    
def get_jaccard_distance(list_1,list_2):
    
    set_1 = set(list_1)
    set_2 = set(list_2)
    
    length_union = len(set_1.union(set_2))
    if length_union == 0:
        return 0
    else:
        return round(len(set_1.intersection(set_2)) / length_union,4)

def get_z_score(series):
    mean = series.mean()
    #std = series.std()
    return series.map(lambda cnt: (cnt-series.mean()) / series.std())
def add_metadata_features(df):
    df["cnt_characters"] = df.text.map(lambda text: len(text))
    df["cnt_words"] = df.text.map(lambda text: len(re.split("\s",text)))
    df["cnt_sentences"] = df.text.map(lambda text: len(re.split("[\?\.\!]",text)))
    df["cnt_questions"] = df.text.map(lambda text: len(re.split("[\?]",text))-1)
    df["cnt_exclamations"] = df.text.map(lambda text: len(re.split("[!?]",text))-1)


    df["cnt_characters_z_score"] = get_z_score(df.cnt_characters) 
    df["cnt_words_z_score"] = get_z_score(df.cnt_words)
    df["cnt_sentences_z_score"] = get_z_score(df.cnt_sentences)
    df["cnt_questions_z_score"] = get_z_score(df.cnt_questions)
    df["cnt_exclamations_z_score"] = get_z_score(df.cnt_exclamations)
    
    df["interesting_numbers"] = df.text.map(lambda text: get_occurring_interesting_numbers(text))
    df["ankerwords"] = df.text.map(lambda text: get_occuring_elements(text,anker_words_countries))
    df["constant"] = 1
add_metadata_features(df_en)
add_metadata_features(df_fr)
df_en[:3]
df_fr[:3]
# constant for cross join
df = df_en.merge(df_fr, on = ["constant"], suffixes = ("_EN","_FR"))
df[:3]<jupyter_output><empty_output><jupyter_text>Compute distanceFirst draft, lot of space for improvement<jupyter_code>df["distance"] =(
    abs(df.cnt_characters_z_score_EN - df.cnt_characters_z_score_FR)
    + abs(df.cnt_words_z_score_EN - df.cnt_words_z_score_FR)
    + abs(df.cnt_sentences_z_score_EN - df.cnt_sentences_z_score_FR)
    + abs(df.cnt_questions_z_score_EN - df.cnt_questions_z_score_FR)
    + abs(df.cnt_exclamations_z_score_EN - df.cnt_exclamations_z_score_FR)
) * (1 - df.apply(lambda row: get_jaccard_distance(row.interesting_numbers_EN, row.interesting_numbers_FR), axis = 1))<jupyter_output><empty_output><jupyter_text>View results<jupyter_code>pd.set_option('display.max_colwidth', 1000)
df[["text_EN","text_FR","distance"]].sort_values(by="distance")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13202357563850686, 'fraction_numerical': 0.0091683038637852, 'mean_word_length': 4.040264026402641, 'pattern_counts': {'":': 1, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '269681', 'license': 'MIT', 'n_tokens_mistral': 2745, 'n_tokens_neox': 2615, 'n_words': 386, 'path': 'Article Matching - Prototype.ipynb', 'repo_name': 'DominikStefancik/Bulletins-Articles-Parser', 'size': '7635'}