<jupyter_start><jupyter_text>Data Loading<jupyter_code>def load_bigrams(path):

    file = open(path, 'r')

    bigrams = {}
    all_oc = 0
    
    for line in tqdm(file):

        oc, word_1, word_2 = line.split()
        oc = int(oc)

        if int(oc) < 5: continue

        if word_2[-1] == "\n": word_2 = word2[:-1]
        all_oc += oc

        if (word_1, word_2) in bigrams.keys():
            bigrams[(word_1, word_2)] += oc
        else:
            bigrams[(word_1, word_2)] = oc

    return bigrams, all_oc
def load_supertags(path):

    file = open(path, 'r')
    word_to_tag = {}

    for line in tqdm(file):

        word, tag = line.split(" ")
        if tag[-1] == "\n": tag = tag[:-1]

        word_to_tag[word.lower()] = tag
    
    file.close()
    return word_to_tag
def load_unigrams(path):

    file = open(path, 'r')
    word_to_occurrence = {}
    all_oc = 0

    for line in tqdm(file):

        words = line.lower().split(" ")
        
        for word in words:

            all_oc += 1
            if word[-1] == "\n": word = word[:-1]

            if word in word_to_occurrence.keys():
                word_to_occurrence[word] += 1
            else:
                word_to_occurrence[word] = 1

    return word_to_occurrence, all_oc
def supertags_stats(unigrams, bigrams, word_to_tag):

    unitags = {}
    unitags_all_oc = 0
    bitags = {}
    bitags_all_oc = 0

    for unigram in unigrams.keys():

        if unigram in word_to_tag.keys():
            unitags_all_oc += unigrams[unigram]
            if word_to_tag[unigram] in unitags.keys():
                unitags[word_to_tag[unigram]] += unigrams[unigram] 
            else:
                unitags[word_to_tag[unigram]] = unigrams[unigram]

    for bigram in bigrams.keys():
        
        if bigram[0] in word_to_tag.keys() and bigram[1] in word_to_tag.keys():
             bitags_all_oc += bigrams[bigram]

             tag_1 = word_to_tag[bigram[0]]
             tag_2 = word_to_tag[bigram[1]]

             if (tag_1, tag_2) in bitags.keys():
                bitags[(tag_1, tag_2)] += bigrams[bigram]
             else:
                bitags[(tag_1, tag_2)] = bigrams[bigram]

    return unitags, unitags_all_oc, bitags, bitags_all_oc
bigrams, bigrams_all_oc = load_bigrams('/content/poleval_2grams.txt')
unigrams, unigrams_all_oc = load_unigrams('/content/polish_corpora.txt')
word_to_tag = load_supertags('/content/drive/My Drive/Colab Notebooks/NLP/Dane/Copy of supertags.txt')
unitags, unitags_all_oc, bitags, bitags_all_oc = supertags_stats(unigrams, bigrams, word_to_tag)<jupyter_output><empty_output><jupyter_text>Colocation testing<jupyter_code>def pmi(word_1, word_2):

    oc = bigrams[(word_1, word_2)]

    res = 0
    if word_1 in unigrams.keys() and word_2 in unigrams.keys():
        res = math.log((oc/bigrams_all_oc)/((unigrams[word_1]/unigrams_all_oc)*
                                          (unigrams[word_2]/unigrams_all_oc)))
    return res
def psm(word_1, word_2):

    oc = bigrams[(word_1, word_2)]

    res = 0
    if word_1 in unigrams.keys() and word_2 in unigrams.keys():
        f_xy = oc
        fhat_xy = unigrams[word_1]*unigrams[word_2]/unigrams_all_oc
        res = f_xy * (math.log(f_xy) - fhat_xy - 1)
    
    return res
def t_student(word_1, word_2):

    oc = bigrams[(word_1, word_2)]

    res = 0
    if word_1 in unigrams.keys() and word_2 in unigrams.keys():
        f_xy = oc/bigrams_all_oc 
        fhat_xy = ((unigrams[word_1]/unigrams_all_oc) * (unigrams[word_2]/unigrams_all_oc))
    
        res = (f_xy - fhat_xy) / math.sqrt(f_xy)
    
    return res
def pmiwords_and_pmitags(word_1, word_2):

    pmi_words = pmi(word_1, word_2)

    pmi_tags = 0
    if word_1 in word_to_tag.keys() and word_2 in word_to_tag.keys():

        tag_1 = word_to_tag[word_1]
        tag_2 = word_to_tag[word_2]

        oc = bitags[(tag_1, tag_2)]

        pmi_tags = (oc/bitags_all_oc)/((unitags[tag_1]/unitags_all_oc)*
                                            (unitags[tag_2]/unitags_all_oc))
        if pmi_tags != 0:
            pmi_tags = math.log(pmi_tags)

    return pmi_words + max(0, pmi_tags)
def best_col(f, word):

    candidates = [(word1, word2) for word1, word2 in bigrams.keys() if word1==word or word2==word]
    results = {}

    for candidate in candidates:

        results[candidate] = f(candidate[0],candidate[1]) 

    return sorted(results.items(), key=lambda x: x[1], reverse=True)
words = ['koń', 'ojciec', 'dom', 'poduszka', 'czerwony', 'pisać', 'koszula', 'kot', 'kobieta', 'pies']
pd.set_option("display.max_rows", None, "display.max_columns", None)

for word in words:

    best_pmi = best_col(pmi, word)[:7]
    best_psm = best_col(psm, word)[:7]
    best_t_student = best_col(t_student, word)[:7]
    best_pmitags = best_col(pmiwords_and_pmitags, word)[:7]


    multicol_pmi = pd.MultiIndex.from_tuples([('PMI', 'Points'), ('PMI', 'Pair')])
    multicol_psm = pd.MultiIndex.from_tuples([('PSM', 'Points'), ('PSM', 'Pair')])
    multicol_pmitags = pd.MultiIndex.from_tuples([('PMITAGS', 'Points'), ('PMITAGS', 'Pair')])
    multicol_t_student = pd.MultiIndex.from_tuples([('T_student', 'Points'), ('T_student', 'Pair')])

    print('------------------------------------------ {} ------------------------------------------'.format(word))
    display(pd.concat([pd.DataFrame(best_pmi, columns=multicol_pmi), pd.DataFrame(best_psm, columns=multicol_psm), 
                     pd.DataFrame(best_pmitags, columns=multicol_pmitags), pd.DataFrame(best_t_student, columns=multicol_t_student)], axis = 1))<jupyter_output>------------------------------------------ koń ------------------------------------------
{'fraction_non_alphanumeric': 0.16093777542746343, 'fraction_numerical': 0.017098536929314297, 'mean_word_length': 2.459756097560976, 'pattern_counts': {'":': 3, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '517193', 'license': 'MIT', 'n_tokens_mistral': 2194, 'n_tokens_neox': 2066, 'n_words': 390, 'path': 'Assignment2/Collocations.ipynb', 'repo_name': 'Sahcim/UWRNLP2020', 'size': '5673'}