<jupyter_start><jupyter_text>Dictionary lookup and word error rates<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
sns.set(style="white")
sns.set_context("notebook", font_scale=1.2, rc={"lines.linewidth": 2.5})


db_sentence = pd.read_pickle("db_trove_sentence_with_lookup.pkl")
db_sentence.head()<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>"""
def human_ocr_percentage_remove_human_not_found(myrow, min_num_letters=4):
    corr_array = np.array(myrow["corrected_dict_lookup_list"]).astype(np.int)
    ocr_array = np.array(myrow["ocr_dict_lookup_list"]).astype(np.int)
    
    corr_array_not_found = len(corr_array[corr_array <= -min_num_letters])
    ocr_array_not_found = len(ocr_array[ocr_array <= -min_num_letters])
    really_not_found = ocr_array_not_found - corr_array_not_found
    if really_not_found < 0:
        return 100
    dict_perc = 1. - really_not_found / max(1, len(ocr_array[abs(ocr_array) >= min_num_letters]))
    return dict_perc*100
"""
def human_ocr_percentage(myrow, colname="ocr_dict_lookup_list", min_num_letters=4):
    myrow_array = np.array(myrow[colname]).astype(np.int)
    ocr_perc = len(myrow_array[myrow_array >= min_num_letters]) / max(1, len(myrow_array[abs(myrow_array) >= min_num_letters]))
    return ocr_perc*100
def jaccard_similarity_df(myrow, colname_1, colname_2, make_lowercase=True):
    """
    Jaccard similarity between two documents (e.g., OCR and Human) on flattened list of words
    """
    list1 = myrow[colname_1]
    list2 = myrow[colname_2]
    if make_lowercase:
        list1 = [x.lower() for x in list1]
        list2 = [x.lower() for x in list2]
    intersection = len(set(list1).intersection(set(list2)))
    union = (len(set(list1)) + len(set(list2))) - intersection
    return float(intersection) / union
def jaccard_similarity_df_each_sentence(myrow, colname_1, colname_2):
    """
    Jaccard similarity between two documents (e.g., OCR and Human) by averaging over all sentences
    """
    jaccard_mean = 0
    counter = 0
    for i in range(len(myrow[colname_1])):
        try:
            intersection = len(list(set(myrow[colname_1][i]).intersection(myrow[colname_2][i])))
            union = (len(myrow[colname_1][i]) + len(set(myrow[colname_2][i]))) - intersection
            jaccard_mean += float(intersection) / union
            counter += 1
        except IndexError:
            jaccard_mean += 0
            counter += 1
    return jaccard_mean / max(1, counter)<jupyter_output><empty_output><jupyter_text>OCR Dataset<jupyter_code>db_sentence['ocr_dict_lookup_list'] = \
    db_sentence.apply(lambda x: [item for sublist in x['ocr_dict_lookup'] for item in sublist], axis=1)
min_num_letters = 3
db_sentence['ocr_dict_perc'] = \
    db_sentence.apply(human_ocr_percentage, args=["ocr_dict_lookup_list", min_num_letters], axis=1)
plt.figure(figsize=(10, 10))

plt.plot(db_sentence['str_similarity'], 
         db_sentence['ocr_dict_perc'], 'ko', 
         alpha=0.1, 
        )

plt.text(0.95, 5, 'Quality band 1', fontsize=16,
        rotation=90, rotation_mode='anchor', color='r')
plt.text(0.85, 5, 'Quality band 2', fontsize=16,
        rotation=90, rotation_mode='anchor', color='r')
plt.text(0.75, 5, 'Quality band 3', fontsize=16,
        rotation=90, rotation_mode='anchor', color='r')
plt.text(0.65, 5, 'Quality band 4', fontsize=16,
        rotation=90, rotation_mode='anchor', color='r')

plt.grid()
plt.xticks([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], size=24)
plt.yticks(size=24)
plt.xlabel("String similarity (Levenshtein)", size=28)
plt.ylabel("%words in dictionary", size=28)
plt.xlim(0.0, 1.0)
plt.ylim(0, 100)
#plt.title("Words with {} letters or more".format(min_num_letters), size=32)
plt.axvline(0.9, 0, 1, c='r', ls='--')
plt.axvline(0.8, 0, 1, c='r', ls='--')
plt.axvline(0.7, 0, 1, c='r', ls='--')

#plt.legend(prop={'size': 24})
plt.show()


"""
#Other:

import seaborn as sns
current_palette = sns.color_palette()

plt.plot(db_sentence[db_sentence['quality_band'] == 1]['str_similarity'], 
         db_sentence[db_sentence['quality_band'] == 1]['ocr_dict_perc'], 'o', 
         alpha=0.1, 
         c=current_palette[0], 
         label='Quality band 1'
        )

plt.plot(db_sentence[db_sentence['quality_band'] == 2]['str_similarity'], 
         db_sentence[db_sentence['quality_band'] == 2]['ocr_dict_perc'], 'o', 
         alpha=0.1, c=current_palette[1], 
         label='Quality band 2'
        )

plt.plot(db_sentence[db_sentence['quality_band'] == 3]['str_similarity'], 
         db_sentence[db_sentence['quality_band'] == 3]['ocr_dict_perc'], 'o', 
         alpha=0.1, c=current_palette[2], 
         label='Quality band 3'
        )

plt.plot(db_sentence[db_sentence['quality_band'] == 4]['str_similarity'], 
         db_sentence[db_sentence['quality_band'] == 4]['ocr_dict_perc'], 'o', 
         alpha=0.1, c=current_palette[3], 
         label='Quality band 4'
        )
"""
db_sentence['ocr_dict_perc']<jupyter_output><empty_output><jupyter_text>Corrected Dataset<jupyter_code>db_sentence['corrected_dict_lookup_list'] = \
    db_sentence.apply(lambda x: [item for sublist in x['corrected_dict_lookup'] for item in sublist], axis=1)
min_num_letters = 3
db_sentence['corr_dict_perc'] = \
    db_sentence.apply(human_ocr_percentage, args=["corrected_dict_lookup_list", min_num_letters], axis=1)
plt.figure(figsize=(10, 10))
plt.plot(db_sentence['str_similarity'], db_sentence['corr_dict_perc'], 'ko', alpha=0.05)
plt.grid()
plt.xticks(size=24)
plt.yticks(size=24)
plt.xlabel("String similarity measure", size=28)
plt.ylabel("Words found in dictionary (%)", size=28)
plt.xlim(0.0, 1.0)
plt.ylim(0, 100)
plt.title("Words with {} letters or more".format(min_num_letters), size=32)
plt.show()<jupyter_output><empty_output><jupyter_text>Jaccard similarity, on flattened list of words<jupyter_code>db_sentence['corrected_sentencizer_list'] = \
    db_sentence.apply(lambda x: [item for sublist in x['corrected_sentencizer'] for item in sublist], axis=1)
db_sentence['ocr_sentencizer_list'] = \
    db_sentence.apply(lambda x: [item for sublist in x['ocr_sentencizer'] for item in sublist], axis=1)
db_sentence['jaccard_similarity'] = \
    db_sentence.apply(jaccard_similarity_df, args=['ocr_sentencizer_list', "corrected_sentencizer_list", True], axis=1)
plt.figure(figsize=(10, 10))

xbins = np.linspace(0, 1, 100)
ybins = np.linspace(0, 1, 100)

plt.hist2d(db_sentence['str_similarity'], db_sentence['jaccard_similarity'], 
           bins=[xbins, ybins],
           vmax=100,
           cmap=plt.cm.get_cmap('viridis', 100)
          )
plt.grid()
plt.xticks(size=24)
plt.yticks(size=24)
plt.xlabel("Levenshtein similarity", size=28)
plt.ylabel("Jaccard similarity", size=28)
plt.xlim(0.0, 1.0)
plt.ylim(0.0, 1.0)

cbar = plt.colorbar(aspect=20, orientation="vertical", fraction=0.05)
cbar.set_label('#entries',size=28)
cbar.ax.tick_params(labelsize=20) 
plt.show()

plt.figure(figsize=(10, 10))
plt.plot(db_sentence['str_similarity'], db_sentence['jaccard_similarity'], 'ko', alpha=0.01)
plt.grid()
plt.xticks(size=24)
plt.yticks(size=24)
plt.xlabel("Levenshtein similarity", size=28)
plt.ylabel("Jaccard similarity", size=28)
plt.xlim(0.0, 1.0)
plt.ylim(0, 1.0)
plt.show()
db_sentence.to_pickle("db_trove_v002.pkl")
db_sentence.head()<jupyter_output><empty_output><jupyter_text>Jaccard similarity, averaging over all sentences<jupyter_code>from pandarallel import pandarallel
# Initialization
pandarallel.initialize()
db_sentence['jaccard_similarity_mean_sentence'] = \
    db_sentence.parallel_apply(jaccard_similarity_df_each_sentence, args=['ocr_sentencizer', "corrected_sentencizer"], axis=1)
plt.figure(figsize=(10, 10))

xbins = np.linspace(0, 1, 100)
ybins = np.linspace(0, 1, 100)

plt.hist2d(db_sentence['str_similarity'], db_sentence['jaccard_similarity_mean_sentence'], 
           bins=[xbins, ybins],
           vmax=25,
           cmap=plt.cm.get_cmap('viridis', 100)
          )
plt.grid()
plt.xticks(size=24)
plt.yticks(size=24)
plt.xlabel("Levenshtein similarity", size=28)
plt.ylabel("Jaccard similarity", size=28)
plt.xlim(0.0, 1.0)
plt.ylim(0.0, 1.0)

cbar = plt.colorbar(aspect=20, orientation="vertical", fraction=0.05)
cbar.set_label('#entries',size=28)
cbar.ax.tick_params(labelsize=20) 
plt.show()
plt.figure(figsize=(10, 10))
plt.plot(db_sentence['str_similarity'], db_sentence['jaccard_similarity_mean_sentence'], 'ko', alpha=0.05)
plt.grid()
plt.xticks(size=24)
plt.yticks(size=24)
plt.xlabel("Levenshtein similarity", size=28)
plt.ylabel("Jaccard similarity", size=28)
plt.xlim(0.0, 1.0)
plt.ylim(0, 1.0)
plt.show()
df = pd.read_pickle("db_trove_v002.pkl")

for b in range(1,5):
    sns.distplot(df[df["quality_band"]==b]["jaccard_similarity"], label="Quality band %d"%b)
plt.xlabel("Word-error rate (Jaccard similarity).")
plt.legend()
plt.tight_layout()
plt.savefig("figures/topic_modelling/OCR_quality_jaccard_distr_by_band.pdf")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15676035996000445, 'fraction_numerical': 0.03799577824686146, 'mean_word_length': 5.1657534246575345, 'pattern_counts': {'":': 1, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '642150', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 3845, 'n_tokens_neox': 3411, 'n_words': 559, 'path': 'dictionary_lookup_word_errorrate.ipynb', 'repo_name': 'Living-with-machines/lwm_ARTIDIGH_2020_OCR_impact_downstream_NLP_tasks', 'size': '9001'}