<jupyter_start><jupyter_text>Create dataframes from raw dataThis notebook shows how a dataframe was created for two of the datasets we used for this project (2kIMPACT and DBNL). See more info, see the [Wiki page](https://github.com/Giovanni1085/KB_OCR_impact/wiki/Datasets).Please note this notebook is only illustrative of the procedure we have followed to prepare the datasets. Throughout the project, we have made use of the [dataframes with evaluation data](evaluation/) provided by the KB. These contain the same dataset but include an extensive [assessment of OCR quality](https://github.com/Giovanni1085/KB_OCR_impact/wiki/Evaluations).<jupyter_code>from glob import glob
import jellyfish
import numpy as np
import os, codecs, re
import pandas as pd
from bs4 import BeautifulSoup
%matplotlib inline
import matplotlib.pyplot as plt

# magics and warnings
%load_ext autoreload
%autoreload 2
import warnings; warnings.simplefilter('ignore')

import random
from tqdm.notebook import tqdm
import pandas as pd
import numpy as np

seed = 43
random.seed(seed)
np.random.seed(seed)

import nltk, gensim, sklearn, spacy
from gensim.models import CoherenceModel
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style="white")
sns.set_context("notebook", font_scale=1.2, rc={"lines.linewidth": 2.5})<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>2k IMPACT<jupyter_code># you will need to change this to your own location.
folder = os.path.join(os.environ['HOME'],"surfdrive/Shared/OCR")

# data folders
ocr_original_folder = "Original ALTO files " # ABBY 8
ocr_new_folder = "ReOCRed (FR11)" # ABBY 11
ground_folder = "Reordered Ground-truth"
def get_contents(soup):
    """
    This function reads a file and gets original filename, tokens and their confidence out of an alto xml file
    """
    filename = soup.find("filename").text
    # find all strings (== tokens) and get text plus word-level confidence
    word_level_confidence = list()
    text = list()
    for match in soup.findAll("string"):
        word_level_confidence.append(float(match["wc"]))
        text.append(match["content"])
    return filename, text, word_level_confidence
# limits how many files to load
# this dataset has 2000 files, so any number above it loads them all
LIMIT = 3000<jupyter_output><empty_output><jupyter_text>LOAD ground truth texts<jupyter_code>data_dict = {'filename':list(),'text_ground':list()}

# load xml files and get data out
for root, dirnames, filenames in os.walk(os.path.join(folder,ground_folder)):
    for f in tqdm(filenames[:LIMIT]):
        if ".xml" in f:
            full_path = os.path.join(root,f)
            soup = BeautifulSoup(codecs.open(full_path,encoding="utf8"))
            filename, text, word_level_confidence = get_contents(soup)
            data_dict['filename'].append(filename)
            data_dict['text_ground'].append(text)
# create a dataframe
df_ground = pd.DataFrame.from_dict(data_dict)
df_ground = df_ground.set_index('filename')
df_ground.head()<jupyter_output><empty_output><jupyter_text>LOAD OCR texts<jupyter_code>data_dict_ocr_1 = {'filename':list(),'text_ocr':list(),'word_level_confidence_ocr':list()}
data_dict_ocr_2 = {'filename':list(),'text_reocr':list(),'word_level_confidence_reocr':list()}

# load xml files and get data out
for root, dirnames, filenames in os.walk(os.path.join(folder,ocr_original_folder)):
    for f in tqdm(filenames[:LIMIT]):
        if ".xml" in f:
            full_path = os.path.join(root,f)
            soup = BeautifulSoup(codecs.open(full_path,encoding="utf8"))
            filename, text, word_level_confidence = get_contents(soup)
            data_dict_ocr_1['filename'].append(filename)
            data_dict_ocr_1['word_level_confidence_ocr'].append(word_level_confidence)
            data_dict_ocr_1['text_ocr'].append(text)
for root, dirnames, filenames in os.walk(os.path.join(folder,ocr_new_folder)):
    for f in tqdm(filenames[:LIMIT]):
        if ".xml" in f:
            full_path = os.path.join(root,f)
            soup = BeautifulSoup(codecs.open(full_path,encoding="utf8"))
            filename, text, word_level_confidence = get_contents(soup)
            data_dict_ocr_2['filename'].append(filename)
            data_dict_ocr_2['word_level_confidence_reocr'].append(word_level_confidence)
            data_dict_ocr_2['text_reocr'].append(text)
flatten = lambda l: [item for sublist in l for item in sublist]
l = flatten(data_dict_ocr_1['word_level_confidence_ocr'])
sns.distplot(l, label="Word confidence oroginal OCR", kde=False, color="red")
plt.legend()
plt.tight_layout()
plt.savefig("figures/2kIMPACT_word_confidence_orOCR.pdf")
l2 = flatten(data_dict_ocr_2['word_level_confidence_reocr'])
sns.distplot(l2, label="Word confidence re-OCR", kde=False, color="blue")
plt.legend()
plt.tight_layout()
plt.savefig("figures/2kIMPACT_word_confidence_reOCR.pdf")
# create dataframes
df_ocr = pd.DataFrame.from_dict(data_dict_ocr_1)
df_ocr = df_ocr.set_index('filename')
df_reocr = pd.DataFrame.from_dict(data_dict_ocr_2)
df_reocr = df_reocr.set_index('filename')
df_ocr.head()
df_reocr.head()
# join data frames
full_df = pd.concat([df_ground, df_ocr], join="inner", axis=1)
full_df = pd.concat([full_df, df_reocr], join="inner", axis=1)
full_df.shape
full_df.head()
full_df["word_level_average_confidence_ocr"] = full_df["word_level_confidence_ocr"].apply(lambda x: np.average(x))
full_df["word_level_average_confidence_reocr"] = full_df["word_level_confidence_reocr"].apply(lambda x: np.average(x))
# compare original and re-OCR
sns.distplot(full_df["word_level_average_confidence_ocr"].values, label="Original OCR", kde=False, color="red", bins=30)
sns.distplot(full_df["word_level_average_confidence_reocr"].values, label="Re-OCR", kde=False, color="blue", bins=30)
plt.title("Word average confidence (document-level)")
plt.legend()
plt.tight_layout()
plt.savefig("figures/2kIMPACT_word_confidence.pdf")
# save
full_df.to_csv("data_frames/full_df_2kIMPACT.csv", compression="gzip")<jupyter_output><empty_output><jupyter_text>DBNL OCR<jupyter_code># you will need to change this to your own location.
folder = os.path.join(os.environ['HOME'],"surfdrive/projects/KB_residence_OCR_2020/data/DBNL_OCR_dataset")

# data folders
ocr_folder = "TXT"
ground_folder = "TEI"
metadata_file = "Metadata_DBNL_OCR_v1.csv"
# load ground data by filtering for Dutch only

import requests

data_dict = {'filename':list(),'text_ground':list(),'genre':list(),'subgenre':list()}
# we use the DBNL API to get genre and subgenre, we will use this for the classification task
baseline_url = "https://dbnl.org/extern/api/titel/read_one.php"

# load xml files and get data out
for root, dirnames, filenames in os.walk(os.path.join(folder,ground_folder)):
    for f in tqdm(filenames):
        if ".xml" in f:
            full_path = os.path.join(root,f)
            soup = BeautifulSoup(codecs.open(full_path,encoding="utf8"), 'html.parser') # GET the correct encoding!
            filename = f.replace("_01.xml","")
            tei_lang = soup.find("language")
            if tei_lang: # many books do not have language information, in that case we assume it's Dutch
                if not tei_lang.text == "Nederlands":
                    continue
            text = soup.find("body").text
            # find genre and subgenre
            response = requests.get(
                baseline_url,
                params={'ti_id': filename})
            response = response.json()
            genre = None
            if "genre" in response.keys():
                genre = ",".join([x[1] for x in response["genre"]])
            subgenre = None
            if "subgenre" in response.keys():
                subgenre = ",".join([x[1] for x in response["subgenre"]])
            data_dict['filename'].append(filename)
            data_dict['text_ground'].append(text)
            data_dict['genre'].append(genre)
            data_dict['subgenre'].append(subgenre)
df_ground = pd.DataFrame.from_dict(data_dict)
df_ground = df_ground.set_index('filename')
df_ground.head()
df_ground["genre"].value_counts()
df_ground["subgenre"].value_counts()[:5]
df_ground.shape
# load OCRed data for the necessary files

data_dict = {'filename':list(),'text_ocr':list()}

# load xml files and get data out
for f_name in df_ground.index.values:
    full_path = os.path.join(folder,ocr_folder)
    f = os.path.join(full_path,f_name+"_01.txt")
    text = codecs.open(f, encoding="utf8", errors="ignore").read() # We ignore errors outside of UTF-8!
    data_dict['filename'].append(f_name)
    data_dict['text_ocr'].append(text)
# create dataframes
df_ocr = pd.DataFrame.from_dict(data_dict)
df_ocr = df_ocr.set_index('filename')
df_ocr.head()
# lastly, get the metadata in (title and year)

df_meta = pd.read_csv(os.path.join(folder,metadata_file), sep=";")
df_meta = df_meta.rename(columns={"ti_id":"filename","titel":"title","jaar":"year","druk":"edition"})
df_meta = df_meta.set_index("filename")
df_meta.head()
# merge dataframes
full_df = pd.concat([df_ground, df_ocr], join="inner", axis=1)
full_df.head()
full_df = pd.concat([full_df, df_meta], join="inner", axis=1)
full_df.head()
full_df.to_csv("data_frames/full_df_DBNL_OCR.csv", compression="gzip")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12458418285223737, 'fraction_numerical': 0.008048073827663912, 'mean_word_length': 4.693341478313989, 'pattern_counts': {'":': 6, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '626407', 'license': 'MIT', 'n_tokens_mistral': 3179, 'n_tokens_neox': 3089, 'n_words': 695, 'path': 'Data frames - setup.ipynb', 'repo_name': 'Giovanni1085/KB_OCR_impact', 'size': '9319'}