<commit_before>import numpy as np
import matplotlib.pyplot as plt

from ip.preprocess import create_word_mask
from search.KNN import KNN
from utils.fio import get_image_roi
from utils.transcription import get_transcription


def compute_central_heights():
    trc = get_transcription()
    wh = []
    for coord, word in trc:
        roi = get_image_roi(coord)
        msk = create_word_mask(roi)
        img = np.copy(roi)
        img = img.max() - img
        img[msk < 1] = 0
        y = img.sum(1)
        y_max = y.max()
        y[y < y_max * 0.6] = 0
        nz = y.nonzero()
        wh.append(nz[0].shape[0])

        return np.array(wh)


def compute_word_dimensions():
    knn = KNN()
    knn.parse()

    stats = dict()
    for word, img in zip(knn.train.Y, knn.train.imgs):
        if word in stats:
            stats[word].append(img)
        else:
            stats[word] = [img]

    wr = []
    hr = []
    rr = []
    for word in stats:
        w = [x[0] for x in stats[word]]
        h = [x[1] for x in stats[word]]
        r = [x[2] for x in stats[word]]

        wr.append(max(w) - min(w))
        hr.append(max(h) - min(h))
        rr.append(max(r) - min(r))

    fig = plt.figure()
    ax1 = fig.add_subplot(131)
    ax1.hist(wr, 100)
    plt.xlabel('width ranges')
    ax2 = fig.add_subplot(132)
    ax2.hist(hr, 50)
    plt.xlabel('height ranges')
    ax3 = fig.add_subplot(133)
    ax3.hist(rr)
    plt.show()


if __name__ == '__main__':
    compute_word_dimensions()<commit_msg>Improve the word image dimensions plot<commit_after>import numpy as np
import matplotlib.pyplot as plt

from ip.preprocess import create_word_mask
from search.KNN import KNN
from utils.fio import get_image_roi
from utils.transcription import get_transcription


def compute_central_heights():
    trc = get_transcription()
    wh = []
    for coord, word in trc:
        roi = get_image_roi(coord)
        msk = create_word_mask(roi)
        img = np.copy(roi)
        img = img.max() - img
        img[msk < 1] = 0
        y = img.sum(1)
        y_max = y.max()
        y[y < y_max * 0.6] = 0
        nz = y.nonzero()
        wh.append(nz[0].shape[0])

        return np.array(wh)


def compute_word_dimensions():
    knn = KNN()
    knn.parse()

    stats = dict()
    for word, img in zip(knn.train.Y, knn.train.imgs):
        if word in stats:
            stats[word].append(img)
        else:
            stats[word] = [img]

    wr = []
    hr = []
    rr = []
    for word in stats:
        h = [x[0] for x in stats[word]]
        w = [x[1] for x in stats[word]]
        r = [x[2] for x in stats[word]]

        wr.append(max(w) - min(w))
        hr.append(max(h) - min(h))
        rr.append(max(r) - min(r))

    fig = plt.figure()
    ax1 = fig.add_subplot(131)
    ax1.hist(wr, 30, normed=1, histtype='step', cumulative=1)
    ax1.margins(x=0)
    plt.xlabel('word width ranges')
    plt.grid(b=True, which='major', linestyle='-')
    plt.grid(b=True, which='minor')
    plt.minorticks_on()
    ax2 = fig.add_subplot(132)
    ax2.hist(hr, 30, normed=1, histtype='step', cumulative=1)
    ax2.margins(x=0)
    plt.grid(b=True, which='major', linestyle='-')
    plt.grid(b=True, which='minor')
    plt.minorticks_on()
    plt.xlabel('word height ranges')
    plt.title('cumulative distributions')
    ax3 = fig.add_subplot(133)
    ax3.hist(rr, normed=1, histtype='step', cumulative=1)
    ax3.margins(x=0)
    plt.xlabel('word rank range')
    plt.grid(b=True, which='major', linestyle='-')
    plt.grid(b=True, which='minor')
    plt.minorticks_on()
    plt.show()


if __name__ == '__main__':
    compute_word_dimensions()
{'fraction_non_alphanumeric': 0.14352488314544953, 'fraction_numerical': 0.020071487489689305, 'mean_word_length': 2.3747680890538034, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6912390', 'n_tokens_mistral': 1421, 'n_tokens_neox': 1367, 'n_words': 307}