<jupyter_start><jupyter_text>User Input<jupyter_code>import numpy as np
import pandas as pd
import dill as pickle
import os
from nltk.tokenize import sent_tokenize
import pytesseract
from PIL import Image, ImageEnhance, ImageFilter
from collections import defaultdict
import shutil


# TODO remove this 
import warnings
warnings.filterwarnings("ignore")



# from .utils import unpickle
# TODO Move to utils 
def unpickle(filename):
    """ Unpickle file """
    with open(filename, 'rb') as f:
        return pickle.load(f)


# data_path = '../../data/raw/word_level'
# meta_json_data_path = '../../data/preprocessed/meta.json'
# word_level_meta_path = '../../data/preprocessed/word_level_meta.csv'
# word_path_mapping_path = '../../data/processed/word_path_mapping.pkl'
# X_path = '../../data/processed/X.npy'
# y_path = '../../data/processed/y.npy'

# X_wiki2_path = '../../data/processed/X_wiki2.npy'
# y_wiki2_path = '../../data/processed/y_wiki2.npy'

# bigram_model_path = '../../data/processed/ngram_models/bigram_likelihood_model.pkl'
# trigram_model_path = '../../data/processed/ngram_models/trigram_likelihood_model.pkl'
# letters_path = '../../data/processed/letters_map.pkl'


# DATA_DIR = '../../../data'

DATA_DIR = '../../data'
# TODO move this 
data_path = os.path.join(DATA_DIR, 'raw/word_level')
meta_json_data_path = os.path.join(DATA_DIR, 'preprocessed/meta.json')
word_level_meta_path = os.path.join(DATA_DIR, 'preprocessed/word_level_meta.csv')
word_path_mapping_path = os.path.join(DATA_DIR, 'processed/word_path_mapping.pkl')
X_path = os.path.join(DATA_DIR, 'processed/X.npy')
y_path = os.path.join(DATA_DIR, 'processed/y.npy')
bigram_model_path = os.path.join(DATA_DIR, 'processed/ngram_models/bigram_likelihood_model.pkl')
trigram_model_path = os.path.join(DATA_DIR, 'processed/ngram_models/trigram_likelihood_model.pkl')
letters_path = os.path.join(DATA_DIR, 'processed/letters_map.pkl')



def ngram_backoff_model(sample_index, X, y, trigram_model, bigram_model, OOV_token=0):
    # get index and target word
    index = int(y[sample_index][1][0])
    target = X[sample_index][index]
    # get previous word(s)
    try:
        prev_word = X[sample_index][index-1]
    except:
        prev_word = '<bos>'
    try:
        prev_prev_word = X[sample_index][index-2]
    except:
        prev_prev_word = '<bos>'
    # model preds 
    pred = trigram_model[(prev_prev_word, prev_word)]
    if pred == OOV_token:
        pred = bigram_model[(prev_word)]
        if pred == OOV_token:
            pred = 'UNK'
    return pred 


def teseract_baseline(sample_index, y, word_path_mapping, letters, tmpdir='tmp/'):
    img_width = 256
    img_height = 100
    # gets image from sample index
    img = word_path_mapping[y[sample_index][0]]
    print(img)
    im = Image.open(img)  # img is the path of the image 
    im = im.convert("RGBA")
    im = im.resize((img_width, img_height))
    newimdata = []
    datas = im.getdata()

    vals = 255

    for item in datas:
        if item[0] < vals or item[1] < vals or item[2] < vals:
            newimdata.append(item)
        else:
            newimdata.append((255, 255, 255))
    im.putdata(newimdata)

    im = im.filter(ImageFilter.MedianFilter())
    enhancer = ImageEnhance.Contrast(im)
    im = enhancer.enhance(2)
    im = im.convert('1')
    
    if not os.path.exists(tmpdir):
        os.makedirs(tmpdir)
    save_img_path = os.path.join(tmpdir, 'temp_img.jpg')
    im.save(save_img_path)
    
    text = pytesseract.image_to_string(Image.open(save_img_path),
                config='-c tessedit_char_whitelist=0123456789abcdefghijklmnopqrstuvwxyz -psm 13', lang='eng')
    
    # os.remove(save_img_path)
    # os.rmdir(tmpdir)
    shutil.rmtree(tmpdir)
    return text, len(text) 


    
def get_ocr_model_pred(sample_index, y, word_path_mapping, letters):
    ocr_pred, len_pred = teseract_baseline(sample_index, y, word_path_mapping, letters)
    return ocr_pred, len_pred

def get_language_model_pred(sample_index, X, y, trigram_model, bigram_model):
    pred = ngram_backoff_model(sample_index, X, y, trigram_model, bigram_model)
    return pred


def get_pos_tags(sample_index):
    return []

def weight_features(sample_index, X, y, trigram_model, bigram_model, word_path_mapping, letters, weights={}):
    ocr_pred, len_pred = get_ocr_model_pred(sample_index, y, word_path_mapping, letters)
    lm_pred = get_language_model_pred(sample_index, X, y, trigram_model, bigram_model)
    pos_pred = get_pos_tags(sample_index)
    
    return ocr_pred, len_pred, lm_pred




def absoluteFilePaths(directory):
    """Walk filepaths"""
    for dirpath,_,filenames in os.walk(directory):
        for f in filenames:
            yield os.path.join(dirpath, f)

def create_image_path(df, data_path, use_s3=False):
    """Create dictionary for mapping of word to data path"""
    if use_s3:
        [f.key for f in files[:5] if '.png' in f.key]
    all_paths = [i for i in absoluteFilePaths(data_path)]
    all_path_endings = [i.split('/')[-1].split('.')[0] for i in all_paths]
    all_path_dict = defaultdict(lambda: 0, dict(zip(all_path_endings, all_paths)))
    df['image_path'] = df['image_name'].map(lambda x: all_path_dict[x])
    return df



def main(sample_index):
    

    # dataset 
    X = list(np.load(X_path))
    y = np.load(y_path)
    # Language Models
    bigram_model = unpickle(bigram_model_path)
    trigram_model = unpickle(trigram_model_path)

    # Meta
    meta = pd.read_json(meta_json_data_path)
    word_level_df = pd.read_csv(word_level_meta_path)
    word_level_df = create_image_path(word_level_df, data_path)

    # word_path_mapping = defaultdict(lambda: 0, dict(zip(word_level_df.token, word_level_df.image_path)))                              
    # with open(os.path.join(DATA_DIR, 'processed', 'word_path_mapping.pkl'), 'wb') as f:
    #     pickle.dump(word_path_mapping, f)

    word_path_mapping = unpickle(word_path_mapping_path)
    letter2idx = unpickle(letters_path)

    letters = list(letter2idx.keys())
    letters = ''.join(letters[1:-2])


    ocr_pred, len_pred, lm_pred = weight_features(sample_index, X, y, trigram_model, bigram_model, word_path_mapping, letters)
    return ocr_pred, len_pred, lm_pred


if __name__ == '__main__':
    # TODO Update for user input!!!!!
    sample_index = 102
    main(sample_index)<jupyter_output>/Users/mattevanoff/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88
  return f(*args, **kwds)
/Users/mattevanoff/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88
  return f(*args, **kwds)
{'fraction_non_alphanumeric': 0.12427788475781366, 'fraction_numerical': 0.011701970078506888, 'mean_word_length': 3.812544547398432, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '590052', 'license': 'MIT', 'n_tokens_mistral': 2529, 'n_tokens_neox': 2415, 'n_words': 466, 'path': 'build/notebooks/Pipeline.ipynb', 'repo_name': 'tuzi123/https-github.com-tuzi123-HandwritingDetection', 'size': '6751'}