<jupyter_start><jupyter_text>This notebook converts the unicode labels to tokens, pads the converted labels with ``, `` and the `` tokens.<jupyter_code>from google.colab import drive
drive.mount('/content/drive', force_remount=True)
import unicodedata
import tarfile
import pandas as pd
from tqdm import tqdm
# tarfile_loc = '/content/drive/My Drive/Project STation/TD/recognition_real_dataset/cropped_data.tar'
tarfile_loc = '/content/drive/My Drive/Project STation/TD/synthetic_cropped.tar.gz'

with tarfile.open(tarfile_loc) as tarf:
    tarf.extractall()
# These ranges are in hexadecimal format
def _unicode_to_idx_map(lower_range, upper_range, start_index):
    l_range = int(lower_range, 16) 
    u_range = int(upper_range, 16)
    name2idx = dict()
    idx = start_index
    for char_code in range(l_range, u_range + 1):
        code_point = chr(char_code)
        char_name = unicodedata.name(code_point)
        name2idx[char_name] = idx
        idx = idx + 1
    return name2idx, idx
start_index = 3  ## Start from 3 as the first three inidices are reserved for <start> <stop> and <PAD> token
name2idx, idx = _unicode_to_idx_map("0900", "094D", start_index)
digits, idx = _unicode_to_idx_map("0966", "096F", idx)
om, idx = _unicode_to_idx_map("0950", "0950", idx)
additional, idx = _unicode_to_idx_map("0958", "095F", idx)

name2idx.update(digits)
name2idx.update(om)
name2idx.update(additional)
def get_labels(text, name2idx):
    labels = list()
    for char in text:
        name = unicodedata.name(char)
        index = name2idx[name]
        labels.append(index)
    return labels
# annotation_file_loc = '/content/cropped_data/annotations.txt'
annotation_file_loc = '/content/cropped_dir/annotation.txt'
with open(annotation_file_loc) as fp:
    # find the max length of the data labels for padding
    labels = list()
    for line in fp.readlines():
        hindi_text = line.split('\t')[1].strip()
        try:
            labels.append(get_labels(hindi_text, name2idx))
        except KeyError:
            pass
    max_len = max([len(label) for label in labels]) + 2  # including start and stop symbols
cols = ['name', 'text', 'labels', 'unpadded_length']
annotation_df = pd.DataFrame()
with open(annotation_file_loc) as fp:
    data = {}
    for line in tqdm(fp.readlines()):
        img_path, hindi_text = line.split('\t')[:2]
        img_path = img_path.strip()
        hindi_text = hindi_text.strip()
        data['name'] = img_path
        data['text'] = f"<start> {hindi_text} <stop>"
        label = [0]  ## Initialize with the <start> token
        try:
            label.extend(get_labels(hindi_text, name2idx))
        except KeyError:
            continue
        label.append(1)  ## End with the <stop> token
        data['unpadded_length'] = int(len(label))
        extra_padding = max_len - len(label)
        label.extend([2] * extra_padding)
        data['labels'] = label
        annotation_df = annotation_df.append(data, ignore_index=True)
print(annotation_df.head())
annotation_df.to_pickle('/content/drive/My Drive/Project STation/TD/annotation_synthetic_preprocessed.pkl')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1151033386327504, 'fraction_numerical': 0.017170111287758347, 'mean_word_length': 3.660740740740741, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '500322', 'license': 'MIT', 'n_tokens_mistral': 1084, 'n_tokens_neox': 1041, 'n_words': 249, 'path': 'project_station_td_preprocess.ipynb', 'repo_name': 'kpandey008/signboard-transliteration', 'size': '3145'}