<jupyter_start><jupyter_text>Declare configs<jupyter_code>OUTPUT_DIR = 'election-polls'
FILENAME = '../data/election-polls-single-sheet-comparison.xlsx'
ANNOTATION_FILE = '../data/election-polls_colombia.json'

CE_MODEL = '../models/ce.model'
FE_MODEL = '../models/fe.model'
CL_MODEL = '../models/cl.model'
W2V = '../models/glove.840B.300d.txt'
INFERSENT_MODEL = '../models/infersent1.pkl'
VOCAB_SIZE = 60000<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>annotations = {}
sheets = {}

with open(ANNOTATION_FILE) as f:
    annotations['colombia'] = json.load(f)

for sheet_name in pd.ExcelFile(FILENAME).sheet_names:
    sheet_df = pd.read_excel(FILENAME, sheet_name=sheet_name, engine='openpyxl', index_col=None, header=None)

    sheet = None
    if sheet_name in annotations:
        sheet = Sheet(sheet_df, annotations[sheet_name])
    else:
        sheet = Sheet(sheet_df)

    sheets[sheet_name] = sheet<jupyter_output><empty_output><jupyter_text>Load model<jupyter_code>model = CellEmbeddingModelWrapper(CE_MODEL, FE_MODEL, CL_MODEL, W2V, VOCAB_SIZE, INFERSENT_MODEL)<jupyter_output>loading word vectors...
loading word embeddings...
creating dict...
embeddings loaded!<jupyter_text>Create embeddings<jupyter_code>embeddings = model.predict_labels(FILENAME)<jupyter_output>initialize 40 text sentences...<jupyter_text>Remove unlabelled data<jupyter_code>print('Started saving sheet pair data')
block_detection_output = f'../output/block_detection/{OUTPUT_DIR}'

if os.path.exists(block_detection_output):
    shutil.rmtree(block_detection_output)

sheet_pairs = combinations(embeddings.keys(), 2)
for sheet_pair in sheet_pairs:
    dir_name = os.path.join(block_detection_output, " & ".join(sheet_pair))
    os.makedirs(dir_name)

    labels = text = coordinates = cell_embeddings = []
    ids = types = roles = []

    metadata = [['sheet', 'text', 'label', 'coordinates']]
    has_annotations = False
    # if any([sheets[s].annotations for s in sheet_pair]):
    #     has_annotations = True
    #     metadata = [['sheet', 'text', 'label', 'coordinates', 'id', 'role', 'type']]

    for s in sheet_pair:
        if has_annotations:
            sheet = sheets[s]
            ids = sheet.represent_transformed_annotations('id').to_numpy()  #.reshape(shape[0] * shape[1])
            types = sheet.represent_transformed_annotations('type').to_numpy()  #.reshape(shape[0] * shape[1])
            roles = sheet.represent_transformed_annotations('role').to_numpy()  #.reshape(shape[0] * shape[1])
        for i in range(0, len(embeddings[s]['table_arrays'])):
            for j in range(0, len(embeddings[s]['table_arrays'][i])):
                value = embeddings[s]['table_arrays'][i][j]
                # Filter based on value
                # if value.replace('.', '', 1).isdigit():
                #     continue


                if value != 'None' and value != '':
                    text = value.replace('\n', '')
                else:
                    continue

                cell_embeddings.append(list(embeddings[s]['embeddings'][i][j]))
                if has_annotations:
                    metadata.append([s, text, embeddings[s]['labels'][i][j], f'|{i + 1}_{j + 1}|', ids[i][j], roles[i][j], types[i][j]])
                else:
                    metadata.append([s, text, embeddings[s]['labels'][i][j], f'|{i + 1}_{j + 1}|'])

                assert len(cell_embeddings) + 1 == len(metadata)

    print('Saving:', sheet_pair)
    np.savetxt(os.path.join(dir_name, 'embeddings.tsv'), cell_embeddings, delimiter='\t')
    np.savetxt(os.path.join(dir_name, 'metadata.tsv'), metadata, delimiter='\t', fmt='%s')<jupyter_output>Started saving sheet pair data
Saving: ('colombia_copy', 'colombia')
{'fraction_non_alphanumeric': 0.1525514293347582, 'fraction_numerical': 0.008282126636387924, 'mean_word_length': 3.5602923264311817, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '47772', 'license': 'MIT', 'n_tokens_mistral': 1310, 'n_tokens_neox': 1236, 'n_words': 213, 'path': 'notebooks/block_detection_new.ipynb', 'repo_name': 'abhinav-kumar-thakur/TabularCellTypeClassification', 'size': '3743'}