<jupyter_start><jupyter_text>Table Type Classifier Training This notebook shows how to train a table type classifier on the **SegmentedTables** dataset. You can download the model weights at https://github.com/paperswithcode/axcell/releases/download/v1.0/models.tar.xz.<jupyter_code>from axcell.helpers.datasets import read_tables_annotations
from pathlib import Path

V1_URL = 'https://github.com/paperswithcode/axcell/releases/download/v1.0/'
SEGMENTED_TABLES_URL = V1_URL + 'segmented-tables.json.xz'

segmented_tables_annotations = read_tables_annotations(SEGMENTED_TABLES_URL)
from axcell.data.paper_collection import PaperCollection

SEGMENTED_TABLES_PAPERS = Path('/mnt/efs/pwc/data/arxiv/sources/segmented-tables/papers')
pc = PaperCollection.from_files(SEGMENTED_TABLES_PAPERS, annotations=segmented_tables_annotations.to_dict(orient='record'))<jupyter_output><empty_output><jupyter_text>We convert papers and annotations into a dataframe with features: table caption, content of table headers, and headlines of sections referencing the table. We later specify in ULMFiTTableTypeExperiment which of these featues to use.<jupyter_code>import pandas as pd
import re
from collections import OrderedDict
from fastai.text import *

anchor_re = re.compile(r'^xxanchor-\S* ')

# find headlines (possibly with duplicates) of sections
# referencing a given table in paper's text
def find_sections(paper, table):
    anchor = table.figure_id
    if anchor is None:
        return []
    anchor = 'xxref-' + anchor.replace('.', '')
    sections = [anchor_re.sub('', fragment.header) for fragment in paper.text.fragments if anchor in fragment.text]
    sections = [section for section in sections if section]
    return sections


def get_tabletype_features(papers, deduplicate=True):
    records = []
    dedup_fn = (lambda x: x) if not deduplicate else lambda x: list(OrderedDict.fromkeys(x))
    for paper in sorted(papers, key=lambda p: p.paper_id):
        for table in paper.tables:
            tags = table.gold_tags.split()
            record = dict(
                id=f'{paper.paper_id}/{table.name}',
                fold=paper.gold_tags,
                caption=table.caption or '',
                sota='leaderboard' in tags,
                ablation='ablation' in tags,
                sections=' ; '.join(dedup_fn(find_sections(paper, table))),
                row0=' ; '.join(dedup_fn(table.matrix.iloc[:,0])),
                col0=' ; '.join(dedup_fn(table.matrix.iloc[0,:]))
            )
            records.append(record)
    return pd.DataFrame(records)


# make sure the training dataframe is batch_size aligned
def align_df(df, batch_size):
    aligned_len = ( len(df) // batch_size ) * batch_size
    return df.iloc[:aligned_len]


def dataframes_to_databunch(sigmoid, base_path, train_df, valid_df, test_df, batch_size, processor):
    columns = []
    if sigmoid:
        label_cols = [x for x in ["sota", "ablation", "irrelevant"] if x in train_df.columns]
    else:
        label_cols = ["class"]
    text_cols = [x for x in ["caption", "sections", "row0", "col0"] if x in train_df.columns]
    columns = label_cols + text_cols
    
    train_df, valid_df, test_df = train_df[columns], valid_df[columns], test_df[columns]
    
    if len(label_cols) > 1:
        classes = label_cols
    else:
        classes = None
    train_tl = TextList.from_df(train_df, base_path, cols=text_cols, processor=processor)
    valid_tl = TextList.from_df(valid_df, base_path, cols=text_cols, processor=processor)
    test_tl  = TextList.from_df(test_df,  base_path, cols=text_cols, processor=processor)
    
    src = ItemLists(base_path, train_tl, valid_tl)\
        .label_from_df(cols=label_cols, classes=classes)
    src.add_test(test_tl)
    
    data_clas = src.databunch(bs=batch_size)
    return data_clas


def get_databunch(experiment, df, processor):
    df = experiment.transform_df(df)
    is_test = df.fold == experiment.test_split
    is_valid = df.fold == experiment.valid_split
    test_df = df[is_test]
    valid_df = df[is_valid]
    train_df = df[(~is_test) & (~is_valid)]
    train_df = align_df(train_df, experiment.BS)
    data_clas = dataframes_to_databunch(
        experiment.sigmoid, BASE_DIR, train_df, valid_df,
        test_df, experiment.BS, processor
    )
    
    return train_df, valid_df, test_df, data_clas
from axcell.models.structure.experiment import experiments_grid
from axcell.models.structure.ulmfit_experiment import ULMFiTTableTypeExperiment

EXPERIMENTS_DIR = './experiments/tabletype'
BASE_DIR = Path('./models')

processor = processor = SPProcessor(
    sp_model=BASE_DIR / 'tmp' / 'spm.model',
    sp_vocab=BASE_DIR / 'tmp' / 'spm.vocab',
    mark_fields=True
)

# parameters common for all experiments
base_experiment = ULMFiTTableTypeExperiment(
    dataset="segmented-tables",
    drop_mult=1.0,
    fp16=False,
    test_split='img_class',
    valid_split='speech_rec',
    pretrained_lm='lm',
    schedule=(
        (1, 1e-2),   # (a,b) -> fit_one_cyclce(a, b)
        (1, 5e-3/2., 5e-3),  # (a, b) -> freeze_to(-2); fit_one_cycle(a, b)
        (10, 1e-2 / (2.6 ** 4), 1e-3)  # (a, b) -> unfreeze(); fit_one_cyccle(a, b)
    ),
    sigmoid=False,
    BS=128,
    dedup_seqs=True,
    caption=True,
    first_row=False,
    first_column=False,
    referencing_sections=False
)
experiments = list(experiments_grid(
    base_experiment,
    seed=[1234, 6671347, 531609, 999999, 135792468]
))
import gc

EXPERIMENTS_DIR = './experiments/tabletype'
df_dedup = get_tabletype_features(pc, deduplicate=True)
df_nodedup = get_tabletype_features(pc, deduplicate=False)

# folds = sorted(df_dedup.fold.unique())
# folds.remove(base_experiment.test_split)

for i in range(len(experiments)):
    experiment = experiments[i]
    df = df_dedup if experiment.dedup_seqs else df_nodedup

    train_df, valid_df, test_df, data_clas = get_databunch(experiment, df, processor)
    print(f'Running experiment {i+1} / {len(experiments)}')
    model = experiment.get_trained_model(data_clas)
    experiment.evaluate(model, train_df, valid_df, test_df)
    experiment.save(EXPERIMENTS_DIR)

#     experiments[i] = None
#     experiment = None
#     gc.collect()
edf = ULMFiTTableTypeExperiment.experiments_to_df(experiments).reset_index()
edf[['valid_accuracy', 'valid_bin_accuracy', 'test_accuracy', 'test_bin_accuracy']]
best_idx = edf.valid_bin_accuracy.idxmax()
best = experiments[best_idx]
best.show_results()
best._model.export('table-type-classifier')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1123027424544201, 'fraction_numerical': 0.011031101578060365, 'mean_word_length': 3.987012987012987, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '204115', 'license': 'Apache-2.0', 'n_tokens_mistral': 2284, 'n_tokens_neox': 2194, 'n_words': 463, 'path': 'notebooks/training/table-type-classifier.ipynb', 'repo_name': 'Kabongosalomon/axcell', 'size': '6527'}