<jupyter_start><jupyter_text>Comparison to SNoW> Given a set of web tables and a target knowledge base, the SNoW method extends each web table with additional context columns, stitches matching web tables into larger tables, and applies functional dependency discovery to identify the relations that are represented in the web tables. Further, it normalises the stitched tables, guided by the schema of the knowledge base, to create an integrated schema.We assume that the tables are already context-enriched, and schema-unioned per Pay-Level Domain. Our task is now to match and stitch these supertables into universal tables, and decompose them into normalised relations.  Matching without FDs<jupyter_code>%%time
from snow_pipeline import *
log.getLogger().setLevel(log.DEBUG)

snow_root = Path('~/snow/').expanduser().absolute()
kb = KB(snow_root)
%%time
from snow_pipeline import *
log.getLogger().setLevel(log.DEBUG)

snow_root = Path('~/snow/').expanduser().absolute()
benchmark_datasets = dict(get_snow_datasets(snow_root))
for name, ds in benchmark_datasets.items():
    print(f'{len(ds[0].fnames):3d}', name)

dataset_name = 'www.nndb.com'
ts = list(takco.TableSet.dataset(benchmark_datasets[dataset_name][0]))#[-50:]
print(f"Loaded {dataset_name},  {len(ts)} tables")
%%time
tabid_df = preprocess_tables(ts)
# Which features of these tables can we use to match them?
from snow_pipeline import *

predict_fkclasses(tabid_df, dataset_name, kb)
# for tabid, df in tabid_df.items():
# #     print(tabid, *('|'.join(c) for c in zip(*df.columns)))
#     for c, s in df.iteritems():
#         top = pd.Series(TfidfMatcher._analyzer(s)).value_counts()[:3]
# #         if c[0].startswith('pr'):
#         print(tabid, c, dict(top))
from snow_pipeline import *

matchers = [
    KBClassMatcher(kb, include_context=False, pred_max_threshold=0.9),
    ExactHeadMatcher(include_context=False),
    TfidfMatcher(num_threshold=0.75, min_df=2),
]
agg_func = "KBClassMatcher * @max(ExactHeadMatcher, TfidfMatcher)"
agg_threshold_col = 0

colsim, idpairs = get_colsim_and_codes(
    tabid_df, matchers,
    agg_func=agg_func, 
    agg_threshold_col=agg_threshold_col
)
partcols = cluster_columns(colsim)
partcolid_to_colids = aggr_by_val(partcols.items())
partcolid_to_colids
partid_to_tabids = partition_connected_components(tabid_df, partcolid_to_colids, idpairs)
for partid, tis in partid_to_tabids.items():
    print(f"part-{partid}", len(tis), tis)
from snow_pipeline import *

stitched = stitch_colclustered_tables(tabid_df, partcols, idpairs)
stitched_df = {}
for partid, (df, cols) in enumerate(stitched):
    df.columns = pd.MultiIndex.from_tuples(cols)
    stitched_df[f"part-{partid}"] = df
    print(*zip(*cols))
    display( df.sample( min(5, len(df)) ) )
#     if len(df.columns) > 10:
#         break
df = stitched_df['part-0']
print(len(df))
sim = kb.get_sim(df)
display(df.describe().T)
pred = pd.DataFrame({"column": list(df.columns[list(sim.columns)]), "class": sim.idxmax(), "score": sim.max()})
display( pred.style.background_gradient() )
tabid_to_colnr_and_fkclass = predict_fkclasses(stitched_df, dataset_name, kb)
tabid_to_colnr_and_fkclass
nary_induction = True
decomposed = iter_decomposed(
    stitched_df, 
    dataset_name, 
    tabid_to_colnr_and_fkclass, 
    nary=nary_induction, 
    nary_stoplevel=2,
    nary_minp= 0.95
)
for t in postprocess_tables(decomposed, numeric_threshold=0.5):
    print(f"{t._id}: {len(t.df)} rows")
    display( t.df.sample(min(len(t.df), 3)) )<jupyter_output>DEBUG:root:[itunes.apple.com] [part-0] Decomposing class Single for col 10 (('nome',))
DEBUG:root:[itunes.apple.com] [part-0] Not decomposing context columns [('page title',), ('uri 1 (se)',), ('disambiguation of name',), ('table heading',), ('disambiguation of album',), ('uri 0',), ('uri 3',), ('uri 2 (album)',), ('uri 1 (music video)',), ('uri 2 (music video)',), ('uri 2 (artist)',), ('uri 2',), ('uri 1',), ('uri 1 (gb)',), ('uri 1 (podcast)',), ('uri 2 (podcast)',), ('disambiguation of page title',), ('uri 1 (hn)',)]
DEBUG:root:[itunes.apple.com] [part-0] Inferring FDs for [page title|NULL|duração|uri 0|uri 2 (album)|preço|artista|FK|name|album|interpret|álbum]
FD candidates:  46%|████▋     | 31/67 [00:29<00:27,  1.30it/s]
{'fraction_non_alphanumeric': 0.14752941176470588, 'fraction_numerical': 0.014588235294117647, 'mean_word_length': 5.6421875, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '329692', 'license': 'MIT', 'n_tokens_mistral': 1577, 'n_tokens_neox': 1501, 'n_words': 367, 'path': 'docs/benchmarks/compare-snow.ipynb', 'repo_name': 'karmaresearch/takco', 'size': '4250'}