<jupyter_start><jupyter_text>Testing bounding<jupyter_code>import sys; sys.path.append('..')
import cadence as cd
from cadence.imports import *
line="Of that Forbidden Tree, whose mortal tast"
# cd.scan(line,phrasebreaks=False)
dfparses=cd.parse(line,phrasebreaks=False)
dfparses
for linedf in cd.iter_lines(line,phrasebreaks=False): pass
# linedf
linedf_parsed=cd.do_iter_parsed_lines(linedf)

def postproc_dfline(dfline,by_line=False,only_best=False,only_unbound=True):
    odf=to_lines(dfline) if by_line else sort_by_total_and_syll(dfline)
    if only_unbound: odf=bound_parses(odf)
    if only_best and 'parse_rank' in set(odf.index.names) or 'parse_rank' in set(odf.columns):
        odf=odf.query('parse_rank==1')
    return odf

def is_bounded(pdf1,pdf2,totalcol=TOTALCOL,cprefix='*'):
    cdf1=pdf1[[c for c in pdf1.columns if c.startswith(cprefix) and c!=totalcol]]
    cdf2=pdf2[[c for c in pdf2.columns if c.startswith(cprefix) and c!=totalcol]]
    c1=cdf1.sum()
    c2=cdf2.sum()
    c1=c1[c1>0]
    c2=c2[c2>0]
    set1,set2=set(c1.index),set(c2.index)
    return set1.issubset(set2), set2.issubset(set1)


def bound_parses(odf):
    grps=sorted(list(odf.groupby('parse_rank')))
    not_ok_ranks=set()
    for parse_rank1,parsedf1 in grps:
        for parse_rank2,parsedf2 in grps:
            if parse_rank1>=parse_rank2: continue
            x_bounds_y, y_bounds_x = is_bounded(parsedf1,parsedf2)
            if y_bounds_x: not_ok_ranks|={parse_rank1}
            if x_bounds_y: not_ok_ranks|={parse_rank2}
    return odf[~odf.index.get_level_values('parse_rank').isin(not_ok_ranks)]
odf=postproc_dfline(linedf_parsed)
odf<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12222887956860395, 'fraction_numerical': 0.023367285799880167, 'mean_word_length': 5.185185185185185, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '420698', 'license': 'MIT', 'n_tokens_mistral': 690, 'n_tokens_neox': 698, 'n_words': 107, 'path': 'notebooks/test-bounding.ipynb', 'repo_name': 'quadrismegistus/cadence', 'size': '1669'}