<jupyter_start><jupyter_text>TODO:1. use htb (plo) model to predict (single,multi,morph-yap)1. create hybrid datasets using htb model1. use htb morph model to predict on hybrid1. train morph/multi models on the hybrid   1. use htb (plo) model to predict (single,multi,morph-yap)<jupyter_code>erdf = pd.read_pickle('final_setup/plo_erdf.pkl')
output_folder = 'naama_reeval/plo/decode_output'
decode_conf_folder = 'naama_reeval/plo/decode_conf'
models_folder = 'final_setup/plo_models/'
import os
plo_data_folder = '../NER/data/for_ncrf/naama_plo'

if not os.path.exists(output_folder):
    os.makedirs(output_folder)

if not os.path.exists(decode_conf_folder):
    os.makedirs(decode_conf_folder)
    

decode_sets = {
    'token': {
  'token_1_test': 'naama_plo.test.1.bmes',
  'token_2_test': 'naama_plo.test.2.bmes',
  'token_3_test': 'naama_plo.test.3.bmes'},
 'morph': {
  'morph_1_test': 'morph/naama_plo_morph_yap.test.1.bmes',
  'morph_2_test': 'morph/naama_plo_morph_yap.test.2.bmes',
  'morph_3_test': 'morph/naama_plo_morph_yap.test.3.bmes'},
 'multitok': {
  'token_1_test': 'naama_plo.test.1.bmes',
  'token_2_test': 'naama_plo.test.2.bmes',
  'token_3_test': 'naama_plo.test.3.bmes'}}


params = { 'status': 'decode' }

for i, row in erdf.iterrows():
    unit = row['unit']
    for name, set_path in decode_sets[unit].items():
        row_par = params.copy()
        row_par['load_model_dir'] = os.path.join(models_folder, row['model_file_name'])
        row_par['dset_dir'] = os.path.join(models_folder, row['dset_file_name'])
        row_par['decode_dir'] = os.path.join(output_folder, name+'.'+row['model_base_name']+'.bmes')
        row_par['raw_dir'] = os.path.join(plo_data_folder, set_path)
        
        conf_path = os.path.join(decode_conf_folder, name+'.'+row['model_base_name']+'.decode.conf')
        if not os.path.exists(conf_path):
            with open(conf_path, 'w', encoding='utf8') as of:
                for k, v in row_par.items():
                    of.write(k+'='+str(v)+'\n')<jupyter_output><empty_output><jupyter_text>Evaluate decoded folder<jupyter_code>scores = {}
if os.path.exists('naama_reeval/basic_scores.pkl'):
    scores = pickle.load(open('naama_reeval/basic_scores.pkl', 'rb'))
for file in os.scandir(output_folder):
    if file.name=='.ipynb_checkpoints':
        continue
    gold_name, inp, arch, w_embed, seed_num = file.name.split('.')[:-1]
    if (gold_name, inp, arch, w_embed, seed_num) not in scores:
        if gold_name in ('dev_dir', 'test_dir'):
            gold_name = '_'.join(inp.split('_')[:2])+'_'+gold_name.split('_')[0]
        if len(gold_name.split('_'))>2:
            unit, cv_num, pred_set = gold_name.split('_')
            gold_path = os.path.join(plo_data_folder, decode_sets[inp][gold_name])
        else:
            unit, pred_set = gold_name.split('_')
            gold_path = decode_sets[unit][unit+'_'+pred_set]
        p, r, f = nem.evaluate_files(gold_path, file, str_join_char='')
        scores[(gold_name, inp, arch, w_embed, seed_num)] = (p, r, f)
    
import pickle
pickle.dump(scores, open('naama_reeval/basic_scores.pkl', 'wb'))
score_tups = [(*k, *v) for k,v in scores.items()]
import numpy as np 
def perc(n):
    def perc_(x):
        return np.percentile(x, n)
    perc_.__name__ = 'perc_%s' % n
    return perc_
mev = pd.DataFrame(score_tups, columns=('gold_name', 'unit', 'arch', 
                                        'w_embed', 'seed_num', 
                                        'p_m', 'r_m', 'f_m'))
(mev[mev.gold_name.str.contains('test')].groupby(['gold_name', 'unit', 'arch'])
 .f_m.agg(['max', 'min', 'mean', 'std', 'median', perc(95)]))
(mev[mev.gold_name.str.contains('dev')].groupby(['gold_name', 'unit', 'arch']).size())
mev.head()
mev['pred_set'] = mev.gold_name.apply(lambda x: '_'.join(x.split('_')[1:]))
mev['pred_set_sub'] = mev.pred_set.apply(lambda x: x.split('_')[0] if '_' in x else '')
mev['pred_set_main'] = mev.pred_set.apply(lambda x: x.split('_')[1] )
mev['unit_main'] = mev.unit.apply(lambda x: x.split('_')[0] )
mev = mev.merge(erdf, how='left')
x = (mev[mev.pred_set.str.contains('test')].groupby(['unit_main', 'pred_set_main', 'arch', 'embed_type', 'cm'])
 .f_m.agg(['mean', 'std']).mul(100).round(2)
 .assign(mean = lambda x: x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(9))).round(1).astype(str))[['mean']]
 .unstack([-2,-1]))
x
x = (mev[mev.pred_set.str.contains('test')].groupby(['unit_main', 'pred_set_main', 'arch', 'embed_type', 'cm'])
 .p_m.agg(['mean', 'std']).mul(100).round(2)
 .assign(mean = lambda x: x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(9))).round(1).astype(str))[['mean']]
 .unstack([-2,-1]))
x
x = (mev[mev.pred_set.str.contains('test')].groupby(['unit_main', 'pred_set_main', 'arch', 'embed_type', 'cm'])
 .r_m.agg(['mean', 'std']).mul(100).round(2)
 .assign(mean = lambda x: x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(9))).round(1).astype(str))[['mean']]
 .unstack([-2,-1]))
x
x = (mev[mev.pred_set.str.contains('dev')].groupby(['unit_main', 'pred_set_main', 'arch', 'embed_type', 'cm'])
 .f_m.agg([ 'mean', 'std']).mul(100).round(2)
 .assign(std = lambda x: (1.96*(x['std']/np.sqrt(9))).round(1))
 .unstack([-2,-1]))
x.columns = x.columns.reorder_levels([1,2,0])
pd.set_option("max_columns", 30)
x.sort_index(axis=1)

mev[(mev.unit_main=='multitok')&(mev.pred_set=='1_test')].sort_values('f_m', ascending=False).head(3)
mev[(mev.unit_main=='multitok')&(mev.pred_set=='2_test')].sort_values('f_m', ascending=False).head(3)
mev[(mev.unit_main=='multitok')&(mev.pred_set=='3_test')].sort_values('f_m', ascending=False).head(3)<jupyter_output><empty_output><jupyter_text>Align morphemes<jupyter_code>def get_biose_count(path, sent_id_shift=1):
    sents = nem.read_file_sents(path, fix_multi_tag=False, sent_id_shift=sent_id_shift)
    bc = []
    for i, sent in sents.iteritems():
        for j, (tok, bio) in enumerate(sent):
            bc.append([i, j+1, tok, bio, len(bio.split('^'))])

    bc = pd.DataFrame(bc, columns=['sent_id', 'token_id', 'token_str', 
                                   'biose', 'biose_count'])
    return bc
import networkx as nx
def get_valid_edges(lattices, bc,
                    non_o_only=True, keep_all_if_no_valid=True):
    valid_edges = []
    for (i, df), (_, biose, biose_count) in zip(lattices.groupby(['sent_id', 'token_id']), 
                                                bc[['biose', 'biose_count']].itertuples()):
        el = df[['ID1', 'ID2']].rename(columns={'ID1': 'source', 'ID2': 'target'})
        #min_node = [n for n,v in G.nodes(data=True) if v['since'] == 'December 2008'][0]

        g = nx.from_pandas_edgelist(el, create_using=nx.DiGraph)
        min_node = el.source.min()
        max_node = el.target.max()
        #print(min_node,max_node)
        #print(biose_count)
        if non_o_only and not '-' in biose:
            vp = list(nx.all_simple_paths(g, min_node, max_node))
        else:
            vp = [path for path in nx.all_simple_paths(g, min_node, max_node, cutoff=biose_count+1) if len(path)==biose_count+1]
        if keep_all_if_no_valid and len(vp)==0:
             vp = nx.all_simple_paths(g, min_node, max_node)
        for path in vp:
            for source, target in zip(path[:-1], path[1:]):
                valid_edges.append((i[0], i[1], source, target))
                
    return valid_edges
def to_lattices(df, path, cols = ['ID1', 'ID2', 'form', 'lemma', 'upostag', 'xpostag', 'feats', 'token_id']):
    with open(path, 'w', encoding='utf8') as of:
        for _, sent in df.groupby('sent_id'):
            for _, row in sent[cols].iterrows():
                of.write('\t'.join(row.astype(str).tolist())+'\n')
            of.write('\n')
            
    
def prune_lattices(lattices_path, ner_pred_path, output_path, keep_all_if_no_valid=True):
    lat = bclm.read_lattices(lattices_path)
    bc = get_biose_count(ner_pred_path, sent_id_shift=1)
    valid_edges = get_valid_edges(lat, bc, non_o_only=False, keep_all_if_no_valid=keep_all_if_no_valid)
    cols = ['sent_id', 'token_id', 'ID1', 'ID2']
    pruned_lat = lat[lat[cols].apply(lambda x: tuple(x), axis=1).isin(valid_edges)]
    to_lattices(pruned_lat, output_path)
prune_lattices('../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.1.tokens.lattices', 
               'naama_reeval/plo/decode_output/token_1_test.multitok.char_lstm.ft_oov_tok.50_seed.bmes',
               '../NER/data/tokens_for_ncrf/naama/pruned/naama.test.1.tokens.pruned.lattices')
prune_lattices('../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.2.tokens.lattices', 
               'naama_reeval/plo/decode_output/token_2_test.multitok.char_lstm.ft_oov_tok.47_seed.bmes',
               '../NER/data/tokens_for_ncrf/naama/pruned/naama.test.2.tokens.pruned.lattices')
prune_lattices('../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.3.tokens.lattices', 
               'naama_reeval/plo/decode_output/token_3_test.multitok.char_lstm.ft_oov_tok.45_seed.bmes',
               '../NER/data/tokens_for_ncrf/naama/pruned/naama.test.3.tokens.pruned.lattices')
yap_path = '/home/nlp/danb/yapproj/src/yap/yap'
!export GOPATH=/home/dan/yapproj
!{yap_path}
pruned_folder = '../NER/data/tokens_for_ncrf/naama/pruned/'
for file in os.scandir(pruned_folder):
    if '.lattices' in file.name and not '.ipynb' in file.name:
        base_out = '.'.join(file.name.split('.')[:-1])
        seg_out, map_out, conll_out = [os.path.join(pruned_folder, base_out+suf)
                                       for suf in ['.seg', '.map', '.conll']]
        if not os.path.exists(map_out):
            !{yap_path} joint -in {file.path} -os {seg_out} -om {map_out} -oc {conll_out} > /dev/null 2>&1
t1 = bclm.read_yap_output(None, '../NER/data/tokens_for_ncrf/naama/naama.test.1.tokens.txt',
                     '../NER/data/tokens_for_ncrf/naama/pruned/naama.test.1.tokens.pruned.conll',
                     '../NER/data/tokens_for_ncrf/naama/pruned/naama.test.1.tokens.pruned.map'
                    )
t2 = bclm.read_yap_output(None,  '../NER/data/tokens_for_ncrf/naama/naama.test.2.tokens.txt',
                     '../NER/data/tokens_for_ncrf/naama/pruned/naama.test.2.tokens.pruned.conll',
                     '../NER/data/tokens_for_ncrf/naama/pruned/naama.test.2.tokens.pruned.map'
                    )
t3 = bclm.read_yap_output(None,  '../NER/data/tokens_for_ncrf/naama/naama.test.3.tokens.txt',
                     '../NER/data/tokens_for_ncrf/naama/pruned/naama.test.3.tokens.pruned.conll',
                     '../NER/data/tokens_for_ncrf/naama/pruned/naama.test.3.tokens.pruned.map'
                    )

sl1, sl2, sl3 = [bclm.get_sentences_list(t, fields=['form']) for t in (t1,t2,t3)]
sl1

for i, sl in enumerate((sl1,sl2,sl3)):
    output_path = f'../NER/data/tokens_for_ncrf/naama/pruned/naama.test.{i+1}.tokens.pruned.dummy_o.bmes'
    with open(output_path, 'w') as of:
        for sent in sl:
            for form in sent:
                of.write(form[0]+' O\n')
            of.write('\n')
params = { 'status': 'decode' }

for i, row in erdf.iterrows():
    unit = row['unit']
    if 'tok' in unit:
        continue
    for i in range(3):
        name = f'morph_{i+1}_test_pruned'    
        row_par['load_model_dir'] = os.path.join(models_folder, row['model_file_name'])
        row_par['dset_dir'] = os.path.join(models_folder, row['dset_file_name'])
        row_par['decode_dir'] = os.path.join(output_folder, name+'.'+row['model_base_name']+'.bmes')
        row_par['raw_dir'] = f'../NER/data/tokens_for_ncrf/naama/pruned/naama.test.{i+1}.tokens.pruned.dummy_o.bmes'
        
        conf_path = os.path.join(decode_conf_folder, name+'.'+row['model_base_name']+'.decode.conf')
        if not os.path.exists(conf_path):
            with open(conf_path, 'w', encoding='utf8') as of:
                for k, v in row_par.items():
                    of.write(k+'='+str(v)+'\n')<jupyter_output>acc                                                             0.9686
p                                                               0.8433
r                                                               0.8076
f                                                               0.8251
model_base_name                      morph.char_cnn.ft_oov_tok.44_seed
arch                                                          char_cnn
unit                                                             morph
w_embed                                                     ft_oov_tok
seed_num                                                       44_seed
model_no_seed                                morph.char_cnn.ft_oov_tok
status                                                           train
model_dir            final_setup/plo_models/morph.char_cnn.ft_oov_t...
train_dir               ../NER/data/for_ncrf/morph_gold_train_plo.bmes
dev_dir                   ../NER/data/for_ncrf/morph_gold_dev_plo.bmes
test_d[...]<jupyter_text><-- NOW RUN PREDICT ON PRUNED  Add Alignments  Token Level Eval  TODO: 1. evaluate yap vs token1. evaluate pruned vs token<jupyter_code>import re

o_re = re.compile('^O+$') 
s_re = re.compile('^O*SO*$|^O*BI*EO*$')
b_re = re.compile('^O*BI*$')
i_re = re.compile('^I+$')
e_re = re.compile('^I*EO*$')
def get_fixed_for_valid_biose(bio_seq):
    if o_re.match(bio_seq):
        return 'O'
    if s_re.match(bio_seq):
        return 'S'
    if b_re.match(bio_seq):
        return 'B'
    if i_re.match(bio_seq):
        return 'I'
    if e_re.match(bio_seq):
        return 'E'
    raise ValueError
    

def get_fixed_for_invalid_biose(parts):
    bio = 'O'
    if parts[0][0] not in ('B', 'I', 'O', 'S', 'E'):
        bio = 'O'
    elif 'S' in parts:
        bio = 'S'
    elif 'B' in parts and 'E' in parts:
        bio='S'
    elif 'E' in parts:
        bio = 'E'
    elif 'B' in parts:
        bio = 'B'
    elif 'I' in parts:
        bio = 'I'
    return bio

valid_bio_re = re.compile('^O*BI*$|^O*BI*EO*$|^I+$|^I*EO*$|^O*SO*$')

from functools import lru_cache


def validate_biose_sequence(full_bio_seq):
    #print(full_bio_seq)
    bio_seq, type_seq = zip(*[('O', None) if b=='O' or b[0] not in ('B', 'I', 'O', 'S', 'E') else b.split('-') for b in full_bio_seq])
    bio_seq = ''.join(bio_seq)
    valid_bio = valid_bio_re.match(bio_seq)
    type_seq = list(filter(lambda x: x is not None, type_seq))
    type_seq_set = set(type_seq)

    if valid_bio:
        fixed_bio = get_fixed_for_valid_biose(bio_seq)
        if fixed_bio!='O':
            fixed_bio += '-' + type_seq[0]
            
    else:
        #take the first BIOSE tag which is not O:
        #fixed_bio = list(filter(lambda x: x!='O', full_bio_seq))[0]
        #rough BIOSE and first category:
        fixed_bio = get_fixed_for_invalid_biose(bio_seq)
        if fixed_bio!='O':
            fixed_bio += '-' + type_seq[0]
        
    return valid_bio is not None, len(type_seq_set)<=1, fixed_bio


@lru_cache(1000)
def get_fixed_bio_sequence(full_bio_seq):
    return validate_biose_sequence(full_bio_seq)[2]
t1_yap = bclm.read_yap_output(None, '../NER/data/tokens_for_ncrf/naama/naama.test.1.tokens.txt',
                     '../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.1.tokens.conll',
                     '../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.1.tokens.map'
                    )
t2_yap = bclm.read_yap_output(None,  '../NER/data/tokens_for_ncrf/naama/naama.test.2.tokens.txt',
                     '../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.2.tokens.conll',
                     '../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.2.tokens.map'                    )
t3_yap = bclm.read_yap_output(None,  '../NER/data/tokens_for_ncrf/naama/naama.test.3.tokens.txt',
                     '../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.3.tokens.conll',
                     '../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.3.tokens.map'                    )
sl1_yap, sl2_yap, sl3_yap = [bclm.get_sentences_list(t, fields=['token_id','token_str']) for t in (t1,t2,t3)]
sl1_yap

def get_fixed_tok(path, orig_sents):
    x = nem.read_file_sents(path, fix_multi_tag=False)
    new_sents = []
    for (i, ner_sent), (sent_id, yap_sent) in zip(x.iteritems(), orig_sents.iteritems()):
        for (form, bio), (token_id, token_str) in zip(ner_sent, yap_sent):
            new_sents.append((sent_id, token_id, token_str, form, bio))
    #print(new_sents[0])
    new_sents = pd.DataFrame(new_sents, columns=['sent_id', 'token_id', 'token_str', 'form', 'bio'])
    new_toks = bclm.get_token_df(new_sents, fields=['bio'])
    #print(new_toks.head())
    new_toks['fixed_bio'] = new_toks.bio.apply(lambda x: get_fixed_bio_sequence(tuple(x.split('^'))))
    return new_toks

def sents_from_df(df, sent_id_col='sent_id', 
                  group_cols=['token_str'], 
                  val_cols=['fixed_bio']):
    sents = bclm.get_sentences_list(df, fields=group_cols+val_cols)
    return sents

def evaluate_dataframes(gold_df, pred_df, fix_multi_tag_pred=True, truncate=None, ignore_cat=False, str_join_char=' '):
    gold_sents = sents_from_df(gold_df)
    pred_sents = sents_from_df(pred_df)
    gold_mentions = nem.sents_to_mentions(gold_sents, truncate=truncate, ignore_cat=ignore_cat, str_join_char=str_join_char)
    pred_mentions = nem.sents_to_mentions(pred_sents, truncate=truncate, ignore_cat=ignore_cat, str_join_char=str_join_char)
    return nem.evaluate_mentions(gold_mentions, pred_mentions, verbose=False)<jupyter_output><empty_output><jupyter_text>Hybrid<jupyter_code>out_folder = '../NER/data/tokens_for_ncrf/naama'
t1_tokens = os.path.join(out_folder, 'naama.test.1.tokens.txt')
t2_tokens = os.path.join(out_folder, 'naama.test.2.tokens.txt')
t3_tokens = os.path.join(out_folder, 'naama.test.3.tokens.txt')
token_paths = {'test_1': t1_tokens, 'test_2': t2_tokens, 'test_3': t3_tokens}
def get_prun_yo(ds, dep_path, map_path):
 
    prun_yo = bclm.read_yap_output(treebank_set=None,
                               tokens_path=token_paths[ds],
                               dep_path=dep_path,
                               map_path=map_path,
                                )
    return prun_yo<jupyter_output><empty_output><jupyter_text>Run on all pruned<jupyter_code>def get_sent_list(ds, dp, mp):
    prun_yo = get_prun_yo(ds, dp, mp)
    return bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str'])
output_folder
pruned_test_files = ()
align_tok_res = []
for file in os.scandir(output_folder):
    if 'morph' in file.name and '_test_pruned.' in file.name and not '.ipynb' in file.name:
        set_num = re.search('morph_(\d)_test_pruned',file.name).groups(0)[0]
        dep_path = f'../NER/data/tokens_for_ncrf/naama/pruned/naama.test.{set_num}.tokens.pruned.conll'
        map_path = f'../NER/data/tokens_for_ncrf/naama/pruned/naama.test.{set_num}.tokens.pruned.map'
        out_path = file.path.replace('_test_pruned.', '_test_pruned_align_tok.')
        print (out_path)
        if not os.path.exists(out_path):
            prun_sents = get_sent_list(f'test_{set_num}', dep_path , map_path)
            #print(prun_sents.head())
            new_toks = get_fixed_tok(file.path, orig_sents=prun_sents)

            new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])
            with open(out_path, 'w') as of:
                for i, sent in new_sents.iteritems():
                    for tok, bio in sent:
                        of.write(tok+' '+bio+'\n')
                    of.write('\n')

        p, r, f = nem.evaluate_files(f'../NER/data/for_ncrf/naama_plo/naama_plo.test.{set_num}.bmes', out_path, str_join_char='')

        align_tok_res.append((f'test_{set_num}', 'token', 'morph', 'hybrid', 'tokens', p, r, f))

at_df = pd.DataFrame(align_tok_res, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'p', 'r', 'f'])

at_df.groupby(['set', 'eval_unit','variant', 'prediction', 'align']).f.mean()<jupyter_output><empty_output><jupyter_text>Run all YAP<jupyter_code>align_tok_res_yg = []
for file in os.scandir(output_folder):
    if 'morph' in file.name and '_test.' in file.name and not '.ipynb' in file.name:
        set_num = re.search('morph_(\d)_test',file.name).groups(0)[0]
        dep_path = f'../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.{set_num}.tokens.conll'
        map_path = f'../NER/data/tokens_for_ncrf/naama/yap_output/naama.test.{set_num}.tokens.map'
        out_path = file.path.replace('_test.', '_test_align_tok.')
        print (out_path)
        if not os.path.exists(out_path):
            prun_sents = get_sent_list(f'test_{set_num}', dep_path , map_path)
            #print(prun_sents.head())
            new_toks = get_fixed_tok(file.path, orig_sents=prun_sents)

            new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])
            with open(out_path, 'w') as of:
                for i, sent in new_sents.iteritems():
                    for tok, bio in sent:
                        of.write(tok+' '+bio+'\n')
                    of.write('\n')

        p, r, f = nem.evaluate_files(f'../NER/data/for_ncrf/naama_plo/naama_plo.test.{set_num}.bmes', out_path, str_join_char='')

        align_tok_res_yg.append((f'test_{set_num}', 'token', 'morph', 'yap', 'tokens', p, r, f))

at_df = pd.DataFrame(align_tok_res+align_tok_res_yg, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'p', 'r', 'f'])

at_df.groupby(['eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16660502958579881, 'fraction_numerical': 0.011002218934911243, 'mean_word_length': 2.9239978233266823, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '355490', 'license': 'Apache-2.0', 'n_tokens_mistral': 8494, 'n_tokens_neox': 8338, 'n_words': 1225, 'path': '26_naama_reeval.ipynb', 'repo_name': 'cjer/NCRFpp', 'size': '21632'}