<jupyter_start><jupyter_text>Analysis<jupyter_code>tense_ct = pd.pivot_table(
    df,
    index='eng_TAM',
    columns='french_tense',
    aggfunc='size',
    fill_value=0,
)

tense_ct = tense_ct.loc[tense_ct.sum(1).sort_values(ascending=False).index]

tense_ct
tense_pr = tense_ct.div(tense_ct.sum(1), 0)

tense_pr<jupyter_output><empty_output><jupyter_text>Export Examples<jupyter_code># import Text-Fabric
from tf.app import use
A = use('bhsa', hoist=globals())
from pathlib import Path
import re

def add_hit_symbols(text, search, symbol='_'):
    """Use re to find matches and insert hit symbols."""
    find = f'({search})'
    replace = f'{symbol}\g<1>{symbol}'
    return re.sub(find, replace, text)

def clean_puncts(text):
    """Fix space-separated punctuations."""
    text = re.sub(' ([,.?!;”:’])', '\g<1>', text)
    text = re.sub('([“‘]) ', '\g<1>', text)
    return text

def clean_roman_nums(text):
    """Replace Roman numerals in the verse ref."""
    text = re.sub('^I ', '1 ', text)
    text = re.sub('^II ', '2 ', text)
    return text

def make_text_examples(df):
    
    """Build text examples."""

    # sort out texts
    # NB that esv and niv texts might also be similarly formatted later on
    bhs_joiner = ''
    bhs_text = ['sentence']
    bhs_text = df[bhs_text].astype(str).agg(bhs_joiner.join, axis=1)
    exs = []
    
    covered_verses = set()
    for i, node in enumerate(df.index):
        ref = df.loc[node]['ref_abbr']
        
        # skip already-covered verses
        if ref in covered_verses:
            continue
        covered_verses.add(ref)
        
        esv = df.loc[node]['esv']
        niv = df.loc[node]['niv'] 
        esv_verse = df.loc[node]['esv_verse']
        niv_verse = df.loc[node]['niv_verse']
        eng_tam = df.loc[node]['eng_TAM']
        #bhs = bhs_text[node].strip()
        bhs = T.text(L.u(node, 'verse')[0])
        heb = df.loc[node]['text_full']
        french = df.loc[node]['french']
        french_tense = df.loc[node]['french_tense']
         
        if type(esv_verse) != str:
            esv_verse = '' 
        if type(niv_verse) != str:
            niv_verse = ''
            
        # add formatting to verse texts
        ref = clean_roman_nums(ref)
        #esv_verse = add_hit_symbols(esv_verse, esv)
        #niv_verse = add_hit_symbols(niv_verse, niv)
        # bhs = add_hit_symbols(bhs, heb)
        esv_verse = clean_puncts(esv_verse)
        niv_verse = clean_puncts(niv_verse)

        # build and add example text
        ex = f'(x)\t{bhs} ({ref})\nNIV\t{niv_verse}\nESV\t{esv_verse}\nFRENCH\t{french}\n'
        exs.append(ex)
    
    return exs 
# Export PC 

pc_df = df[
    (df.eng_TAM == 'PAST')
    & (df.french_tense == 'PC')
]

ps_df = df[
    (df.eng_TAM == 'PAST')
    & (df.french_tense == 'PS')
]

impf_df = df[
    (df.eng_TAM == 'PAST')
    & (df.french_tense == 'IM')
]

pc_exs = make_text_examples(pc_df)
ps_exs = make_text_examples(ps_df)
im_exs = make_text_examples(impf_df)
french_data = [
    ('pc', pc_exs),
    ('ps', ps_exs),
    ('im', im_exs),
]

outdir = Path('../../../results/text_examples/French_comparisons')

for name, data in french_data:
    doc = '\n'.join(data)
    filename = f'wqtl_{name}_examples.txt'
    outfile = outdir.joinpath(filename)
    outfile.write_text(doc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15734370326054442, 'fraction_numerical': 0.003290457672749028, 'mean_word_length': 2.848101265822785, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '574093', 'license': 'MIT', 'n_tokens_mistral': 1327, 'n_tokens_neox': 1303, 'n_words': 238, 'path': 'workflow/notebooks/french/analyze_french.ipynb', 'repo_name': 'CambridgeSemiticsLab/translation_traditions_HB', 'size': '3343'}