<jupyter_start><jupyter_text>Clause Type Analysis<jupyter_code>from scripts.imports import *

out = Exporter(
    paths['outdir'],
    'clause',
)<jupyter_output><empty_output><jupyter_text>Basic cl type statistics  Clause Type<jupyter_code>cltype_cts = df.cl_type2.value_counts()
cltype_pr = cltype_cts / cltype_cts.sum()

cltype_ctpr =\
pd.concat([cltype_cts, (cltype_pr.round(2)*100).astype(int).astype(str)+'%'], 1)

cltype_ctpr.columns = ['count', 'proportion']

out.table(
    cltype_ctpr,
    'cltype_cts',
    caption='ETCBC Clause Type Counts with Time Adverbials',
)


cltype_ctpr
out.number(
    cltype_cts.shape[0],
    'n_clausetypes'
)
out.number(
    cltype_cts.sum(),
    'n_clauses'
)<jupyter_output><empty_output><jupyter_text>Clause type by function  Clause Relation<jupyter_code>rela_cts = df.cl_rela.value_counts()

rela_cts<jupyter_output><empty_output><jupyter_text>Count Clause Types with TA<jupyter_code>df.cl_type.value_counts()<jupyter_output><empty_output><jupyter_text>Various Composite Counts<jupyter_code>typfun_ct = pivot_ct(
    df,
    'cl_type2',
    'function',
)

typfun_pr = typfun_ct.div(typfun_ct.sum(1), 0)
typfun_dp = sig.apply_deltaP(typfun_pr, 0, 1)

typgendom_ct = pivot_ct(
    df,
    'cl_type2',
    'gendom',
)

typgendom_pr = typgendom_ct.div(typgendom_ct.sum(1), 0)

typgendom_dp = sig.apply_deltaP(typgendom_ct, 0, 1)
top = 7
out.table(
    typfun_ct.iloc[:, :top+1].rename_axis(index='clause type'),
    'type_function_ct',
    caption=f'Clause Type Counts by the Top {top} Time Functions',
    adjustbox=True,
)
top = 7
out.table(
    (typfun_pr.iloc[:, :top+1].rename_axis(index='clause type')*100)\
        .round().astype(int).astype(str) + '%'
    ,
    'type_function_pr',
    caption=f'Function Percentages of Clause Type (Top {top} Time Functions)',
    adjustbox=True,
)<jupyter_output><empty_output><jupyter_text>Clause Rela<jupyter_code>rel_ct = df.cl_rela.value_counts()
rel_pr = rel_ct / rel_ct.sum()

rel_pr
rel_both = pd.concat(
    [
        rel_ct,
        rel_pr.mul(100).round().astype(int).astype(str)+'%',
    ],
    1
).set_axis(['count', 'percent'], 1)

out.table(
    rel_both,
    'rel_both',
    caption='Time Adverbial Clause Relation Counts'
)
fig, ax = plt.subplots()
rel_pr.plot(kind='bar', ax=ax)
ax.set_xticklabels(ax.get_xticklabels(), rotation=0)
out.plot(
    'bar_rel_pr',
)
typerel_ct = pivot_ct(
    df,
    'cl_rela',
    'function'
)

typerel_ct = typerel_ct.rename_axis(index='clause relation')

typerel_pr = typerel_ct.div(typerel_ct.sum(1), 0)

typerel_dp = sig.apply_deltaP(typerel_ct, 0, 1)
out.table(
    typerel_ct.iloc[:, :top],
    'typerel_ct',
    caption=f'Clause Relationship Counts by Time Functions (Top {top} Functions)',
    adjustbox=True
)
typerel_pr.iloc[:, :top]

out.table(
    typerel_pr.iloc[:, :top]\
        .mul(100).round()\
        .astype(int).astype(str)+'%',
    'typerel_pr',
    caption=f'Function Percentages of Clause Relationship (Top {top} Functions)',
    adjustbox=True,
)<jupyter_output><empty_output><jupyter_text>Special Clause Types  MSyn<jupyter_code>ms_df = df[df.cl_type == 'MSyn']

ts.show(ms_df, spread=5)
out.number(
    ms_df.shape[0],
    'n_MSyn',
)

out.number(
    cltype_pr['MSyn']*100,
    'p_MSyn',
)
ms_df.lex_token.value_counts()
msyn_genredp = typgendom_dp.loc['MSyn'].sort_values(ascending=False)

msyn_genredp
out.number(
    typgendom_pr.loc['MSyn']['prose.Q']*100,
    'p_MSyn_genre'
)<jupyter_output><empty_output><jupyter_text>All `Msyn` are \<TH clauses. Which \<THs are not macros?<jupyter_code>other_now =     df[
        (df.times_etcbc == '<TH')
        & (df.cl_type != 'MSyn')
]

ts.show(
    other_now,
    extra=['cl_type'],
    spread=10
)<jupyter_output>showing 10 of 110<jupyter_text>Ellp<jupyter_code>el_df = df[
    df.cl_type == 'Ellp'
]

el_df.shape
out.number(
    el_df.shape[0],
    'n_Ellp'
)

out.number(
    cltype_pr['Ellp']*100,
    'p_Ellp',
)
out.table(
    el_df.function.value_counts(),
    'Ellp_funct_ct',
    caption='Adverbial Function Counts with Ellp',
)
el_df.gendom.value_counts()
typfun_dp.loc['Ellp'].sort_values(ascending=False).head(5)
out.number(
    typfun_dp.loc['Ellp']['atelic_ext']*100,
    'Ellp_atelext_dp'
)
out.table(
    typgendom_dp.loc['Ellp'].sort_values(ascending=False).head(5),
    'dp_Ellp_genre',
    caption='Top 5 ΔP Genre Scores with Ellp'
)
el_df.cl_kind.value_counts()
ts.show(el_df)<jupyter_output>showing 25 of 100<jupyter_text>Cpen<jupyter_code>cpen_df = df[df.cl_type == 'CPen']

ts.show(cpen_df)
out.number(
    cpen_df.shape[0],
    'n_CPen',
)
out.number(
   cltype_pr['CPen']*100,
    'p_CPen',
    roundto=2
)
cpen_df.verbform.value_counts()<jupyter_output><empty_output><jupyter_text>Xpos<jupyter_code>ts.show(
    df[df.cl_type  == 'XPos']
, extra=['sentence'])<jupyter_output>showing 1 of 1<jupyter_text>ויהי<jupyter_code>way_df = df[
    (df.verb_text == 'יהי')
    & (df.verbform == 'wayq')
]

way_df.shape
out.number(
    way_df.shape[0],
    'n_WayH'
)
out.number(
    cltype_pr['WayH']*100,
    'p_WayH'
)
out.table(
    way_df.gendom.value_counts(),
    'WayH_genre_ct',
    caption='Genre Counts for WayH',
)
out.table(
    way_df.function.value_counts(),
    'WayH_fun_ct',
    caption='Function Counts with WayH'
)
post_ct = typfun_ct['posterior'].sort_values(ascending=False)
post_pr = post_ct / post_ct.sum()

post_pr
post_pr = post_ct / post_ct.sum()

out.table(
    (post_pr.round(2).head()*100).astype(int).astype(str)+'%',
    'post_pr_top5',
    caption='Top 5 Clause Types with Posterior'
)
out.number(
    typfun_dp.loc['WayH'].sort_values(ascending=False)['posterior']*100,
    'dp_WayH_post',
)
ts.show(
    way_df[way_df.function.isin(['posterior'])],
    spread=10,
)
ts.show(
    way_df[way_df.gendom.isin(['prophetic.N'])],
    extra=['sentence'],
)
ts.show(
    way_df[way_df.gendom.isin(['prose.Q', 'prophetic.Q'])],
    extra=['sentence'],
)
way_df.cl_type.value_counts()
ts.show(way_df)<jupyter_output>showing 25 of 163<jupyter_text>There appears to be some difference with the WayX type wayehi clauses:<jupyter_code>ts.show(
    way_df[way_df.cl_type == 'WayX']
)<jupyter_output>showing 17 of 17<jupyter_text>והיה<jupyter_code>vhyh_df = df[
    (df.cl_type2 == 'WQtH')
]

vhyh_df.shape
out.number(
    vhyh_df.shape[0],
    'n_WQtH',
)
vehaya_genre_ct = vhyh_df.genre.value_counts()
vehaya_genre_pr = vehaya_genre_ct / vehaya_genre_ct.sum()

vehaya_genre_both = pd.concat([
    vehaya_genre_ct, (vehaya_genre_pr.round(2)*100).astype(int).astype(str)+'%'
], 1)
vehaya_genre_both.columns = ['count', 'percent']

out.table(
    vehaya_genre_both,
    'WQtH_gen_ct',
    caption='WQtH Genre Counts'
)
wqth_dp = typfun_dp.loc['WQtH'].sort_values(ascending=False).head()
wqth_ct = typfun_ct.loc['WQtH'][wqth_dp.index]
wqth_both = pd.concat([wqth_ct, wqth_dp.round(2)], 1)
wqth_both.columns = ['count', 'ΔP']
out.table(
    wqth_both,
    'wqth_functs',
    caption='WQtH Function Counts + ΔP'
)
out.number(
    wqth_both['ΔP']['simultaneous']*100,
    'wqth_simul_dp',
)
typfun_pr.loc['WQtH']
ts.show(vhyh_df)
vhyh_df.cl_type.value_counts()<jupyter_output><empty_output><jupyter_text>NmCl<jupyter_code>nmcl_df = df[
    df.cl_type == 'NmCl'
]

nmcl_df.shape
ts.show(nmcl_df)<jupyter_output>showing 25 of 201<jupyter_text>AjCl<jupyter_code>ajcl_df = df[
    df.cl_type == 'AjCl'
]

ajcl_df.shape
ts.show(ajcl_df)<jupyter_output>showing 25 of 26<jupyter_text>SubAdvb<jupyter_code>sadvb_df = df[
    df.cl_rela == 'SubAdv'
]
out.number(
    sadvb_df.shape[0],
    'n_SubAdv',
)
front_ct = sadvb_df.firstw.value_counts()

out.table(
    front_ct,
    'SubAdv_front',
    caption='First Word Counts in SubAdv Clauses'
)
ts.show(sadvb_df[sadvb_df.firstw=='\texthebrew{כי}'], extra=['cl_rela', 'verbform'])
sadvb_df[sadvb_df.firstw=='\texthebrew{ל}'].verbform.value_counts()
b_counts = sadvb_df[sadvb_df.firstw=='\texthebrew{ב}'].verbform.value_counts()

b_counts
b_counts = sadvb_df[sadvb_df.firstw=='\texthebrew{כ}'].verbform.value_counts()

b_counts
out.number(
    b_counts['infc'],
    'sadvb_b_inf'
)

out.number(
    b_counts.sum(),
    'sadvb_b_ct'
)
ts.show(sadvb_df[
    (sadvb_df.firstw=='\texthebrew{ל}')
    & (sadvb_df.verbform == 'qtl')
])
ts.show(sadvb_df[
    (sadvb_df.firstw=='\texthebrew{ב}')
    & (sadvb_df.verbform != 'infc')
])
ts.show(sadvb_df[
    (sadvb_df.firstw=='\texthebrew{כ}')
    & (sadvb_df.verbform == 'infc')
])
ts.show(sadvb_df[sadvb_df.firstw=='\texthebrew{ל}'], extra=['cl_rela', 'verbform'])
subadv_lman = sadvb_df[sadvb_df.firstw=='\texthebrew{למען}']

ts.show(subadv_lman, extra=['cl_rela', 'verbform'])
#ts.show(sadvb_df[sadvb_df.firstw=='\texthebrew{ב}'], extra=['cl_rela', 'verbform'])
sadvb_df_b = sadvb_df[
    (sadvb_df.firstw.isin(['\texthebrew{ב}', '\texthebrew{כ}']))
    & (sadvb_df.verbform == 'infc')
]
sadb_ct = sadvb_df_b.function.value_counts()
sadb_pr = sadb_ct / sadb_ct.sum()
sadb_both = pd.concat([
    sadb_ct,
    sadb_pr.mul(100).round().astype(int).astype(str)+'%'
], 1).set_axis(['count', 'percent'], 1)


out.table(
    sadb_both,
    'sadb_both',
    caption='\texthebrew{בְּ} + infc Time Adverbial Function Counts'
)
ts.show(sadvb_df_b)
subadv_im = sadvb_df[sadvb_df.firstw=='\texthebrew{אם}']

ts.show(subadv_im, extra=['cl_rela', 'verbform'])
subadv_im.function.value_counts()
subadv_Kasher = sadvb_df[
    (sadvb_df.firstw=='\texthebrew{כ}')
    & (sadvb_df.verbform != 'infc')
]

ts.show(subadv_Kasher, extra=['cl_rela', 'verbform'])
subadv_asher = sadvb_df[sadvb_df.firstw=='\texthebrew{אשר}']

ts.show(subadv_asher, extra=['cl_rela', 'verbform'])<jupyter_output>showing 6 of 6<jupyter_text>Look at syndetic adverbial clauses<jupyter_code>subadv_syn = sadvb_df[sadvb_df.firstw=='\texthebrew{ו}']

ts.show(subadv_syn, extra=['cl_rela', 'verbform'])
out.number(
    subadv_syn.shape[0],
    'subadv_syn_ct'
)<jupyter_output><empty_output><jupyter_text>SubMod<jupyter_code>submod_df = df[df.cl_rela=='SubMod']

front_mod = submod_df.firstw.value_counts()

out.table(
    front_mod.head(5),
    'SubMod_front',
    caption='First Word Counts in SubMod Clauses (Top 5)'
)
submod_func_dp = typerel_dp.loc['SubMod'].sort_values(ascending=False).round(2)

submod_func_dp.head()
out.number(
    submod_func_dp['simultaneous']*100,
    'submod_simul_dp'
)
out.number(
    submod_func_dp['atelic_ext']*100,
    'submod_atel_dp'
)
submod_func_cp = join_ct_pr(
    typerel_ct.loc['SubMod'].sort_values(ascending=False).head(5),
    typerel_pr.loc['SubMod']
)

submod_func_cp['ΔP'] = submod_func_dp.loc[submod_func_cp.index]

out.table(
    submod_func_cp,
    'submod_func_cp',
    caption='Attributive Clause Time Adverbial Counts (Top 5)',
)
submod_genre = submod_df.gendom.value_counts()
submod_genre_pr = submod_genre / submod_genre.sum()

out.table(
    join_ct_pr(submod_genre, submod_genre_pr).head(5),
    'submod_genre',
    caption='Top 5 genre Counts for Attributive Clauses with Time Adverbials'
)
for firstw in front_mod.head(5).index:
    print(firstw.__repr__())
    display(
        ts.show(submod_df[submod_df.firstw == firstw], spread=15)
    )
ts.show(submod_df[submod_df.function == 'atelic_ext'])<jupyter_output>showing 17 of 17
{'fraction_non_alphanumeric': 0.16384484457583087, 'fraction_numerical': 0.01325808474424438, 'mean_word_length': 5.120614035087719, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '613113', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 5161, 'n_tokens_neox': 4964, 'n_words': 627, 'path': 'workflow/notebooks/analysis/clause_minortypes.ipynb', 'repo_name': 'CambridgeSemiticsLab/BH_time_collocations', 'size': '11163'}