<jupyter_start><jupyter_text>Clause Word Order (Main Clauses)<jupyter_code>from scripts.imports import *

# designate some additional parameters for a clause to be "main"
non_main = [
    'InfC', 'InfA', 'Reop', 
    'Voct', 'CPen', 'Ellp', 
    'MSyn',
]

main_df = df[
    (df.cl_rela == 'Main')
    & (~df.cl_type2.isin(non_main))
    & (df.n_times == 1)
    & (~df.cl_type.isin(['NmCl', 'AjCl']))
]

out = Exporter(
    paths['outdir'],
    'pragmatics',
)

main_df.shape<jupyter_output><empty_output><jupyter_text>Exploring Word Order in TA ClausesWe have a strict / literal word order tag. We will eventially process that tag into narrower categories. But first, we can do a formal analysis of the strict tag.<jupyter_code>clargs_count = main_df.cl_args.value_counts()

print(clargs_count.shape)

clargs_count.head(10)
out.number(
    clargs_count.shape[0],
    'n_clargtags'
)
', '.join(sorted(main_df[main_df['Time Position'] == 'medial'].cl_args.unique()))
', '.join(sorted(main_df[main_df['Time Position'] == 'preposed'].cl_args.unique()))
', '.join(sorted(main_df[main_df['Time Position'] == 'final'].cl_args.unique()))<jupyter_output><empty_output><jupyter_text>Overall Position Counts<jupyter_code>pos_ct = main_df['Time Position'].value_counts()
pos_pr = pos_ct / pos_ct.sum()
pos_cp = join_ct_pr(pos_ct, pos_pr)
pos_cp.rename_axis(index='Time Position', inplace=True)

out.table(
    pos_cp,
    'posi_cp',
    caption='Counts for Time Clause Positions in Hebrew Bible'
)

pos_cp
out.number(
    pos_pr.loc['preposed']*100,
    'prepose_all_pr',
)<jupyter_output><empty_output><jupyter_text>Playing Around with Distribution Through HB<jupyter_code>sns.swarmplot(x=main_df[main_df['Time Position'] == 'preposed'].cl_clust10)
sns.swarmplot(x=main_df[main_df['Time Position'] == 'medial'].cl_clust10)<jupyter_output><empty_output><jupyter_text>Position Counts by Genre<jupyter_code>posg_ct = pivot_ct(
    main_df,
    'gendom',
    'Time Position',
)

main_gen = ['prose.N', 'prose.Q', 'prophetic.Q', 'poetry.Q', 'instruction.Q']

posg_ct = posg_ct.loc[main_gen]

posg_pr = posg_ct.div(posg_ct.sum(1), 0).round(2)

out.table(
    posg_ct,
    'posig_ct',
    caption='Counts for Time Clause Positions by Genre',
)
out.table(
    posg_pr.mul(100).round(2).astype(int).astype(str)+'%',
    'posig_pr',
    caption='Percentages for Time Clause Positions by Genre',
)
out.number(
    posg_pr.loc['prose.N']['preposed']*100,
    'proseN_pre_pr',
)
out.number(
    posg_pr.loc['prose.Q']['preposed']*100,
    'proseQ_pre_pr',
)
out.number(
    posg_pr.loc['poetry.Q']['preposed']*100,
    'poetry_pre_pr',
)
out.number(
    posg_pr.loc['prophetic.Q']['preposed']*100,
    'proph_pre_pr',
)
out.number(
    posg_pr.loc['instruction.Q']['preposed']*100,
    'instr_pre_pr',
)
# todo: make 2x3 subplots with barplots but have the last row
# omit a plot in order to plot 5 of them

# https://stackoverflow.com/questions/26767281/position-5-subplots-in-matplotlib<jupyter_output><empty_output><jupyter_text>Position Counts by Tokenlex<jupyter_code>poslt_ct = pivot_ct(
    main_df,
    'lex_token',
    'Time Position',
)

print(poslt_ct.shape)

out.number(
    poslt_ct.shape[0],
    'shape_posltct_prefilt'
)

# filter out by occurrences at least 5
poslt_ct = poslt_ct[poslt_ct.sum(1) >= 5]

print(poslt_ct.shape)

out.number(
    poslt_ct.shape[0],
    'shape_posltct_filt'
)

out.number(
    poslt_ct.size,
    'size_posltct_filt'
)

out.table(
    poslt_ct.head(25),
    'poslt_ct_25',
    caption='Lexeme-Modifier Token Counts by Position in Clause (top 25)',
    hebindex=True
)
poslt_pr = poslt_ct.div(poslt_ct.sum(1), 0)

poslt_pr.head()
poslt_dp = sig.apply_deltaP(poslt_ct, 0, 1)

poslt_dp.head()
poslt_pr_plot = poslt_pr.copy()

poslt_pr_plot.index = [get_display(tok) for tok in poslt_pr_plot.index]
g = sns.clustermap(
    poslt_pr_plot, 
    dendrogram_ratio=(0.25, 0),
    cmap='magma', 
    col_cluster=False,
    cbar_pos=(1.07, .2, .03, .5),
    annot=True,
#    cbar_kws={'label': 'proportion'}
)

g.ax_heatmap.set_yticklabels(
    g.ax_heatmap.get_ymajorticklabels(), 
    fontsize = 16,
    fontfamily='SBL Biblit'
)

g.ax_heatmap.set_xlabel('Time Position', size=12)

g.ax_heatmap.set_xticklabels(
    g.ax_heatmap.get_xmajorticklabels(), 
    fontsize = 12,
)

g.ax_cbar.set_ylabel('proportion', size=12, rotation=270, labelpad=25)

out.plot(
    'clust_position_lextok',
)<jupyter_output><empty_output><jupyter_text>Assign Clusters for Later Reference<jupyter_code># retrieve data in its sorted form
poslt_pr2 = g.data2d
poslt_pr2.index = [get_display(h) for h in poslt_pr2.index]
poslt_pr2.head()<jupyter_output><empty_output><jupyter_text>Cluster 1<jupyter_code>cluster1 = poslt_pr2.loc['מחר':'אחר']

clust1_df = main_df[main_df.lex_token.isin(cluster1.index)]

cluster1
cluster1.index.to_list()
clust1_df.shape
out.number(
    cluster1['preposed'].mean() * 100,
    'pos_clust1_premean',
)<jupyter_output><empty_output><jupyter_text>Cluster 2<jupyter_code>cluster2 = poslt_pr2.loc['ל.ה.זמן': 'כ.ה.זמן']

cluster2
toget=[
    'ל.ה.זמן',
    'כ.ה.זמן'
]

clust2_df = main_df[main_df.lex_token.isin(toget)]
print(clust2_df.shape)

clust2_ct = pivot_ct(
    clust2_df,
    'lex_token',
    'verbtense',
)

clust2_pr = clust2_ct.div(clust2_ct.sum(1), 0).round(2)

out.table(
    clust2_ct,
    'clust2_tense_ct',
    caption='English Verb Translation Counts with \texthebrew{ל.ה.זמן} and \texthebrew{כ.ה.זמן}',
    hebindex=True,
)
out.table(
    clust2_pr.mul(100).round().astype(int).astype(str)+'%',
    'clust2_tense_pr',
    caption='English Verb Translation Percents with \texthebrew{ל.ה.זמן} and \texthebrew{כ.ה.זמן}',
    hebindex=True,
)
for ta in toget:
    print('-'*20)
    print(ta)
    print()
    for pos in poslt_pr2.columns:
        exdf = main_df[
            (main_df.lex_token == ta)
            & (main_df['Time Position'] == pos)
        ]
        print(pos)
        display(ts.show(exdf, spread=5, extra=['verbform', 'verbtense']))<jupyter_output>--------------------
ל.ה.זמן

final
showing 5 of 7<jupyter_text>Cluster 3<jupyter_code>cluster3 = poslt_pr2.loc['ב.ה.זמן': 'ב.ה.מ׳.זמן']
clust3_df = main_df[main_df.lex_token.isin(cluster3.index)]

cluster3
out.number(
    cluster3.final.mean()*100,
    'clust3_final_mean',
)
clust3_vf_ct = clust3_df.verbform.value_counts()
clust3_vf_pr = clust3_vf_ct / clust3_vf_ct.sum()

clust3_vf_cp = join_ct_pr(
    clust3_vf_ct,
    clust3_vf_pr
)

out.table(
    clust3_vf_cp,
    'clust3_verb_ct',
    caption='Verb Preference within Cluster 3'
)
out.number(
    clust3_df.shape[0],
    'n_clust3',
)
out.number(
    clust3_vf_pr['wayq']*100,
    'clust3_way_pr'
)
out.number(
    clust3_vf_pr['wqtl']*100,
    'clust3_wqtl_pr'
)
out.number(
    (clust3_vf_pr['wayq']+clust3_vf_pr['wqtl'])*100,
    'clust3_consec_pr'
)<jupyter_output><empty_output><jupyter_text>Cluster 4<jupyter_code>cluster4 = poslt_pr2.loc['ב.ה.זמן.הם':'מן.זמן']
clust4_df = main_df[main_df.lex_token.isin(cluster4.index)]

cluster4
cluster4.index.to_list()
out.number(
    clust4_df.shape[0],
    'n_clust4',
)
clust4_df.shape
out.number(
    cluster4.final.mean()*100,
    'clust4_final_mean',
)
out.number(
    cluster4.preposed.mean()*100,
    'clust4_pre_mean',
)
ts.show(
    clust4_df[clust4_df.lex_token == 'ב.זמן']
)<jupyter_output>showing 25 of 175<jupyter_text>Pull out examples of interest  B.H.ZMN.HMH<jupyter_code>exs = [
  #  'ב.ה.זמן.הם',
    
]

inthosedays = main_df[(main_df.lex_token == 'ב.ה.זמן.המה')]

ts.show(
 inthosedays, extra=['Time Position', 'verbtense']
)
ts.show(
    main_df[(main_df.times_etcbc == 'MXR/') & (main_df['Time Position'] == 'final')], 
    extra=['Time Position', 'verbform', 'verbtense']
)
inthosedays.verbtense.value_counts()<jupyter_output><empty_output><jupyter_text>Does verb choice interact with position?<jupyter_code>vcpos_ct = pivot_ct(
    main_df[~main_df.lex_token.isin(cluster1.index)], # exclude preposed advbs
    'verbform',
    'Time Position',
)

vcpos_ct
vcpos_pr = vcpos_ct.div(vcpos_ct.sum(1), 0)

vcpos_pr
vcpos_dp = sig.apply_deltaP(vcpos_ct, 0, 1)

vcpos_dp
fig, ax = plt.subplots(figsize=(5, 5))
heatmap(vcpos_dp, square=False, annot=True)
ax.set_yticklabels(ax.get_yticklabels(), rotation=0)
plt.show()<jupyter_output><empty_output><jupyter_text>Function and Word Order<jupyter_code>fpos_ct = pivot_ct(
    main_df,
    'function',
    'Time Position'
)

fpos_ct = fpos_ct.head(10)

fpos_ct
fpos_dp = sig.apply_deltaP(fpos_ct, 0, 1)

fpos_dp
fig, ax = plt.subplots(figsize=(8, 8))
heatmap(fpos_dp.round(2), robust=True, ax=ax, square=False, annot=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1552124448572092, 'fraction_numerical': 0.024843278384026005, 'mean_word_length': 4.789650537634409, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 56, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '210736', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 3871, 'n_tokens_neox': 3734, 'n_words': 542, 'path': 'workflow/notebooks/analysis/prag_wordorder.ipynb', 'repo_name': 'CambridgeSemiticsLab/BH_time_collocations', 'size': '8614'}