<jupyter_start><jupyter_text>*best viewed in [nbviewer](https://nbviewer.jupyter.org/github/CambridgeSemiticsLab/BH_time_collocations/blob/master/results/notebooks/6_definite_modifier.ipynb)*  Time Adverbial Distribution and Collocations Definite modifier investigations <NAME><jupyter_code>! echo "last updated:"; date<jupyter_output>last updated:
Mon  6 Jul 2020 14:26:55 BST<jupyter_text>IntroductionThis notebook will produce data for my in-progress article on time adverbial components.   PythonNow we import the modules and data needed for the analysis.<jupyter_code># see .py's for variables
from config import *
from get_hmod_sample import hm_df
from statsmodels.graphics.mosaicplot import mosaic
from matplotlib import cm
TF.load('domain', add=True)
pd.set_option('display.max_rows', 100)
pd.set_option('display.max_colwidth', 0)
def show_examples(value_counts, df, n_examples=50, condenseType='sentence', **kwargs):
    """Prepares and shows examples from a supplied Series.value_counts()"""
    
    n_to_get = ((value_counts / value_counts.sum()) * n_examples).round()
    samples = []
    
    for word in n_to_get.index:
        n = int(n_to_get.loc[word]) or 1
        examples = df[df['head_voc'] == word]
        get_ilocs = get_spread(examples.shape[0], n)
        return_ex = examples.iloc[get_ilocs, :]
        samples.extend(return_ex.index)
        
    samples = [(ph, L.u(ph, 'sentence')[0]) for ph in samples]
    A.table(samples, condenseType=condenseType, **kwargs)<jupyter_output><empty_output><jupyter_text>Examples  Definite Time and Locas<jupyter_code>loca = hm_df[hm_df.function == 'Loca']
time = hm_df[hm_df.function == 'Time'].copy()
defi_time = hm_df[(hm_df.function=='Time') & hm_df.definite]
defi_loca  = hm_df[(hm_df.function=='Loca') & hm_df.definite]
def_loc_counts = defi_loca.classi.value_counts()

def_loc_counts
def_loc_counts.sum()
def_loc_counts / def_loc_counts.sum()
def_time_counts = defi_time.classi.value_counts()

def_time_counts
def_time_counts.sum()
def_time_counts / def_time_counts.sum()<jupyter_output><empty_output><jupyter_text>single.prep.definite<jupyter_code>show = ['function', 'ref', 'text', 'sentence']
most_common = 'single.prep.definite' # <- this is the phrase classi that has most overlap between Loca and Time

most_common_loca = defi_loca[defi_loca.classi == most_common]
most_common_time = defi_time[defi_time.classi == most_common]
most_common_loca.leading_prep.value_counts()
most_common_time.leading_prep.value_counts()
# Loca heads in prep + def + noun context
loca_heads = pd.DataFrame(most_common_loca.head_voc.value_counts()[:20])

loca_heads
top_loca_heads = pd.DataFrame(loca.head_voc.value_counts().head(20))

top_loca_heads
top_loca_heads.sum()
top_loca_heads / top_loca_heads.sum() 
# Time heads in prep + def + noun context

time_heads = pd.DataFrame(most_common_time.head_voc.value_counts())

time_heads
time.head_voc.value_counts()
time.head_voc.value_counts().sum()
time.head_voc.value_counts() / time.head_voc.value_counts().sum() 
# export table for paper
all_heads = [
    pd.DataFrame(time_heads.head(10).reset_index().values, columns=['Time head', 'freq']),
    pd.DataFrame(loca_heads.head(10).reset_index().values, columns=['Loca head', 'freq'])
]

all_heads = pd.concat(all_heads, 1)

all_heads.to_clipboard(index=False)
all_heads<jupyter_output><empty_output><jupyter_text>Extract examples of definite versus non-definite entities<jupyter_code># build a column of text types (Narrative or Quotation)

text_type = []

for tp in time.index:
    clause = L.u(tp, 'clause')[0]
    txt = F.domain.v(clause)
    text_type.append(txt)
    
time['text_type'] = text_type

time.text_type.value_counts()
# import nominal definitions from earlier PCA pos
pca_pos = pd.read_csv('../../spreadsheets/advb_article/pos_PCA.csv')

pca_pos.head()
# isolate true nominals based on the PCA analysis
nom_lexs = pca_pos[pca_pos.PC1 > 0.2].head_lexn.values
time[
    (time.head_lexn.isin(nom_lexs))
    & (time.has_nom == False)
][show][:200].head(200)
select_cases = time[
    (time.head_etcbc.isin(['BQR=/', '<RB/']))
    & (time.head_lexn.isin(nom_lexs))
    & (time.leading_prep.isin({'ל', 'עד', 'מן'}))
]

select_cases[show]
refs = []
for node in select_cases[select_cases.book == 'Exodus'].index:
    book, chapter, verse = T.sectionFromNode(node)
    refs.append(f'{chapter}:{verse}')
    
print(', '.join(refs))
# Select a spread of additional examples of anchoring 
# using demonstratives, ordinals, and genitives

examples = []

spread_per_modi = 10
for modi in ['demonstrative', 'ordinal', 'genitive']:
    samples = time[time[modi]]
    spread = get_spread(samples, spread_per_modi)
    examples.extend(spread.index)
    
examples
time.loc[examples][['ref', 'text', 'sentence']].to_clipboard(index=False)
time.loc[examples][['ref', 'text', 'sentence']]<jupyter_output><empty_output><jupyter_text>Select Non-prep Examples<jupyter_code>non_preps = time[
     (time.head_lexn.isin(nom_lexs))
     & (~time.preposition)
]

# non_preps[show][:50]
non_preps.groupby(['quantified', 'definite']).size()
# have a look at those cases which are neither definite or quantified
# many are marked plural, others are single adverbial uses like לילה

# non_preps[
#     (non_preps.definite == False)
#     & (non_preps.quantified == False)
# ][show]
# prepare spread of definite versus quantified cases

n_spread = 10

examples = []

for modi in ['quantified', 'definite']:
    samples = non_preps[non_preps[modi] == True]
    samples = get_spread(samples, n_spread)
    examples.extend(samples.index)
    
examples
non_preps.loc[examples][['ref', 'text', 'sentence']].to_clipboard(index=False)
non_preps.loc[examples][show]
time.loc[730191]['quantified']<jupyter_output><empty_output><jupyter_text>Annotate Kinds of Definiteness in Simple Prepositional Phrases<jupyter_code>annote_location = paths.project.joinpath('results/spreadsheets/advb_article')
## prepare manual tagging dataset


#dataset = []
#cols = ['ref', 'head_voc', 'text', 'sentence']

#for function, df in [('Time', most_common_time), ('Loca', most_common_loca)]:
#    export_df = df[cols].copy()
#    export_df['link'] = [A.webLink(ph, urlOnly=True) for ph in df.index]
#    export_df.to_csv(annote_location.joinpath(f'{function}_definite.csv'))
most_common_time.shape[0] 
most_common_loca.shape[0]
most_common_loca.shape[0] + most_common_time.shape[0]<jupyter_output><empty_output><jupyter_text>Import annotated data<jupyter_code>ann_loca = pd.read_csv(annote_location.joinpath('Loca_definite_annotated.csv'))
ann_time = pd.read_csv(annote_location.joinpath('Time_definite_annotated.csv'))
for df in [ann_loca, ann_time]:
    df.set_index('node', inplace=True)
# merge annotations with additional data
combined_annotes = pd.concat([ann_loca['category'], ann_time['category']])
ann_df = pd.concat([hm_df, combined_annotes], 1, join='inner').fillna(False)
ann_count = pd.pivot_table(
    ann_df,
    index='function',
    columns=['genre', 'category'],
    aggfunc='size',
    fill_value=0,
)
ann_count
np.zeros(2)
count_tables = {} 
prop_tables = {}
stacked_tables = {}

all_genre = ann_count.groupby(level='category', axis=1).sum()
count_tables['all'] = all_genre
prop_tables['all'] = all_genre.div(all_genre.sum(1), 0)
stacked_tables['all'] = prop_tables['all'].stack()

print('all')
display(count_tables['all'])
display(prop_tables['all'])
print()

for genre in ['prose', 'prophetic', 'poetry']:
    genre_count = ann_count[(genre,)]
    sum_count = genre_count.sum(1)
    prop_count = genre_count.div(sum_count, 0)
    count_tables[genre] = sum_count
    prop_tables[genre] = prop_count
    stacked_tables[genre] = prop_count.stack()
    
    print(genre)
    display(genre_count)
    display(prop_count)
    print()
    
# add missing values to poetry
#for missing in ['imme', 'anap', 'name']:
#    prop_tables['poetry'][missing] = np.zeros(2)
# configs for the mosaic plot

prop_indices = prop_tables['all'].stack().index

def labelizer(key):
    cat = key[1]
    value = round(stacked_tables[genre][key], 2)
    return f'{cat} ({value})'

tile_properties = lambda k: {'facecolor': 'white', 'edgecolor': 'black', 'linewidth': 0.6}

# make the plots
fig = plt.figure(figsize=(20, 5))

i = 1

for genre in ('prose', 'prophetic', 'poetry', 'all'):
    
    props = prop_tables[genre]
    
    ax = fig.add_subplot(1, 4, i)
    
    # config and sort the prop table
    props = props.loc[['Time', 'Loca'], :]
    plot_order = props.quantile(0.75).sort_values().index
    plot_prop = props[plot_order]
    
    mosaic(
        plot_prop.stack(), 
        ax=ax, 
        properties=tile_properties, 
        labelizer=labelizer,
        axes_label=True,
    )
    
    ax.set_title(genre)
    ax.set_yticks([], [])
    ax.set_yticklabels([])
    
    # change fontsize
    new_texts = []
    for text, sample in zip(ax.texts, plot_prop.values.flatten()):
        if 0.01 < sample < 0.05:
            text.set_size(5)
        elif sample >= 0.05:
            text.set_size(9)
        else:
            text.set_text('')
    i += 1
    
savefig('prop_defi_mosaic')
from scipy.stats import linregress
x = np.array([0.41, 0.78, 0.85])
y = np.array([0.02, 0.05, 0.12])

slope, intercept, r, p, std_err = linregress(x,y)
lin_reg_y = slope * x + intercept

plt.scatter(x, y)
plt.plot(x, lin_reg_y, color='red')
lin_reg_y
p
r
std_err<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14285714285714285, 'fraction_numerical': 0.011496700021290185, 'mean_word_length': 4.972663699936427, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '558033', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 3630, 'n_tokens_neox': 3461, 'n_words': 649, 'path': 'archive/2021-03-7/results/notebooks/advb_article/6_definite_modifier-Copy1.ipynb', 'repo_name': 'CambridgeSemiticsLab/BH_time_collocations', 'size': '9400'}