<jupyter_start><jupyter_text>Testing the difference of mean values of sentence lengths (Bootstrapping)<jupyter_code>only_summaries = wikihow[wikihow['is_summary'] == 1]['sentence_len']
non_summary = wikihow[wikihow['is_summary'] == 0]['sentence_len']
plt.figure(figsize=(20,10))
plt.hist(non_summary,bins = 160, normed=True,histtype="stepfilled")
plt.hist(only_summaries,bins = 250, normed=True,histtype="stepfilled", color = 'red', alpha = 0.4)
plt.axvline(wikihow['sentence_len'].mean(), color = 'blue')
plt.axvline(only_summaries.mean(), color = 'red')
plt.xlim((1, 80)) 
plt.xlabel('Lenght Category')
plt.ylabel('Frequency')
plt.legend(['All Sentences','Summaries'],loc = 'upper right')
plt.title('Distribution of sentence lengths')
# Reconstructing the mean under the null hypothesis
# Compute the combined mean of the dataset
combined_mean = wikihow_subset['sentence_len'].mean()

# First shift the samples Should I shift the median ? 
summary = wikihow_subset[wikihow_subset['is_summary'] == 1]['sentence_len']- wikihow_subset[wikihow_subset['is_summary'] == 1]['sentence_len'].mean() + combined_mean 
no_summary = wikihow_subset[wikihow_subset['is_summary'] == 0]['sentence_len'] - wikihow_subset[wikihow_subset['is_summary'] == 0]['sentence_len'].mean() + combined_mean

#Get bootstrap replicates for shifted data 
np.random.seed(47)
N_rep = 10000

#Initialize empty array of same size than reps
bs_mean_diffs = np.empty(10000)

for rep in range(N_rep):
    
    # Generate bootstrap sample: bs_sample
    summarized = np.random.choice(summary, size=len(summary))
    no_sumarized = np.random.choice(no_summary, size=len(no_summary))
    bs_mean_summary = np.mean(summarized)
    bs_mean_no_summary = np.mean(no_sumarized)
    
    # Computing differences in means
    bs_mean_diff = bs_mean_no_summary - bs_mean_summary
    bs_mean_diffs[rep] = bs_mean_diff
bs_mean_diffs
# Calculating confidence interval 
conf_int = np.percentile(bs_mean_diffs, [2.5, 97.5])
conf_int
#Calculating the p-value
mean_diff = np.mean(wikihow_subset[wikihow_subset['is_summary'] == 0]['sentence_len']) - np.mean(wikihow_subset[wikihow_subset['is_summary'] == 1]['sentence_len'])
# Compute the p-value
p = np.sum(bs_mean_diffs >= mean_diff) / len(bs_mean_diffs)
p
mean_diff
bs_mean_diffs
plt.figure(figsize=(20,10))
plt.hist(bs_mean_diffs)
plt.title('mean difference between summary and non summary sentences')
plt.axvline(conf_int[0], color = 'red')
plt.axvline(conf_int[1], color = 'red')
#plt.axvline(mean_diff, color = 'green')
plt.show()<jupyter_output><empty_output><jupyter_text>**Null hypothesis**: There is no difference the average sentence length of a summary sentence and the average sentence length of a non summary sentence.**Alternative hypothesis**:There is a difference the average sentence length of a summary sentence and the average sentence length of a non summary sentence.The p-value = 0: we can reject the null hypothesis and assume the average sentence length for a summary sentence is significantly smaller than the sentence length for a non summary sentence  Testing the difference of median values of sentence lengths (Bootstrapping)<jupyter_code># Reconstructing the median under the null hypothesis
# Compute the combined median of the dataset
combined_median = wikihow_subset['sentence_len'].median()

# First shift the samples Should I shift the median ? 
shift_summary = wikihow_subset[wikihow_subset['is_summary'] == 1]['sentence_len'] - wikihow_subset[wikihow_subset['is_summary'] == 1]['sentence_len'].median() + combined_median
shift_no_summary = wikihow_subset[wikihow_subset['is_summary'] == 0]['sentence_len'] - wikihow_subset[wikihow_subset['is_summary'] == 0]['sentence_len'].median() + combined_median

shift_summary.describe()
shift_no_summary.describe()
#Get bootstrap replicates for shifted data 
np.random.seed(42)
N_rep = 10000

#Initialize empty array of same size than reps
bs_median_diffs = np.empty(10000)

for rep in range(N_rep):
    
    # Generate bootstrap sample: bs_sample
    summarized = np.random.choice(shift_summary, size=len(summary))
    no_sumarized = np.random.choice(shift_no_summary, size=len(no_summary))
    bs_median_summary = np.median(summarized)
    bs_median_no_summary = np.median(no_sumarized)
    
    # Computing differences in means
    bs_median_diff = bs_median_no_summary - bs_median_summary
    bs_median_diffs[rep] = bs_median_diff
bs_median_diffs
# Calculating confidence interval 
conf_int = np.percentile(bs_median_diffs, [2.5, 97.5])
conf_int
#Calculating the p-value
median_diff = np.median(wikihow_subset[wikihow_subset['is_summary'] == 'no']['sentence_len']) - np.median(wikihow_subset[wikihow_subset['is_summary'] == 'yes']['sentence_len'])
# Compute the p-value
p = np.sum(bs_median_diffs >= median_diff) / len(bs_median_diffs)
p
median_diff
bs_median_diffs
pd.Series(bs_median_diffs).describe()
plt.figure(figsize=(20,10))
plt.hist(bs_median_diffs, bins = 100)
plt.title('median difference between summary and non summary sentences')
plt.axvline(conf_int[0], color = 'red')
plt.axvline(conf_int[1], color = 'red')
#plt.axvline(median_diff, color = 'green')
plt.show()<jupyter_output><empty_output><jupyter_text>Testing the difference in proportions of top common word between summary sentences and non summary sentences (Permutation Test)<jupyter_code>top_occurences = ['may','help','use','get', 'make','like','need', 'take', 'try', 
                  'avoid','keep', 'want', 'many', 'ask', 'good', 'find', 'work', 
                  'go', 'include', 'important', 'know']
#Calculating means of 0-1 for summary category and non summary category
occurence_means_summary = dict()
occurence_means_no_summary = dict()
for occurence in top_occurences:
    mean_summary = wikihow_subset[wikihow_subset['is_summary']==1][occurence].mean()
    occurence_means_summary[occurence] = mean_summary
    
    mean_no_summary = wikihow_subset[wikihow_subset['is_summary']==0][occurence].mean()
    occurence_means_no_summary[occurence] = mean_no_summary
summary_mean = pd.DataFrame(occurence_means_summary.items(), columns = ['word','mean'])
no_summary_mean = pd.DataFrame(occurence_means_no_summary.items(), columns = ['word','mean'])
labels = top_occurences
summary_means = summary_mean['mean']
non_summary_means = no_summary_mean['mean']

x = np.arange(len(labels))  # the label locations
width = 0.40  # the width of the bars

fig, ax = plt.subplots(figsize=(20,10))
plt.figure(figsize=(20,40))
rects1 = ax.bar(x - width/2, summary_means, width, label='Summary')
rects2 = ax.bar(x + width/2, non_summary_means, width, label='No Summary')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_ylabel('Word')
ax.set_title('Proportion of most frequent words in sentences for summary and no summary sentences.')
ax.set_xticks(x)
ax.set_xticklabels(labels, rotation='vertical')
ax.legend()
summary_means<jupyter_output><empty_output><jupyter_text>Let's pick the word `make` for example, how likely is it that this word is more present in summary sentences ?<jupyter_code>def diff_prop(data_a, data_b):
    frac_a = np.sum(data_a) / len(data_a)
    frac_b = np.sum(data_b) / len(data_b)
    
    return frac_b - frac_a 
make_summary = wikihow_subset[wikihow_subset['is_summary']== 1]['make']
make_no_summary = wikihow_subset[wikihow_subset['is_summary']== 0]['make']
diff_prop_obs_make = diff_prop(make_summary, make_no_summary)
diff_prop_obs_make
def permutation_sample(data1, data2):
    """Generate a permutation sample from two data sets."""

    # Concatenate the data sets: data
    data = np.concatenate((data1, data2))

    # Permute the concatenated array: permuted_data
    permuted_data = np.random.permutation(data)

    # Split the permuted array into two: perm_sample_1, perm_sample_2
    perm_sample_1 = permuted_data[:len(data1)]
    perm_sample_2 = permuted_data[len(data1):]

    return perm_sample_1, perm_sample_2
def draw_perm_reps(data_1, data_2, func, size=1):
    """Generate multiple permutation replicates."""

    # Initialize array of replicates: perm_replicates
    perm_replicates = np.empty(size)

    for i in range(size):
        # Generate permutation sample
        perm_sample_1, perm_sample_2 = permutation_sample(data_1, data_2)

        # Compute the test statistic
        perm_replicates[i] = func(perm_sample_1, perm_sample_2)

    return perm_replicates
perm_replicates = np.empty(10000)
for i in range(10000):
    perm_replicates[i] = draw_perm_reps(make_summary, make_no_summary, diff_prop)
p_value = np.sum(perm_replicates <= diff_prop_obs_make) / len(perm_replicates)
p_value<jupyter_output><empty_output><jupyter_text>The p-value here is so low that we can reject the null hypothesis: the word 'make' is more present in summary sentences.  Testing for Independence * Between `is_summary` and `title_sim_categories`(Chi-Square test)* Between `tfidf_score` and `title_similarity` (Pearson correlation tests)  Chi Square test<jupyter_code>table, results = rp.crosstab(wikihow_subset['title_sim_categories'], wikihow_subset['is_summary'], prop= 'col', test= 'chi-square')
table
results<jupyter_output><empty_output><jupyter_text>The null hypothesis is rejected, since p < 0.000, and a conclusion is made that `title_sim` is associated with `is_summary`  Pearson correlation test<jupyter_code>def pearson_r(x, y):
    """Compute Pearson correlation coefficient between two arrays."""
    # Compute correlation matrix: corr_mat
    corr_mat = np.corrcoef(x,y)


    # Return entry [0,1]
    return corr_mat[0,1]
tfidf_score = wikihow_subset['tfidf_score']
title_similarity = wikihow_subset['title_similarity']
#Scatter plot
plt.figure(figsize=(20,10))
# Make a scatter plot
plt.plot(tfidf_score, title_similarity, marker = '.', linestyle = 'none')

# Label the axes
plt.xlabel('tf-idf')
plt.ylabel('title_similarity')<jupyter_output><empty_output><jupyter_text>These two variables look like they are positively correlated<jupyter_code># Compute observed correlation: r_obs
r_obs = pearson_r(tfidf_score, title_similarity)
# Initialize permutation replicates: perm_replicates
perm_replicates = np.empty(10000)
# Draw replicates
for i in range(10000):
    # Permute  tfidf scores:
    tfidf_permuted = np.random.permutation(tfidf_score)
    
    # Compute Pearson correlation
    perm_replicates[i] = pearson_r(tfidf_permuted, title_similarity)
# Compute p-value: p
p = np.sum(perm_replicates >= r_obs)/len(perm_replicates)
p<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11501049017737937, 'fraction_numerical': 0.014400152584398244, 'mean_word_length': 6.222451790633609, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '470226', 'license': 'MIT', 'n_tokens_mistral': 3775, 'n_tokens_neox': 3418, 'n_words': 816, 'path': 'notebooks/statistical_inference.ipynb', 'repo_name': 'sarisboo/tl_dr_extractive', 'size': '10486'}