<jupyter_start><jupyter_text>Analyzing the segmentation results<jupyter_code>import os
import numpy as np
import pdb
import matplotlib.pyplot as plt
import seaborn as sns
questions = {}
answers = {}
for root, dirs, files in os.walk("../../segments/"):
    for file in files:
        if file.endswith(".txt") :
            with open(os.path.join(root,file)) as f:
                this_files_text = f.read().lower().replace('<br>', ' ')
                this_files_text = this_files_text.replace('<br >', ' ')
                if 'context' in file:
                    questions[file.split('-')[0]] = this_files_text.split('\n')
                else:
                    if answers.get(file.split('_')[0]):
                        answers[file.split('_')[0]] += this_files_text
                    else:
                        answers[file.split('_')[0]] = this_files_text


replies = {}
import re
for key in answers.keys():
    replies[key] = {}
    split = re.split('-----------[-]+', answers[key])

    for reply in split:
        if reply=='\n':
            continue
        reply_num = reply.strip().split('\n')[0]
        replies[key][reply_num] = reply.strip().split('\n')[1:]
all_reply_segments = sum([[replies[rep_key][rep_num] for rep_num in replies[rep_key].keys()] for rep_key in replies.keys()], [])
for a,b in enumerate(all_reply_segments[0]):
    print(a,b)
sum(all_reply_segments,[])[0].split()
segment_lengths=[len(a.split()) for a in sum(all_reply_segments,[])]
sns.set_style("darkgrid")

fig = sns.distplot(segment_lengths, kde=False, hist_kws={'alpha':0.75})
fig.set(xlabel='Length of EDU', ylabel='Frequency"')
sns.despine()
fig.xaxis.grid(False)
# plt.xticks([0,20,40,60,80,100])
plt.savefig('figs/edu-lengths.png', dpi=600)
def is_it_a_discourse_marker(text, pos, word):
    if word in text[pos].split()[:2]:
        return True
    else:
        return False
        
    

though_positions_summed = sum([[(i*(100/len(rep))) for i,x in enumerate(rep) if is_it_a_discourse_marker(rep, i, 'though')] for rep in all_reply_segments], [])

# though_positions_lens = sum([[len(rep) for i,x in enumerate(rep) if is_it_a_discourse_marker(rep, i, 'though')] for rep in all_reply_segments], [])
# though_positions_lens
though_positions_summed
sns.set_style("darkgrid")

fig = sns.distplot(though_positions_summed, kde=False, hist_kws={'alpha':0.75})
fig.set(xlabel='Position in advice text', ylabel='Frequency of $though_{disc}$')
sns.despine()
fig.xaxis.grid(False)
plt.xticks([0,20,40,60,80,100])
plt.savefig('figs/edu-a-though-transparent.png', dpi=600, transparent=True)
since_positions_summed = sum([[(i*(100/len(rep))) for i,x in enumerate(rep) if is_it_a_discourse_marker(rep, i, 'since')] for rep in all_reply_segments], [])
since_positions_summed

sns.set_style("darkgrid")

fig = sns.distplot(since_positions_summed, kde=False, hist_kws={'alpha':0.75})
fig.set(xlabel='Position in advice text', ylabel='Frequency of "since"')
sns.despine()
fig.xaxis.grid(False)
if_positions_summed = sum([[(i*(100/len(rep))) for i,x in enumerate(rep) if is_it_a_discourse_marker(rep, i, 'if')] for rep in all_reply_segments], [])
if_positions_summed

sns.set_style("darkgrid")

fig = sns.distplot(if_positions_summed, kde=False, hist_kws={'alpha':0.75})
fig.set(xlabel='Position in advice text', ylabel='Frequency of "if"')
sns.despine()
fig.xaxis.grid(False)<jupyter_output><empty_output><jupyter_text>Analysing question discourse segments<jupyter_code>num_of_segments = {id: len(questions[id]) for id in questions.keys()}

plt.hist(num_of_segments.values())
or_positions_summed = sum([[i*(100/len(questions[ques_key])) for i,x in enumerate(questions[ques_key]) if is_it_a_discourse_marker(questions[ques_key], i, 'or')] for ques_key in questions.keys()], [])
or_positions_summed
import seaborn as sns
sns.set_style("darkgrid")

fig = sns.distplot(or_positions_summed, kde=False, hist_kws={'alpha':0.75})
fig.set(xlabel='Position in question text', ylabel='Frequency of $or_{disc}$')
sns.despine()
fig.xaxis.grid(False)
plt.xticks([0,20,40,60,80,100])
plt.savefig('figs/edu-q-or.png', dpi=600)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1678372487285057, 'fraction_numerical': 0.01961734076047469, 'mean_word_length': 4.649794801641587, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '358895', 'license': 'MIT', 'n_tokens_mistral': 1602, 'n_tokens_neox': 1511, 'n_words': 277, 'path': 'Notebooks/Segmentation.ipynb', 'repo_name': 'venkatasg/Advice-EMNLP2020', 'size': '4129'}