<jupyter_start><jupyter_text>Stopwords<jupyter_code>import numpy as np
import pandas as pd
import re
import json

import matplotlib.pyplot as plt
df = pd.read_csv('../data/euroleaks/cleaned.csv')<jupyter_output><empty_output><jupyter_text>make stopwords out of speaker names<jupyter_code>with open('../data/euroleaks/amend_names.json', 'r') as f:
    amend_names = json.load(f)
    
# make stopwords out of names
stopnames = []

for names in amend_names.values():
    for name in names:
        if not re.search('\[.*\]', name):
            stopnames += name.split(' ')
            
for name in df.speaker.unique():
    if 'speaker' not in name:
        stopnames += name.split(' ')

stopnames = set(stopnames)
        
stopnames.remove('greek')
stopnames.remove('representative')
stopnames.remove('de')
stopnames.remove('van')
stopnames.add('alex')
stopnames.add('dusan')
stopnames.add('janis')
stopnames.add('joreon')
#stopnames
# back of envelope calculation for how many stopnames there are
np.sum(pd.Series(' '.join(df.speech).lower().split()).apply(lambda s:
    re.sub(r'[^\w\s]','',s) in stopnames
))<jupyter_output><empty_output><jupyter_text>disfluency<jupyter_code>disfluency = set([
    'erm', # synonym for hmm in https://euroleaks.diem25.org/leaks/mar17ewg/
    'um',
    'uh'
])
# back of envelope calculation for how many disfluency occurrences there are
np.sum(pd.Series(' '.join(df.speech).lower().split()).apply(lambda s:
    re.sub(r'[^\w\s]','',s) in disfluency
))<jupyter_output><empty_output><jupyter_text>common courtesy words<jupyter_code>courtesy = set([
    'thank',
    'colleague',
    'comment',
    'ask',
    'let',
    'point',
    'think',
    'question',
    'answer',
    'know',
    'need',
    'want',
    'welcome',
    'ahead'
])
# back of envelope calculation for how many courtesy occurrences there are
np.sum(pd.Series(' '.join(df.speech).lower().split()).apply(lambda s:
    re.sub(r'[^\w\s]','',s) in courtesy
))<jupyter_output><empty_output><jupyter_text>dump to json<jupyter_code>stopwords = {
    'names': list(stopnames),
    'disfluency': list(disfluency),
    'courtesy': list(courtesy)
}
import json

json = json.dumps(stopwords)
with open('../data/euroleaks/stopwords.json', 'w') as f:
    f.write(json)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16666666666666666, 'fraction_numerical': 0.0017497812773403325, 'mean_word_length': 4.2695852534562215, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '596066', 'license': 'MIT', 'n_tokens_mistral': 845, 'n_tokens_neox': 825, 'n_words': 168, 'path': 'notebooks/stopwords.ipynb', 'repo_name': 'ariansajina/master-thesis', 'size': '2286'}