<jupyter_start><jupyter_text>Data loading<jupyter_code># import dataset

FILES = {
    "DNC": "raw/2020 Debate Transcripts.csv",
    "Presidential": "raw/Presidential_debate_USA_final_October_2020.csv",
    "Vice Presidential": "raw/VP_debate_2020.csv",
}

# preview first 5 records

df = pd.read_csv(FILES["DNC"], encoding='cp1252')
df = df.replace('\n',' ', regex=True)
print(df.head())

# explore debate topics

topics = df['Topic'].unique()
print('\n number of topics: ', len(topics))

print('\n debate topics:')
for topic in topics:
    print('\n', topic)
# topics by utterance frequency

speaker_topic = df[['Speaker','Topic']]
speaker_topic.groupby(['Topic']).count().sort_values(by='Speaker').plot(kind='barh', figsize=(5,20));
# check missing values

print("Missing 'Question' values? \n\n", df.loc[df.Question.isnull()], "\n\n")
print("Missing 'Topic' values? \n\n", df.loc[df.Topic.isnull()], "\n\n")
print("Missing 'SpeakerInfo' values? \n\n", df.loc[df.SpeakerInfo.isnull()])
# make obvious corrections
# DIAZ-BALART is a moderator, not a candidate
# MADDOW is a moderator, not a candidate
#TODO: where df['Speaker'] == '<NAME>' strip first 8 chars from ['Discourse w/o names']

df.loc[df['Speaker'] == 'DIAZ-BALART', 'SpeakerInfo'] = 'Moderator'
df.loc[df['Speaker'] == 'MADDOW', 'SpeakerInfo'] = 'Moderator'
print(df.loc[df.Speaker.eq('DIAZ-BALART')].head())
print(df.loc[df.Speaker.eq('MADDOW')].head())

# df.loc[df['Speaker'] == '<NAME>', 'Discourse w/o names'] = df.loc[df['Speaker'] == '<NAME>', 'Discourse w/o names'].str[8:]
# print(df.loc[df.Speaker.eq('<NAME>')].head())<jupyter_output>RowID  Original Row information       Date     Party        Location  \
8       9                         9  6/26/2019  Democrat  Miami, Florida   
41     42                        42  6/26/2019  Democrat  Miami, Florida   
43     44                        44  6/26/2019  Democrat  Miami, Florida   
45     46                        46  6/26/2019  Democrat  Miami, Florida   
47     48                        48  6/26/2019  Democrat  Miami, Florida   

        Speaker SpeakerInfo  Question               Topic  \
8   DIAZ-BALART   Moderator       0.0             Opening   
41  DIAZ-BALART   Moderator       6.0   Income Inequality   
43  DIAZ-BALART   Moderator       6.0   Income Inequality   
45  DIAZ-BALART   Moderator       6.0   Income Inequality   
47  DIAZ-BALART   Moderator       7.0  Manufacturing Jobs   

                                    Discourse w/names  \
8   DIAZ-BALART: And some quick rules of the road....   
41  DIAZ-BALART: Mayor De Blasio, good evening. Yo...   
43   [...]<jupyter_text>Parsing of discourse into monlogues and dialogues<jupyter_code># 'Opening', <NAME> invokes <NAME>, <NAME> and <NAME> take over without major technical difficulties --> skip
# 'Closing Statements', 'Their closing statements' excluding SpeakerInfo == 'moderator' --> monologues
# 'Biggest Geopolitical Threat', 'Candidates on what their first issue would be if elected' --> dialogues

# parse monologues and dialogues

monologues = df[0:0]
dialogues = df[0:0]

topics = np.delete(df['Topic'].unique(), [0, 1, 13, 15, 16, 19, 20, 21, 26, 29, 30, 32, 34, 41, 40, 45]) 

# manual parsing of 'Economy' (1)
tp = df.loc[df['Topic'] == 'Economy']
monologues = monologues.append(tp.iloc[3])
dialogues = dialogues.append(tp.iloc[0:2])

# manual parsing of 'Guns' (13)
tp = df.loc[df['Topic'] == 'Guns']
monologues = monologues.append(tp.iloc[22])
dialogues = dialogues.append(tp.iloc[23:46])

# manual parsing of '<NAME>' (15)
tp = df.loc[df['Topic'] == '<NAME>']
monologues = monologues.append(tp.iloc[3])
dialogues = dialogues.append(tp.iloc[4:12])

# manual parsing of 'Climate Change' (16)
tp = df.loc[df['Topic'] == 'Climate Change']
monologues = monologues.append(tp.iloc[5])
dialogues = dialogues.append(tp.iloc[6:34])

# manual parsing of 'Biggest Geopolitical Threat' (19)
tp = df.loc[df['Topic'] == 'Biggest Geopolitical Threat']
dialogues = dialogues.append(tp)

# manual parsing of 'Impeachment' (20)
tp = df.loc[df['Topic'] == 'Impeachment']
monologues = monologues.append(tp.iloc[2])
dialogues = dialogues.append(tp.iloc[3:18])

# manual parsing of 'Buttigieg on why he doesn’t support free college' (26)
tp = df.loc[df['Topic'] == 'Buttigieg on why he doesn’t support free college']
monologues = monologues.append(tp.iloc[3])
dialogues = dialogues.append(tp.iloc[[0, 2],:])

# manual parsing of 'Candidates weigh in on healthcare' (29)
tp = df.loc[df['Topic'] == 'Candidates weigh in on healthcare']
monologues = monologues.append(tp.iloc[5])
dialogues = dialogues.append(tp.iloc[6:77])

# manual parsing of 'Candidates on how they would address undocumented immigrants' (30)
tp = df.loc[df['Topic'] == 'Candidates on how they would address undocumented immigrants']
monologues = monologues.append(tp.iloc[3])
dialogues = dialogues.append(tp.iloc[4:66])

# <NAME> dramatically confronts Biden on his civil rights record (34)
tp = df.loc[df['Topic'] == '<NAME> dramatically confronts Biden on his civil rights record']
monologues = monologues.append(tp.iloc[7])
dialogues = dialogues.append(tp.iloc[8:31])

# manual parsing of 'Candidates on what their first issue would be if elected' (41)
tp = df.loc[df['Topic'] == 'Candidates on what their first issue would be if elected']
dialogues = dialogues.append(tp)

for topic in topics:
    # print(df.loc[df['Topic'] == topic])
    tp = df.loc[df['Topic'] == topic]
    idx_mono = tp.SpeakerInfo.eq('Candidate').idxmax()
    # print('\n\n monologue idx: ', idx_mono, '\n\n')
    monologues = monologues.append(df.iloc[[idx_mono]])
    
    # determine last record; use idx_mono and idx_last to grab remaining discourse on the topic; this constitutes a dialogue
    idx = df.index
    idx_last = idx[df.RowID.eq(tp['RowID'].iloc[[-1]])][0]
    # print('\n\n last RowID of topic: ', idx_last, '\n\n')
    dialogues = dialogues.append(df.iloc[idx_mono:idx_last,:])

cs = df.loc[(((df['Topic'] == 'Closing Statements') | (df['Topic'] == 'Their closing statements')) & (df['SpeakerInfo'] == 'Candidate'))]
monologues = monologues.append(cs)

monologues.rename(columns = {'Discourse w/o names': 'Text'}, inplace = True)
dialogues.rename(columns = {'Discourse w/o names': 'Text'}, inplace = True)

print('\n number of monologues: ', len(monologues))
# print('\n\n monologues: \n\n', monologues)
print('\n\n number of dialogues: ', len(dialogues))
# print('\n\n dialogues: \n\n', dialogues)<jupyter_output>number of monologues:  68


 number of dialogues:  751<jupyter_text>Determine long monologues that should be manually parsed into halves/thirds<jupyter_code># count characters in 'text'; if greater than 1024 put into 'thirds'; else put into 'halves'; manually parse duplicates
halves = []
thirds = []

for row in monologues[['RowID', 'Text']].itertuples():
    if len(row.Text) > 1024:
        thirds.append(row.RowID)
    elif 512 < len(row.Text) < 1024:
        halves.append(row.RowID)

print('\n\n Monologues to be halved: \n\n', halves)
print('\n\n Monologues to be cut in thirds: \n\n',thirds)
m = monologues['Text'].tolist()
d = dialogues['Text'].tolist()
print(len(m))
print(len(d))<jupyter_output>68
751<jupyter_text>Tag utterances for POS using Flair library<jupyter_code># tag utterances

tagger = SequenceTagger.load('pos-fast')

corpus = m
tagged = []

for i in tqdm(corpus):
  sentence = Sentence(i)
  tagger.predict(sentence)
  tagged.append(sentence.to_tagged_string())

for i in range(1):
  print(tagged[i])
  print(corpus[i])
# extract tags 

for i in tqdm(range(len(tagged))):
  for j in corpus[i].split():
    tagged[i] = str(tagged[i]).replace(j,"",1)

  # remove < > characters
  for j in  ['<','>']:
    tagged[i] = str(tagged[i]).replace(j,"")
    # remove redundant spaces
    tagged[i] = re.sub(' +', ' ', str(tagged[i]))
    tagged[i] = str(tagged[i]).lstrip()

for i in range(1):
  print(tagged[i])
  print(corpus[i])
for i in tqdm(range(len(corpus))):
  # remove symbols
  # corpus[i] = re.sub('[^a-zA-Z]', ' ', str(corpus[i]))
  tagged[i] = re.sub('[^a-zA-Z]', ' ', str(tagged[i]))

  ## remove HYPH SYM
  tagged[i] = str(tagged[i]).replace('HYPH',"")
  tagged[i] = str(tagged[i]).replace('SYM',"")                    

  # remove redundant space
  # corpus[i] = re.sub(' +', ' ', str(corpus[i]))
  tagged[i] = re.sub(' +', ' ', str(tagged[i]))
# check tags 

for i in range(3):
  print('monologue:\n\n'+corpus[i]+'\n') 
  print('tagged:\n\n'+tagged[i]+'\n')
  print('-'*90+'\n')
from collections import Counter

# unpack tagged

pos = []

for i in tqdm(range(len(tagged))):
    pos.extend(tagged[i].split())

c = Counter(pos)
count = Counter({k: c for k, c in c.items() if k.isupper() if c > 8})

print(count)

tags = pd.DataFrame().from_dict(count, orient='index')
tags.columns = ['pos']
tags.sort_values(by='pos', ascending=False).plot(kind='bar')
plt.show()

# NN - Noun, singular or mass, PRP - Personal pronoun, DT - Determiner, 
# IN - Preposition or subordinating conjunction, VB - Verb, base form, VBP - Verb, non-3rd person singular present, 
# RB - Adverb, JJ - Adjective, NNS - Noun, plural, NNP - Proper noun, singular, CC - Coordinating conjunction, 
# VBZ - Verb, 3rd person singular present, TO - to, VBG - Verb, gerund or present participle, 
# VBD - Verb, past tense, VBN - Verb, past participle, MD - Modal, CD - Cardinal number, WP - Wh-pronoun, 
# WDT - Wh-determiner, WRB - Wh-adverb, RP - Particle, UH - Interjection, JJR - Adjective, comparative, 
# JJS - Adjective, superlative, NNPS - Proper noun, plural, EX - Existential there, RBR Adverb, comparative, 
# POS - Possessive ending, RRB - ?, FW - Foreign word, RBS - Adverb, superlative, PDT - Predeterminer
# LRB - ?, NFP - Superfluous punctuation<jupyter_output>100%|██████████| 68/68 [00:00<00:00, 6121.89it/s]<jupyter_text>Write out monologues and dialogues to TSV files<jupyter_code>mono_output = monologues[['RowID', 'Speaker', 'SpeakerInfo', 'Question', 'Topic', 'Text']]
dia_output = dialogues[['RowID', 'Speaker', 'SpeakerInfo', 'Question', 'Topic', 'Text']]

# mono_output.to_csv('processed/dnc_debate_monologues.tsv', sep = '\t', index=False)
# dia_output.to_csv('processed/dnc_debate_dialogues.tsv', sep = '\t', index=False)
# dnc monologues for dnc_mono = pd.read_csv('processed/dnc_debate_monologues.tsv', sep = '\t')
# dnc_mono = dnc_mono[['RowID', 'Topic', 'Speaker', 'Text']]
# dnc_mono.to_csv('processed/dnc_monologues_for_pres.tsv', sep = '\t', index=False)<jupyter_output><empty_output><jupyter_text>Process TSV files for MLM The manually parsed monologue dataset is named `dnc_debate_monologues_parsed.tsv`. A copy named `dnc_debate_monologues_for_MLM.tsv` is made in order to preserve the naming convention.<jupyter_code>_df = pd.read_csv('processed/dnc_debate_dialogues.tsv', sep = '\t')
df = _df.sample(n = 155)
df.to_csv('processed/dnc_debate_dialogues_for_MLM.tsv', sep = '\t', index=False)<jupyter_output><empty_output><jupyter_text>Process TSV files for NSP<jupyter_code># import dataset

FILES = {
    "Monologue": "processed/dnc_debate_monologues.tsv",
    "Dialogue": "processed/dnc_debate_dialogues.tsv",
}

# preview first 5 records

df = pd.read_csv(FILES["Dialogue"], sep = '\t', usecols = ['Topic','Text'])
print(df.head())
topics = df['Topic'].unique().tolist()
text = df[0:0]
# text = pd.DataFrame(columns=['Text'])

for topic in topics:
    tp = df.loc[df['Topic'] == topic]
    for index, row in tp.iterrows():
        sentences = re.split(r"(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?|!)\s", row['Text'])
        for i in range(len(sentences)):
            _row = [topic, sentences[i]]
            _series = pd.Series(_row, index = df.columns)
            text = text.append(_series, ignore_index=True)
            
    _series = pd.Series([np.nan, np.nan], index = df.columns)
    text = text.append(_series, ignore_index=True)
    
print(text.head())

# text.to_csv('processed/dnc_debate_dialogues_for_NSP.tsv', sep = '\t', index=False)

# file = 'processed/dnc_debate_dialogues_for_NSP.tsv'

f = open(file, 'r')
text = f.read()
f.close()
text = text.replace("\"\"","")
f = open(file, 'w')
f.write(text)
f.close()
topics = df['Topic'].unique().tolist()
print(topics)<jupyter_output>['Economy', 'Guns', '<NAME>', 'Climate Change', 'Biggest Geopolitical Threat', 'Impeachment', 'Buttigieg on why he doesn’t support free college', 'Candidates weigh in on healthcare', 'Candidates on how they would address undocumented immigrants', 'Sen. Harris dramatically confronts Biden on his civil rights record', 'Candidates on what their first issue would be if elected', "Klobuchar's rivals programs", 'Top tax rate', 'breaking up of tech companies', 'Gender Pay gap', 'Income Inequality', 'Manufacturing Jobs', 'Healthcare', 'Abortion', 'Opiod Crisis', 'Undocumented Immigrants', 'Iran Nuclear Deal', 'Supreme Court Nominees', 'LGBTQ', 'Foreign Policy', 'Opening statements', 'Biden addresses income inequality', 'Candidates are asked about ‘socialism’', 'Rep. Swalwell and Biden on generational change, and Yang explains his economic plan', 'Sen. Harris’ ‘food fight’ moment', 'Yang and Mayor Buttigieg on concerns about China', 'Mayor Buttigieg speaks out on civil rights and the s[...]<jupyter_text>At this point I take half of the topics from the dialogue dataset and combine them with the other half of the topics from the monologue dataset. This makes up the training dataset. The inverse topics are then selection from the inverse datasets and this makes up the validation dataset. Each topic constitutes a 'document' in the setting of next sentence prediction. The resulting datasets are named `dnc_debate_train_for_NSP.tsv` and `dnc_debate_val_for_NSP.tsv`.  Process TSV files for SeqClassif I take half of the topics from the dialogue dataset and combine them with the other half of the topics from the monologue dataset. This makes up the training dataset. The inverse topics are then selection from the inverse datasets and this makes up the validation dataset. Utterances/Sentences from the monologue dataset are given the label 1 and those from the dialogue dataset are given the label 0. Extra examples are deleted in order to maintain balanced training and validation datasets. The resulting datasets are named `dnc_debate_train_for_SC.tsv` and `dnc_debate_val_for_SC.tsv`.<jupyter_code># import dataset

FILES = {
    "train": "processed/dnc_debate_train_for_SC.tsv",
    "val": "processed/dnc_debate_val_for_SC.tsv",
}

# preview first 5 records

df = pd.read_csv(FILES["train"], sep = '\t', usecols = ['label','text'])
df['text'] = df['text'].astype('str') 
print(df.head())
print(df.dtypes)<jupyter_output>label                                               text
0      1                                         All right.
1      1  So with that business out of the way, we want ...
2      1  And we’ll start this evening with Senator Eliz...
3      1                                         Thank you.
4      1                                   Good to be here.
label     int64
text     object
dtype: object
{'fraction_non_alphanumeric': 0.15151313601596275, 'fraction_numerical': 0.018423678084469573, 'mean_word_length': 4.017017017017017, 'pattern_counts': {'":': 7, '<': 48, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '525326', 'license': 'Apache-2.0', 'n_tokens_mistral': 5504, 'n_tokens_neox': 5149, 'n_words': 1420, 'path': 'data/DNC Debate Discourse Preprocessing.ipynb', 'repo_name': 'eusip/BERT-explainability-discourse', 'size': '15117'}