<jupyter_start><jupyter_text>AskParents<jupyter_code>train = pd.read_csv('../../annotated_data/askparents_train.tsv', sep='\t', header=0)
train['Sentence'] = train['Sentence'].apply(lambda x: x.lower())

dev = pd.read_csv('../../annotated_data/askparents_dev.tsv', sep='\t', header=0)
dev['Sentence'] = dev['Sentence'].apply(lambda x: x.lower())
df = pd.concat([train,dev]).reset_index()
labels_DS = df['DS_Label'].values
labels_Maj = df['Majority_label'].values
df
sentences = df['Sentence'].tolist()
text = " ".join(sent for sent in sentences)
doc = nlp(text)
lemmas = [word.lemma for sent in doc.sentences for word in sent.words]
advice_sentences = df[df['DS_Label']==1]['Sentence'].tolist()
advice_text = " ".join(sent for sent in advice_sentences)
doc_advice = nlp(advice_text)
advice_lemmas = [word.lemma for sent in doc_advice.sentences for word in sent.words]
advice_count = Counter(advice_lemmas)
logoddsdf = pd.DataFrame.from_dict(Counter(lemmas), orient='index').reset_index()
logoddsdf = logoddsdf.rename(columns={'index':'lemma',0:'total_count'})
logoddsdf
logoddsdf.loc[:, 'advice_count'] = logoddsdf['lemma'].apply(lambda x: advice_count[x] if x in advice_count.keys() else 0)
logoddsdf.loc[:, 'nonadvice_count'] = logoddsdf.apply(lambda x: x['total_count'] - x['advice_count'], axis=1)

num_advice = len(df[df['DS_Label']==1])
num_nonadvice = len(df[df['DS_Label']==0])

logoddsdf = logoddsdf[(logoddsdf['advice_count']>1) & (logoddsdf['nonadvice_count']>1) & (logoddsdf['total_count']>20)]

logoddsdf.loc[:, 'logodds'] = logoddsdf.apply(lambda x: math.log((x['advice_count']*num_nonadvice)/(x['nonadvice_count']*num_advice)), axis=1)
logoddsdf.loc[:, 'p-value'] = logoddsdf.apply(lambda y: binom_test(x=y['advice_count'], n=y['total_count'],
                                                                   p=0.5), axis=1)
print("Old shape:", logoddsdf.shape)
print("Old median:", logoddsdf['logodds'].median())
logoddsdf = logoddsdf[logoddsdf['p-value']<0.05].sort_values(by=['logodds'])
print("New shape:", logoddsdf.shape)
print("New median:", logoddsdf['logodds'].median())
logoddsdf['log_total'] = logoddsdf['total_count'].apply(lambda x: math.log(x))
# logoddsdf.plot(x='log_total', y='logodds', kind='scatter')
logoddsdf[logoddsdf['logodds']>0].sort_values(by=['logodds'], ascending=False).head(50)
logoddsdf[logoddsdf['logodds']<0].sort_values(by=['logodds']).head(50)<jupyter_output><empty_output><jupyter_text>Needadvice<jupyter_code>train_na = pd.read_csv('../../annotated_data/needadvice_train.tsv', sep='\t', header=0)
train_na['Sentence'] = train_na['Sentence'].apply(lambda x: x.lower())

dev_na = pd.read_csv('../../annotated_data/needadvice_dev.tsv', sep='\t', header=0)
dev_na['Sentence'] = dev_na['Sentence'].apply(lambda x: x.lower())

df_na = pd.concat([train_na,dev_na]).reset_index()
sentences_na = df_na['Sentence'].tolist()
text_na = " ".join(sent for sent in sentences_na)
doc_na = nlp(text_na)
lemmas_na = [word.lemma for sent in doc_na.sentences for word in sent.words]
advice_sentences_na = df_na[df_na['DS_Label']==1]['Sentence'].tolist()
advice_text_na = " ".join(sent for sent in advice_sentences_na)
doc_na_advice = nlp(advice_text)
advice_lemmas_na = [word.lemma for sent in doc_na_advice.sentences for word in sent.words]
advice_count_na = Counter(advice_lemmas_na)
logoddsdfna = pd.DataFrame.from_dict(Counter(lemmas_na), orient='index').reset_index()
logoddsdfna = logoddsdfna.rename(columns={'index':'lemma',0:'total_count'})

logoddsdfna.loc[:, 'advice_count'] = logoddsdfna['lemma'].apply(lambda x: advice_count[x] if x in advice_count.keys() else 0)
logoddsdfna.loc[:, 'nonadvice_count'] = logoddsdfna.apply(lambda x: x['total_count'] - x['advice_count'], axis=1)

num_advice = len(df_na[df_na['DS_Label']==1])
num_nonadvice = len(df_na[df_na['DS_Label']==0])

logoddsdfna = logoddsdfna[(logoddsdfna['advice_count']>1) & (logoddsdfna['nonadvice_count']>1) & (logoddsdfna['total_count']>20)]

logoddsdfna.loc[:, 'logodds'] = logoddsdfna.apply(lambda x: math.log((x['advice_count']*num_nonadvice)/(x['nonadvice_count']*num_advice)), axis=1)
print("Old shape:", logoddsdfna.shape)
print("Old median:", logoddsdfna['logodds'].median())

logoddsdfna.loc[:, 'p-value'] = logoddsdfna.apply(lambda y: binom_test(x=y['advice_count'], n=y['total_count'],
                                                                   p=0.5), axis=1)
logoddsdfna = logoddsdfna[logoddsdfna['p-value']<0.05].sort_values(by=['logodds'])

print("New shape:", logoddsdfna.shape)
print("New median:", logoddsdfna['logodds'].median())
logoddsdfna['log_total'] = logoddsdfna['total_count'].apply(lambda x: math.log(x))
# logoddsdfna.plot(x='log_total', y='logodds', kind='scatter')
logoddsdfna[logoddsdfna['logodds']>0].sort_values(by=['logodds'], ascending=False).head(50)
logoddsdfna[logoddsdfna['logodds']<0].sort_values(by=['logodds']).head(50)
df_na.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19048587111201465, 'fraction_numerical': 0.010164667615368977, 'mean_word_length': 7.0261011419249595, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '535624', 'license': 'MIT', 'n_tokens_mistral': 1942, 'n_tokens_neox': 1849, 'n_words': 264, 'path': 'Notebooks/Lexical_Analysis.ipynb', 'repo_name': 'venkatasg/Advice-EMNLP2020', 'size': '4919'}