<jupyter_start><jupyter_text>Negative IndexerThis semi-supervised approach seem to be working fine** aJavaher: this is the working version.<jupyter_code>from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import time
import pandas as pd
import numpy as np
import re
from hatesonar import Sonar
analyzer = SentimentIntensityAnalyzer() 
# reading the dataset

data = pd.read_csv('../Data/data_IRA_Ads.csv', sep='\t') 

data['pos'] = np.nan
data['neg'] = np.nan
data['neu'] = np.nan
data['compound'] = np.nan
data['flag'] = np.nan

for index, eachAdText in data['AD_TEXT'].iteritems():
    # skip empty texts
    if pd.isnull(eachAdText):
        continue
        
    # calculate the senti-index for each advert text
    tempRespond = analyzer.polarity_scores(eachAdText)    
    
    data.loc[index, 'pos'] = tempRespond['pos']
    data.loc[index, 'neg'] = tempRespond['neg']
    data.loc[index, 'neu'] = tempRespond['neu']
    data.loc[index, 'compound'] = tempRespond['compound']
    
    # make the flags
    if 0.5 <= tempRespond['compound']  and tempRespond['compound'] <= 1:
        data.loc[index, 'flag'] = 'positive'
    elif -0.5 < tempRespond['compound']  and tempRespond['compound'] < 0.5:
        data.loc[index, 'flag'] = 'neutral'
    else:
        data.loc[index, 'flag'] = 'negative'
    
    
#print(data[['AD_ID', 'pos', 'neg', 'neu', 'compound']])  
# find classes of ads
tempClass = data.loc[ (data['compound'] >= -1) &  (data['compound'] <= -0.5)  ]
print ('- Ads:', len(tempClass))

# find classes of ads
tempClass = data.loc[ (data['compound'] > -0.5) &  (data['compound'] < 0.5)  ]
print ('~ Ads:', len(tempClass))

# find classes of ads
tempClass = data.loc[ (data['compound'] >= 0.5) &  (data['compound'] <= 1)  ]
print ('+ Ads:', len(tempClass))
# finalize column names
data.rename(columns={'pos':'positivity_index'}, inplace=True)
data.rename(columns={'neg':'negativity_index'}, inplace=True)
data.rename(columns={'neu':'neutrality_index'}, inplace=True)
data.rename(columns={'compound':'compound_measure'}, inplace=True)



# save the processed data 
data.to_csv('../Data/processed_data.csv', index=False, sep='\t', encoding='utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17292892711634222, 'fraction_numerical': 0.008148483476686284, 'mean_word_length': 4.45679012345679, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '464227', 'license': 'MIT', 'n_tokens_mistral': 794, 'n_tokens_neox': 768, 'n_words': 172, 'path': 'Notebooks/negativeIndexer.ipynb', 'repo_name': 'gocontractdev/css_political_ads', 'size': '2209'}