<jupyter_start><jupyter_text>DATA & PRE-PROCESSING<jupyter_code># Preprocessing steps
stemmer = LancasterStemmer()

def decodeHTMLencoding(tweets):
    decoded_tweets = tweets.applymap(lambda tweet: BeautifulSoup(tweet, 'lxml').get_text())
    return decoded_tweets

def removeStopWords(text):
    stopw = stopwords.words('english')
    words = [word for word in text.split() if len(word) > 3 and not word in stopw]
    # get stems from words
    for i in range(len(words)):
        words[i] = stemmer.stem(words[i])
    return (" ".join(words)).strip()

def cleanTweets(tweets):
    # decode tweets from html tags
    cleaned_tweets = decodeHTMLencoding(tweets)
    # remove URLs that starts with http
    cleaned_tweets = cleaned_tweets.applymap(lambda tweet: re.sub(
    r'https?:\/\/(www\.)?[-a-zA-Z0–9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0–9@:%_\+.~#?&//=]*)', '', tweet, flags=re.MULTILINE) )
    # remove URLs that does not start with http
    cleaned_tweets = cleaned_tweets.applymap(lambda tweet: re.sub(
    r'[-a-zA-Z0–9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0–9@:%_\+.~#?&//=]*)', '', tweet, flags=re.MULTILINE))
    # remove @
    cleaned_tweets = cleaned_tweets.applymap( lambda tweet: re.sub(r'@[A-Za-z0-9_]+', '', tweet, flags=re.MULTILINE) )
    # remove #
    cleaned_tweets = cleaned_tweets.applymap( lambda tweet: re.sub(r'#[A-Za-z0-9_]+', '', tweet, flags=re.MULTILINE) )
    # remove RT
    cleaned_tweets = cleaned_tweets.applymap( lambda tweet: re.sub('RT ', '', tweet, flags=re.MULTILINE) )
    # remove symbols and numbers (i.e keep letters only)
    cleaned_tweets = cleaned_tweets.applymap( lambda tweet: re.sub("[^a-zA-Z]"," ",tweet, flags=re.MULTILINE) )
    #replace consecutive non-ASCII characters with a space
    cleaned_tweets = cleaned_tweets.applymap( lambda tweet: re.sub(r'[^\x00-\x7F]+'," ",tweet.lower(), flags=re.MULTILINE) )
    
    cleaned_tweets.drop_duplicates(inplace=True)
    cleaned_tweets.replace('', np.nan, inplace=True)
    cleaned_tweets.dropna(inplace=True)
    
    return cleaned_tweets
def get_text_vectors(tweets, model):
    # dataset should be a pandas dataframe
    dimension = 300
    data_array = np.empty(shape=[0, dimension])
    indexes = []
    
    for i, tweet in enumerate(tweets):
        words = tweet.split()
        if len(words) !=0:
            feature = 0
            for word in words:
                try:
                    feature += model[word]
                except:
                    pass
            feature /= len(words)
            try:
                if feature.size == dimension:  
                    data_array = np.append(data_array, [feature], axis=0)
                    indexes.append(i)
            except:
                continue
    indexes = np.asarray(indexes)
    assert indexes.size == data_array.shape[0]
    return data_array, indexes
def remove_indices(weak_signals):
    # remove indexes of weak_signals that do not have coverage
    indices = np.where(np.sum(weak_signals, axis=1) == -1*weak_signals.shape[1])[0]
    weak_signals = np.delete(weak_signals, indices, axis=0)
    
    return weak_signals, indices
df = pd.read_csv('../datasets/glove.42B.300d.txt', sep=" ", quoting=3, header=None, index_col=0)
glove_model = {key: val.values for key, val in df.T.items()}
# test word vectors
from scipy import spatial
result = 1 - spatial.distance.cosine(glove_model['horrible'], glove_model['terrible'])
result
def keyword_labeling(data, keywords, sentiment='pos'):
    mask = 1 if sentiment == 'pos' else 0
    weak_signals = []
    for terms in keywords:
        weak_signal = []
        for text in data:
            label=-1
            for word in terms:
                if word in text.lower():
                    label = mask
            weak_signal.append(label)
        weak_signals.append(weak_signal)
    return np.asarray(weak_signals).T

POSITIVE_LABELS =  [['good','great','nice','delight','wonderful'], 
                    ['love', 'best', 'genuine','well', 'thriller'], 
                    ['clever','enjoy','fine','deliver','fascinating'], 
                    ['super','excellent','charming','pleasure','strong'], 
                    ['fresh','comedy', 'interesting','fun','entertain', 'charm', 'clever'], 
                    ['amazing','romantic','intelligent','classic','stunning'],
                    ['rich','compelling','delicious', 'intriguing','smart']]

NEGATIVE_LABELS = [['bad','better','leave','never','disaster'], 
                   ['nothing','action','fail','suck','difficult'], 
                   ['mess','dull','dumb', 'bland','outrageous'], 
                   ['slow', 'terrible', 'boring', 'insult','weird','damn'],
                   ['drag','awful','waste', 'flat','worse'],
                   #['drag','no','not','awful','waste', 'flat'], 
                   ['horrible','ridiculous','stupid', 'annoying','painful'], 
                   ['poor','pathetic','pointless','offensive','silly']]<jupyter_output><empty_output><jupyter_text>YELP<jupyter_code>datapath = '../datasets/yelp/'
size = 10000
review = pd.read_json(datapath+'yelp_review.json', lines=True,
                      dtype={'review_id':str,'user_id':str,
                             'business_id':str,'stars':int,
                             'date':str,'text':str,'useful':int,
                             'funny':int,'cool':int},
                      chunksize=size)
# There are multiple chunks to be read
count=0
chunk_list = []
for chunk_review in review:
    # Drop columns that aren't needed
    chunk_review = chunk_review.drop(['review_id','user_id','useful','funny','cool','business_id','date'], axis=1)
    chunk_list.append(chunk_review)
    count +=1
    if count==6:
        break
# After trimming down the review file, concatenate all relevant data back to one dataframe
df = pd.concat(chunk_list, ignore_index=True, join='outer', axis=0)
csv_name = datapath+"yelp_reviews.csv"
df.to_csv(csv_name, index=False)
df.head()
positive_labels = keyword_labeling(df.text.values, POSITIVE_LABELS, sentiment='pos')
negative_labels = keyword_labeling(df.text.values, NEGATIVE_LABELS, sentiment='neg')
weak_signals = np.hstack([positive_labels, negative_labels])
weak_signals, indices = remove_indices(weak_signals)
weak_signals.shape
df = df.drop(df.index[indices])
df.reset_index(drop=True, inplace=True)
train_data = df.text.values
train_labels = np.zeros(df.shape[0])
train_labels[df.stars.values >3]=1
train_data = cleanTweets(df.drop(columns=['stars']))
train_labels = train_labels[train_data.index]
weak_signals = weak_signals[train_data.index]
train_data.shape, train_labels.shape
train_features, train_index = get_text_vectors(train_data.values.ravel(), glove_model)
train_features.shape, train_index.shape
# get test data
np.random.seed(5000)
test_indexes = np.random.choice(train_index.size, 10000, replace=False)
test_labels = train_labels[test_indexes]
test_data = train_features[test_indexes]

train_data = np.delete(train_features, test_indexes, axis=0)
weak_signals = np.delete(weak_signals, test_indexes, axis=0)
train_labels = np.delete(train_labels, test_indexes)

train_data.shape,train_labels.shape,weak_signals.shape,test_labels.shape
# save the weak_signals signals
np.save(datapath+'weak_signals.npy', weak_signals)

# save yelp data
np.save(datapath+'data_features.npy', train_data)
np.save(datapath+'test_features.npy', test_data)

# save yelp labels
np.save(datapath+'data_labels.npy', train_labels)
np.save(datapath+'test_labels.npy', test_labels)
train_data.shape,train_labels.shape,weak_signals.shape,test_labels.shape<jupyter_output><empty_output><jupyter_text>SST-2<jupyter_code>datapath = '../datasets/sst-2/'
train_data = pd.read_csv(datapath+'sst2-train.csv')
test_data = pd.read_csv(datapath+'sst2-test.csv')
train_data.head()
NEGATIVE_LABELS = [['bad','better','leave','never','disaster'], 
                   ['nothing','action','fail','suck','difficult'], 
                   ['mess','dull','dumb', 'bland','outrageous'], 
                   ['slow', 'terrible', 'boring', 'insult','weird','damn'],
                   # ['drag','awful','waste', 'flat','worse'],
                   ['drag','no','not','awful','waste', 'flat'], 
                   ['horrible','ridiculous','stupid', 'annoying','painful'], 
                   ['poor','pathetic','pointless','offensive','silly']]

positive_labels = keyword_labeling(train_data.sentence.values, POSITIVE_LABELS)
negative_labels = keyword_labeling(train_data.sentesnce.values, NEGATIVE_LABELS, sentiment='neg')
weak_signals = np.hstack([positive_labels, negative_labels])
weak_signals.shape
weak_signals, indices = remove_indices(train_data, weak_signals)
weak_signals.shape
train_labels = train_data.label.values
test_labels = test_data.label.values

n,m = weak_signals.shape
weak_signal_probabilities = weak_signals.T.reshape(m,n,1)

weak_signals_mask = weak_signal_probabilities >=0

from model_utilities import get_error_bounds
true_error_rates = get_error_bounds(train_labels, weak_signal_probabilities, weak_signals_mask)
print("error: ", np.asarray(true_error_rates))
# Clean data and reset index
train_data.reset_index(drop=True, inplace=True)

# apply on train data
train_data = cleanTweets(train_data.drop(columns=['label']))
train_data = post_process_tweets(train_data)

# apply on test data
test_data = cleanTweets(test_data.drop(columns=['label']))
test_data = post_process_tweets(test_data)

print(train_data[0].shape, train_labels.shape)
print(test_data[0].shape, test_labels.shape)
train_features, train_index = get_text_vectors(train_data[0].values.ravel(), glove_model)
test_features, test_index = get_text_vectors(test_data[0].values.ravel(), glove_model)

# save sst-2 data
np.save(datapath+'data_features.npy', train_features)
np.save(datapath+'test_features.npy', test_features)

indexes = train_data[1]
indexes = indexes[train_index]
# save sst-2 labels
np.save(datapath+'data_labels.npy', train_labels[indexes])
np.save(datapath+'test_labels.npy', test_labels[test_data[1]])

# save the one-hot signals
np.save(datapath+'weak_signals.npy', weak_signals[indexes])<jupyter_output><empty_output><jupyter_text>IMDB Dataset<jupyter_code>datapath = '../datasets/imdb/'
df = pd.read_csv(datapath+'IMDB Dataset.csv')

# apply on train data
cleaned_data = cleanTweets(df.drop(columns=['sentiment']))
indexes = cleaned_data.index.values
df.shape, indexes.size
n = indexes.size
# get test data
np.random.seed(50)
test_indexes = np.random.choice(indexes, int(n*0.2), replace=False)
test_labels = np.zeros(test_indexes.size)
test_labels[df.sentiment.values[test_indexes]=='positive'] = 1
test_data = df.review.values[test_indexes]

train_indexes = np.delete(indexes, [np.where(indexes == i)[0][0] for i in test_indexes])
train_labels = np.zeros(train_indexes.size)
train_labels[df.sentiment.values[train_indexes]=='positive'] = 1
train_data = df.review.values[train_indexes]

print(train_data.shape, train_labels.shape)
print(test_data.shape, test_labels.shape)
positive_labels = keyword_labeling(train_data, [['good'],['wonderful'],['great'],['amazing'],['excellent']], sentiment='pos')
negative_labels = keyword_labeling(train_data, [['bad'],['horrible'],['sucks'],['awful'],['terrible']], sentiment='neg')
weak_signals = np.hstack([positive_labels, negative_labels])
weak_signals, indices = remove_indices(weak_signals)
weak_signals.shape
# add signals not covered to test data
test_data = np.append(test_data, train_data[indices])
test_labels = np.append(test_labels, train_labels[indices])

# delete train data not covered by weak signals
train_data = np.delete(train_data, indices, axis=0)
train_labels = np.delete(train_labels, indices)

# get data features
train_features, train_index = get_text_vectors(train_data, glove_model)
test_features, test_index = get_text_vectors(test_data, glove_model)

print(train_index.size, train_data.shape[0])
test_index.size, test_labels.size
# save imdb data
np.save(datapath+'data_features.npy', train_features)
np.save(datapath+'test_features.npy', test_features)

# save imdb labels
np.save(datapath+'data_labels.npy', train_labels[train_index])
np.save(datapath+'test_labels.npy', test_labels[test_index])

# save the weak_signals
np.save(datapath+'weak_signals.npy', weak_signals[train_index])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15002031694433157, 'fraction_numerical': 0.008126777732629013, 'mean_word_length': 4.411609498680739, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '94751', 'license': 'MIT', 'n_tokens_mistral': 4344, 'n_tokens_neox': 4185, 'n_words': 705, 'path': 'generate_weak_signals.ipynb', 'repo_name': 'VTCSML/Constrained-Labeling-for-Weakly-Supervised-Learning', 'size': '12305'}