<jupyter_start><jupyter_text>The global warming issue and Narratives around it Part 6: Testing the trained model onto a new datasetIn this notebook, I tested the trained NLP model onto a new dataset, which belonged to a more general class of [climate change](https://www.reddit.com/r/climatechange/) issues. The hope was to filter the global warming concerns from the others. Importing the required libraries:<jupyter_code>#imports
import requests
import pandas as pd
import time

#Importing the built function, prior to that added the assets path to the system path
#Inspiration: https://stackoverflow.com/questions/4383571/importing-files-from-different-folder

import sys
# inserting the parent directory into current path
sys.path.insert(1, '../assets')

from get_reddit_posts import get_reddit_posts



import pandas as pd
import regex as re
import warnings
warnings.filterwarnings('ignore')
from nltk.corpus import stopwords # Import the stopword list
import pickle
from nltk.stem.porter import PorterStemmer
from nltk.stem import WordNetLemmatizer


import numpy as np
import pandas as pd
import matplotlib.pylab as plt
import seaborn as sns

import regex as re
import nltk
#nltk.download('vader_lexicon')
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LinearRegression


from nltk.corpus import stopwords

import pickle<jupyter_output><empty_output><jupyter_text>Calling in the created function and reading global warming data:<jupyter_code>#Defining API pull initial parameteres:

par = {"subreddit": "climatechange", #The subreddit title
       "post_num": 100, # Numer of posts to pull from
        "time_1": int(time.mktime(time.strptime('1 July, 2020', '%d %B, %Y'))), # The latest pull time
       "API_limit": 100, # API pull number limits for reddit per time
       "API_wait": 1 #API wait time berfore the next pull
      }



df_test_reddit = get_reddit_posts(par["subreddit"], par["post_num"], par["time_1"], par[ "API_limit"], par["API_wait"])<jupyter_output>100 posts downloaded, oldest post:2020-06-20 10:40:38 - status code: 200, now waiting 1 seconds before next pull. Patience...
[92m Does the imported dataframe match the request? True[00m
Final DataFrame shape: (100, 71), there are 0 duplicates<jupyter_text>Cleaning the new dataset<jupyter_code>to_keep_clmns = ['author', 'created_utc', 'domain', 'id', 'num_comments', 'over_18',
       'post_hint', 'score', 'selftext',
       'subreddit_subscribers', 'title']

df_test_reddit = df_test_reddit[to_keep_clmns]

df_keep_reddit = df_test_reddit.copy()

df_test_reddit.head(10)
#For title and selftext columns, I filled them with " " as they will be striped later, so I can merge them later.

df_test_reddit["title"].fillna(" ", inplace=True)
df_test_reddit["selftext"].fillna(" ", inplace=True)

#Merging the title and selftext for further processing

df_test_reddit['text_merged'] = df_test_reddit['title'] + " " + df_test_reddit['selftext']
df_test_reddit.drop(columns = ["title", "selftext"], inplace=True)

#For subscribers, I imputed them with median valeus
df_test_reddit['subreddit_subscribers'].fillna(df_test_reddit['subreddit_subscribers'].median(), inplace=True)

#For post_hint, I imputed them with "Empty"
df_test_reddit['post_hint'].fillna("Empty", inplace=True)
#Lots pof cleaning on text


#Removing "\n" characters
df_test_reddit['text_merged'] = df_test_reddit['text_merged'].apply(lambda x: re.sub('\n', ' ', x))

#Removing the [removed] characters
df_test_reddit['text_merged'] = df_test_reddit['text_merged'].replace('[removed]', ' ')


# Use regular expressions to do a find-and-replace
df_test_reddit['text_merged'] = df_test_reddit['text_merged'].apply(lambda x: re.sub("[^a-zA-Z]", " ", x))

df_test_reddit['text_merged'] = df_test_reddit['text_merged'].apply(lambda x: x.lower())



#Laste step
#Replacing multiple spaces
#Source: https://pythonexamples.org/python-replace-multiple-spaces-with-single-space-in-text-file/
df_test_reddit['text_merged'] = df_test_reddit['text_merged'].apply(lambda x: ' '.join(x.split()))


#Removing stop words and stemming

def remove_stops_stem(item):

    stops = stopwords.words('english')
    words = [w for w in item.split() if w not in stops]#stops
    
    #lemmatizer = WordNetLemmatizer()
    #words = [lemmatizer.lemmatize(i) for i in words]
    
    # Instantiate object of class PorterStemmer.
    p_stemmer = PorterStemmer()
    words = [p_stemmer.stem(i) for i in words]
    
    
    
    words = " ".join(list(words)) # Adding space
    
    return words

df_test_reddit['text_merged'] = df_test_reddit['text_merged'].apply(remove_stops_stem)



#Stemming


df_test_reddit.reset_index(drop=True, inplace=True)<jupyter_output><empty_output><jupyter_text>Engineering the features<jupyter_code>#Counting the charcaters and word in "text_merged"
df_test_reddit["text_char_count"] = df_test_reddit["text_merged"].map(lambda x: len(x))
df_test_reddit["text_word_count"] = df_test_reddit["text_merged"].map(lambda x: len(x.split(" ")))

#Sentiment analyzer
sent = SentimentIntensityAnalyzer()

df_test_reddit['sentiment_score'] = df_test_reddit["text_merged"].apply(lambda x: sent.polarity_scores(x)['compound'])


df_test_reddit['date'] = pd.to_datetime(df_test_reddit['created_utc'],unit='s')


def get_uwa(item):

    words = [w for w in item.split()]
    #print(words)
    
    index_list = []
    
    for i in range(len(words)):
        
        if (words[i] in list(top_words)):
            index_list.append(i)
    
    dist = 0
    
    dist = np.sum(np.array(index_list[1:]) - np.array(index_list[0:-1]))
    
    return dist
top_words = pickle.load(open('../datasets/top_words_overlap.pkl', 'rb'))
df_test_reddit['wua'] = df_test_reddit['text_merged'].apply(get_uwa)<jupyter_output><empty_output><jupyter_text>Loading the pickled trained model and testing it on the new dataset<jupyter_code>lr = pickle.load(open('../datasets/lr.pkl', 'rb'))
X = df_test_reddit['text_merged']
vectorizer = pickle.load(open('../datasets/vectorizer.pkl', 'rb'))
list_stop_words = ["dec", "global", "http", "www", "com", "conspiraci", "warm", "climat", "remov", "theori", "theactualshadow", "co"]



# vectorizer = CountVectorizer(analyzer = "word",
#                              #ngram_range=(1,2),
#                              tokenizer = None,
#                              preprocessor = None,
#                              stop_words = list_stop_words,
#                              max_features = 3000) 


#X = vectorizer.fit_transform(X_train)

#pickle.dump(vectorizer, open('../datasets/vectorizer.pkl', 'wb'))

X_features = vectorizer.transform(X)

X_features.shape
pred = lr.predict(X_features)
df_test_reddit.head(5)
pred[18]
df_test_reddit["preds"] = pred<jupyter_output><empty_output><jupyter_text>-  Filtered posts as non-global warming:<jupyter_code>indices = df_test_reddit[df_test_reddit["preds"]==0].index
indices
df_test_reddit.loc[indices,:]["text_merged"]
df_test_reddit.loc[0,:]["text_merged"]
df_keep_reddit.loc[47,:]["title"]<jupyter_output><empty_output><jupyter_text>-  Filtered posts as global warming:<jupyter_code>indices = df_test_reddit[df_test_reddit["preds"]==1].index
indices
df_test_reddit.loc[indices,:]["text_merged"]
df_test_reddit.loc[97,:]["text_merged"]
print("Hello World !")<jupyter_output>Hello World !
{'fraction_non_alphanumeric': 0.12641968631692807, 'fraction_numerical': 0.010546241211465657, 'mean_word_length': 5.113223140495868, 'pattern_counts': {'":': 5, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '639029', 'license': 'MIT', 'n_tokens_mistral': 2597, 'n_tokens_neox': 2522, 'n_words': 577, 'path': 'Codes/P06_Select_Model_test.ipynb', 'repo_name': 'saiedmighani/Global_warming_NLP_analysis', 'size': '7396'}