<jupyter_start><jupyter_text>Get Tweets<jupyter_code>!pip install nest_asyncio
!pip install twint
import pandas as pd
import nest_asyncio
import twint
#UPLOAD THE DATASETS FIRST
fakenews = pd.read_csv(path)
realnews = pd.read_csv(path)<jupyter_output><empty_output><jupyter_text>User Details<jupyter_code>import pandas as pd
import requests
import time
import twint
##pip install --upgrade -e git+https://github.com/twintproject/twint.git@origin/master#egg=twint
#Upload Fake tweets and Real tweets dataset
df1 = pd.read_csv("faketweetsfinal.csv")
df = pd.DataFrame(df1["username"].unique().tolist()
for i in df["0"]:
    try:
        c = twint.Config()
        c.Username = str(i)
        c.Custom["id"] = ["name","followers", "following"]
        c.Output = "userfake.csv"
        c.Store_csv = True
        twint.run.Lookup(c)
    except Exception as e:
        print(e)
        print(i)

df2 = pd.read_csv("realtweetsfinal.csv")
df = pd.DataFrame(df2["username"].unique().tolist()
for i in df["0"]:
    try:
        c = twint.Config()
        c.Username = str(i)
        c.Custom["id"] = ["name","followers", "following"]
        c.Output = "userreal.csv"
        c.Store_csv = True
        twint.run.Lookup(c)
    except Exception as e:
        print(e)
        print(i)<jupyter_output><empty_output><jupyter_text>Fake Tweets Visualization<jupyter_code>import matplotlib.pyplot as plt
fake = pd.read_csv('/content/drive/My Drive/Internship/Scrapped Data/faketweetsfinal.csv')
fake
fake[['nlikes','nretweets','nreplies']].mode()#.plot(kind="bar")
fake['nlikes'].value_counts().plot(kind='pie')
fake['nretweets'].value_counts().plot(kind='pie')
fake['nreplies'].value_counts().plot(kind='pie')
fake.iloc[1235,:]<jupyter_output><empty_output><jupyter_text>Real Tweets Visualization<jupyter_code>import matplotlib.pyplot as plt
real = pd.read_csv('/content/drive/My Drive/Internship/Scrapped Data/realtweetsfinal.csv')
real
real['nlikes'].value_counts().plot(kind='pie')
real['nretweets'].value_counts().plot(kind='pie')
real['nreplies'].value_counts().plot(kind='pie')
real['nlikes'].max()
real['nlikes'].min()
real['nretweets'].max()
real['nretweets'].min()
real['nreplies'].max()
real['nreplies'].min()<jupyter_output><empty_output><jupyter_text>Tweet Features<jupyter_code># Import libraries
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import nltk 
import string
import re
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.neural_network import MLPClassifier
nltk.download('stopwords')
nltk.download('wordnet')
# Load dataset
real = pd.read_csv("/content/realtweetsfinal.csv")
fake = pd.read_csv("/content/faketweetsfinal.csv")<jupyter_output><empty_output><jupyter_text><jupyter_code>real['class'] = 1
fake['class'] = 0
#df  = real.append(fake,ignore_index=True)
#df = df[['username', 'tweet', 'class']]
df = df[['username', 'tweet']]
user = pd.read_csv("/content/MLPUser.csv")
df = pd.merge(left = user, right=df, on='username', how='inner')
stopword = nltk.corpus.stopwords.words('english')
stopword.extend(['yr', 'year', 'woman', 'man', 'girl','boy','one', 'two', 'sixteen', 'yearold', 'fu', 'weeks', 'week','treatment', 'associated', 'patients', 'may','day', 'case','old'])
def remove_punct(text):
    text = re.sub(r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))", '', text)
    text  = "".join([char for char in text if char not in string.punctuation])
    text = re.sub('[0-9]+', '', text)
    return text

def tokenization(text):
    text = re.split('\W+', text)
    return text

def remove_stopwords(text):
    text = [word for word in text if word not in stopword]
    return text
    
ps = nltk.PorterStemmer()
def stemming(text):
    text = [ps.stem(word) for word in text]
    return text

def url(text):
    regex = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))"
    url = re.findall(regex,text)
    return len(url)

def special(text):
    length = len(text)
    text = remove_punct(text)
    special = length - len(text)
    return special

df['tweet_punct'] = df['tweet'].apply(lambda x: remove_punct(x))
df['tweet_tokenized'] = df['tweet_punct'].apply(lambda x: tokenization(x.lower()))
df['tweet_nonstop'] = df['tweet_tokenized'].apply(lambda x: remove_stopwords(x))
df['tweet_stemmed'] = df['tweet_nonstop'].apply(lambda x: stemming(x))
df['tweet_length'] = df['tweet'].apply(len)
df['special'] = df['tweet'].apply(special)
df['url'] = df['tweet'].apply(url)
df['tokens'] = df['tweet_tokenized'].apply(len)
df['nonstop'] = df['tweet_nonstop'].apply(len)
df['stemmed'] = df['tweet_stemmed'].apply(len)
df.columns
real = pd.read_csv('/content/drive/My Drive/Internship/Scrapped Data/realtweetsfinal.csv')
fake = pd.read_csv('/content/drive/My Drive/Internship/Scrapped Data/faketweetsfinal.csv')<jupyter_output><empty_output><jupyter_text>Box Plots<jupyter_code>fake_tweets.boxplot(column = ['Characters', 'urls', 'SpecialCharacters', 'Words', 'UsefulWords'])
real_tweets.boxplot(column = ['Characters', 'urls', 'SpecialCharacters', 'Words', 'UsefulWords'])<jupyter_output><empty_output><jupyter_text>Characters<jupyter_code>fake_tweets.boxplot(column = ['Characters'])
real_tweets.boxplot(column = ['Characters'])<jupyter_output><empty_output><jupyter_text>Useful Words<jupyter_code>fake_tweets.boxplot(column = ['UsefulWords'])
real_tweets.boxplot(column = ['UsefulWords'])<jupyter_output><empty_output><jupyter_text>URLS<jupyter_code>fake_tweets.boxplot(column = [ 'urls'])
real_tweets.boxplot(column = [ 'urls'])<jupyter_output><empty_output><jupyter_text>Special chaaracters<jupyter_code>fake_tweets.boxplot(column = [ 'SpecialCharacters'])
real_tweets.boxplot(column = [ 'SpecialCharacters'])<jupyter_output><empty_output><jupyter_text>Words<jupyter_code>fake_tweets.boxplot(column = [ 'Words'])
real_tweets.boxplot(column = [ 'Words'])<jupyter_output><empty_output><jupyter_text>**Models**  Without Bio<jupyter_code>data = data[['age','tweets','following','followers','likes','media','class']]
data.head()
data.groupby("class").describe()
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import accuracy_score
data = shuffle(data)
data.reset_index(inplace=True, drop=True)
X = data.drop('class',axis=1)
y = data['class']
sc = StandardScaler()
X = sc.fit_transform(X)
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.2,stratify=y,random_state=1) 
clf = MLPClassifier(hidden_layer_sizes=(25,20,10), max_iter=500,activation = 'tanh',solver='adam',random_state=1)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
from sklearn.metrics import classification_report, confusion_matrix
conf_mat= confusion_matrix(y_test, y_pred)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
sn.heatmap(conf_mat, annot=True, cmap="rocket_r")
acc = conf_mat.trace()/conf_mat.sum()
print("Accuracy: %.2f percent"%(acc*100))<jupyter_output>Accuracy: 76.72 percent<jupyter_text>Using Bio<jupyter_code>import re
import string
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import accuracy_score
data = data[['age','bio','tweets','following','followers','likes','media','class']]
data['bio'].isnull().values.sum()
data.columns
data = shuffle(data)
data.reset_index(inplace=True, drop=True)
X = data.drop(['class','bio'],axis=1)
y = data['class']
sc = StandardScaler()
X = sc.fit_transform(X)
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.2,stratify=y,random_state=1) 
clf = MLPClassifier(hidden_layer_sizes=(25,20,10), max_iter=500,activation = 'tanh',solver='adam',random_state=1)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
from sklearn.metrics import classification_report, confusion_matrix
conf_mat= confusion_matrix(y_test, y_pred)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
sn.heatmap(conf_mat, annot=True, cmap="rocket_r")
acc = conf_mat.trace()/conf_mat.sum()
print("Accuracy: %.2f percent"%(acc*100))<jupyter_output>Accuracy: 74.00 percent<jupyter_text>Feature Model<jupyter_code>def model(data):
    y = data['class']
    X = data.drop('class',axis=1)
    min_max_scaler = MinMaxScaler()
    X = min_max_scaler.fit_transform(X)
    X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.2,stratify=y,random_state=1) 
    clf = MLPClassifier(hidden_layer_sizes=(30,20,15), max_iter=500,activation = 'relu',solver='adam',random_state=1)
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    from sklearn.metrics import classification_report, confusion_matrix
    conf_mat= confusion_matrix(y_test, y_pred)
    print(confusion_matrix(y_test, y_pred))
    print(classification_report(y_test, y_pred))
    acc = conf_mat.trace()/conf_mat.sum()
    print("Accuracy: %.2f percent"%(acc*100))
    print(data.columns)
col = ['tweet_length', 'special', 'url', 'tokens', 'nonstop', 'stemmed']
for i in range(1,len(col)+1):
    temp = ['class']
    for j in range(0,i):
        temp.append(col[j])
    data = df[temp]
    data = data.dropna()
    data = shuffle(data)
    data.reset_index(inplace=True, drop=True)
    model(data)
    print('*'*100)<jupyter_output>[[8434 1837]
 [2356 3499]]
              precision    recall  f1-score   support

           0       0.78      0.82      0.80     10271
           1       0.66      0.60      0.63      5855

    accuracy                           0.74     16126
   macro avg       0.72      0.71      0.71     16126
weighted avg       0.74      0.74      0.74     16126

Accuracy: 74.00 percent
Index(['class', 'tweet_length'], dtype='object')
****************************************************************************************************
[[9180 1091]
 [ 804 5051]]
              precision    recall  f1-score   support

           0       0.92      0.89      0.91     10271
           1       0.82      0.86      0.84      5855

    accuracy                           0.88     16126
   macro avg       0.87      0.88      0.87     16126
weighted avg       0.88      0.88      0.88     16126

Accuracy: 88.25 percent
Index(['class', 'tweet_length', 'special'], dtype='object')
**********************************[...]
{'fraction_non_alphanumeric': 0.18745935903390618, 'fraction_numerical': 0.024616813748258245, 'mean_word_length': 5.233931673422119, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 70, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '605164', 'license': 'MIT', 'n_tokens_mistral': 4158, 'n_tokens_neox': 3971, 'n_words': 651, 'path': 'Code.ipynb', 'repo_name': 'Sanchit112/FakeNewsTweets', 'size': '10765'}