<jupyter_start><jupyter_text>**Automatic Account Detection** Dataset From - https://www.kaggle.com/c/twitter-bot-classification | https://www.kaggle.com/charvijain27/detecting-twitter-bot-data **Libraries**<jupyter_code># Basic libraries
import numpy as np
import pandas as pd
import pickle

# Data Visualization
import matplotlib.pyplot as plt
import seaborn as sns
%config InlineBackend.figure_format = 'retina'
%matplotlib inline

# Natural Language Processing
from sklearn.feature_extraction import stop_words
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import re
import string
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')
from nltk.stem import PorterStemmer
from nltk.stem import WordNetLemmatizer
stemmer= PorterStemmer()
lemmatizer=WordNetLemmatizer()

# Modeling
import sklearn.ensemble
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn import metrics
from sklearn.metrics import accuracy_score, recall_score, precision_score, confusion_matrix,roc_curve,auc
from imblearn.over_sampling import SMOTE
url_bot_train = 'data/Bot/training_data_2_csv_UTF.csv'
df_bot_train = pd.read_csv(url_bot_train)
training_data = df_bot_train
sns.countplot(training_data['bot'])
training_data.apply(lambda x: sum(x.isnull()))
text_cols=training_data[['screen_name','location','description','url','created_at','lang','status','has_extended_profile','name']]
text_cols.head()
text_cols.rename(columns={'screen_name':'screen_name_processed'}, inplace=True)
text_cols.rename(columns={'name':'name_processed'}, inplace=True)
stop_words = set(stopwords.words('english'))

text_cols['screen_name_processed_num_count'] = ""
for index, row in text_cols.iterrows():
    row['screen_name_processed'] = row['screen_name_processed'].lower() #Convert text to lowercase
    row['screen_name_processed_num_count'] = (sum(c.isdigit() for c in row['screen_name_processed'])) #create new column to get number of numbers
    row['screen_name_processed'] = re.sub(r'\d+','', row['screen_name_processed']) #Remove numbers
    row['screen_name_processed']= row['screen_name_processed'].translate(str.maketrans('','',string.punctuation)) #Remove punctuation
    row['screen_name_processed'] = row['screen_name_processed'].strip() #Remove whitespaces
    row['screen_name_processed'] = [i for i in word_tokenize(row['screen_name_processed']) if not i in stop_words] #Tokenization - REMOVE STOP WORDS
    for word in row['screen_name_processed']:
        row['screen_name_processed'] = lemmatizer.lemmatize(word)
text_cols['name_processed_num_count'] = ""
for index, row in text_cols.iterrows():
    row['name_processed'] = row['name_processed'].lower() #Convert text to lowercase
    row['name_processed_num_count'] = (sum(c.isdigit() for c in row['name_processed'])) #create new column to get number of numbers
    row['name_processed'] = re.sub(r'\d+','', row['name_processed']) #Remove numbers
    row['name_processed']= row['name_processed'].translate(str.maketrans('','',string.punctuation)) #Remove punctuation
    row['name_processed'] = row['name_processed'].strip() #Remove whitespaces
    row['name_processed'] = [i for i in word_tokenize(row['name_processed']) if not i in stop_words] #Tokenization - REMOVE STOP WORDS
text_cols.head()
def impute_na(variable):
    for i, row in text_cols[text_cols[variable].isnull()].iterrows():
        obs_sample = text_cols[variable].dropna().sample(1, random_state=int(row.screen_name_processed_num_count))
        obs_sample.index = [i]
        text_cols.at[i, variable] = obs_sample
impute_na('has_extended_profile')
word_list = r'bot|b0t|cannabis|tweet me|mishear|follow me|updates every|gorilla|yes_ofc|forget' \
                    r'expos|kill|clit|bbb|butt|fuck|XXX|sex|truthe|fake|anony|free|virus|funky|RNA|kuck|jargon' \
                    r'nerd|swag|jack|bang|bonsai|chick|prison|paper|pokem|xx|freak|ffd|dunia|clone|genie|bbb' \
                    r'ffd|onlyman|emoji|joke|troll|droop|free|every|wow|cheese|yeah|bio|magic|wizard|face'
listofwords = pickle.load(open("model/words_in_not_credible.pickle", 'rb'))
listofwords2 = pickle.load(open("model/word_couples_in_not_credible.pickle", 'rb'))
list_ofwords=list(listofwords)
str1 = '|'.join(str(e) for e in list_ofwords)
list_ofwords2=list(listofwords2)
str2 = '|'.join(str(e) for e in list_ofwords2)
text_cols['screen_name_binary'] = training_data.screen_name.str.contains(word_list, case=False, na=False)
text_cols['name_binary'] = training_data.name.str.contains(word_list, case=False, na=False)
text_cols['description_binary'] = training_data.description.str.contains(word_list, case=False, na=False)
text_cols['status_binary'] = training_data.status.str.contains(word_list, case=False, na=False)
word_list = str1 + '|' + str2 + '|' + word_list
text_cols['listed_count_binary'] = (training_data.listed_count>20000)==False
for column in training_data:
    text_cols[column+'_NA'] = np.where(training_data[column].isnull(), 1, 0)
NA_cols = [col for col in text_cols.columns if 'NA' in col]
NA_cols
text_cols.has_extended_profile = text_cols.has_extended_profile.astype(int)
text_cols.rename(columns={'has_extended_profile':'has_extended_profile_processed'}, inplace=True)
training_data['des_hashtags'] = training_data['description'].str.count('#')
training_data['des_mentions'] = training_data['description'].str.count('@')
training_data['des_length'] = training_data['description'].str.len()
training_data['status_hashtags'] = training_data['status'].str.count('#')
training_data['status_mentions'] = training_data['status'].str.count('@')
training_data['status_length'] = training_data['status'].str.len()
training_data['des_link_count'] = training_data['description'].str.count(':')

training_data['status_punctuation'] = training_data['status'].str.count('\.')
training_data['des_punctuation'] = training_data['description'].str.count('\.')
training_data['status_quote'] = training_data['status'].str.count('"')
training_data['des_quote'] = training_data['description'].str.count('"')
text_cols_features = text_cols[['has_extended_profile_processed','name_processed_num_count','screen_name_processed_num_count','screen_name_binary', 'name_binary', 'description_binary', 'status_binary', 'listed_count_binary','location_NA','description_NA','url_NA','status_NA','has_extended_profile_NA']].copy()
training_data_features = training_data[['verified', 'followers_count', 'friends_count', 'statuses_count','bot']].copy()
feature_set = training_data[['status_punctuation','des_punctuation','status_quote','des_quote','des_link_count','des_hashtags', 'des_mentions', 'des_length', 'status_hashtags','status_mentions','status_length']].copy().fillna(0)
feature_set.head(100)
result = pd.concat([feature_set,text_cols_features, training_data_features], axis=1, sort=False)
result.to_csv('data/Bot/processed.csv', index=False)
mask_on = training_data['bot'] == 1
training_data_bot_description = training_data[mask_on]['description']

# Instantiate a CountVectorizer
cv1 = CountVectorizer(stop_words = 'english')

# Fit and transform the vectorizer on our corpus
bot_cvec = cv1.fit_transform(training_data_bot_description.values.astype('U'))

# Convert onion_cvec into a DataFrame
bot_cvec_df = pd.DataFrame(bot_cvec.toarray(),
                   columns=cv1.get_feature_names())

# Inspect head of Onion Titles cvec
print(bot_cvec_df.shape)
def bar_plot(x, y, title, color):    
    
    # Set up barplot 
    plt.figure(figsize=(9,5))
    g=sns.barplot(x, y, color = color)    
    ax=g

    # Label the graph
    plt.title(title, fontsize = 15)
    plt.xticks(fontsize = 10)

    # Enable bar values
    # Code modified from http://robertmitchellv.com/blog-bar-chart-annotations-pandas-mpl.html
    # create a list to collect the plt.patches data
    totals = []

    # find the values and append to list
    for p in ax.patches:
        totals.append(p.get_width())

    # set individual bar lables using above list
    total = sum(totals)

    # set individual bar lables using above list
    for p in ax.patches:
        # get_width pulls left or right; get_y pushes up or down
        ax.text(p.get_width()+.3, p.get_y()+.38, \
                int(p.get_width()), fontsize=10)
bot_wc = bot_cvec_df.sum(axis = 0)
bot_top_10 = bot_wc.sort_values(ascending=False).head(10)

# Call function
bar_plot(bot_top_10.values, bot_top_10.index, 'Top 10 words on Description of bots','r') 
result_no_f = training_data_features
training_result_no_f = result_no_f.drop('bot', axis=1)
X = training_result_no_f.as_matrix()
X_train, X_test, y_train, y_test = train_test_split(X, result_no_f.bot, test_size=0.3,
                                                    random_state=0)
ovrsmple = SMOTE(ratio='minority')
X_train, y_train = ovrsmple.fit_sample(X_train, y_train)
rfc = sklearn.ensemble.RandomForestClassifier(n_estimators=500,
                                              min_samples_split=5,
                                              criterion='gini', 
                                              max_features='auto',
                                              max_depth = 8,
                                              oob_score=True,
                                              random_state=42,
                                              n_jobs=-1)
rfc.fit(X_train, y_train)

print('Random Forest Classifier Train Accuracy Score :', rfc.score(X_train, y_train))
print('Random Forest Classifier Test Score :', rfc.score(X_test, y_test))
training = result.drop('bot', axis=1)
X = training.as_matrix()
X_train, X_test, y_train, y_test = train_test_split(X, result.bot, test_size=0.3,
                                                    random_state=0)
X_train.shape, X_test.shape
ovrsmple = SMOTE(ratio='minority')

X_train, y_train = ovrsmple.fit_sample(X_train, y_train)
np.bincount(y_train)
rfc = sklearn.ensemble.RandomForestClassifier(n_estimators=500,
                                              min_samples_split=5,
                                              criterion='gini', 
                                              max_features='auto',
                                              max_depth = 8,
                                              oob_score=True,
                                              random_state=42,
                                              n_jobs=-1)
rfc.fit(X_train, y_train)

print('Random Forest Classifier Train Accuracy Score :', rfc.score(X_train, y_train))
print('Random Forest Classifier Test Score :', rfc.score(X_test, y_test))
from sklearn.metrics import classification_report
predt = rfc.predict(X_test)
report = classification_report(y_test, predt)
print(report)
plt.figure(figsize=(10,9))
feat_importances = pd.Series(rfc.feature_importances_, index=training.columns)
feat_importances.nlargest(30).plot(kind='barh')
from sklearn.tree import DecisionTreeClassifier
dtc = DecisionTreeClassifier(criterion='gini',
                            max_depth = 10,
                            max_features = 'auto',
                            random_state = 1,
                            splitter = 'best')
dtc.fit(X_train, y_train)
print("Train Score :", dtc.score(X_train, y_train))
print("Test Score :", dtc.score(X_test, y_test))
from sklearn.preprocessing import StandardScaler 
from sklearn.linear_model import LogisticRegressionCV,LogisticRegression

ss = StandardScaler()
X_train_s= ss.fit_transform(X_train) # saved the mean and std from X_train
X_test_s=ss.transform(X_test)


# Create regularization penalty space
penalty = ['l1', 'l2']

# Create regularization hyperparameter space
C = np.logspace(0, 4, 10)

# Create hyperparameter options
hyperparameters = dict(C=C, penalty=penalty)

#initialize grid search and Logistic regression

lr = LogisticRegression(n_jobs=-1)
# clf = GridSearchCV(lr, hyperparameters, cv=5, verbose=0)
# # Train the model using the training sets
# clf.fit(X_train_s, y_train)

lr.fit(X_train_s, y_train)
lr_y_pred = lr.predict(X_test_s)
print("LR train score :")
print(lr.score(X_train_s, y_train))
print('')
print("LR test score  :")

print(lr.score(X_test_s, y_test))
sns.set_style("whitegrid", {'axes.grid' : False})

scores_train = rfc.predict_proba(X_train)
scores_test = rfc.predict_proba(X_test)

y_scores_train = []
y_scores_test = []
for i in range(len(scores_train)):
    y_scores_train.append(scores_train[i][1])

for i in range(len(scores_test)):
    y_scores_test.append(scores_test[i][1])
    
fpr_rf_train, tpr_rf_train, _ = roc_curve(y_train, y_scores_train, pos_label=1)
fpr_rf_test, tpr_rf_test, _ = roc_curve(y_test, y_scores_test, pos_label=1)

plt.plot(fpr_rf_train, tpr_rf_train, color='darkblue', label='Train AUC: %5f' %auc(fpr_rf_train, tpr_rf_train))
plt.plot(fpr_rf_test, tpr_rf_test, color='red', ls='--', label='Test AUC: %5f' %auc(fpr_rf_test, tpr_rf_test))
plt.title("Random Forest ROC Curve for Engineered Features")
plt.xlabel("False Positive Rate (FPR)")
plt.ylabel("True Positive Rate (TPR)")
plt.legend(loc='lower right')
labels=['bot','human']
def plot_confusion_matrix(cm, title='Confusion matrix', cmap=plt.cm.YlOrRd):
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar(shrink=0.7)
    tick_marks = np.arange(len(labels))
    plt.xticks(tick_marks, labels, rotation=45, ha='right', fontsize=12)
    plt.yticks(tick_marks, labels , fontsize=12)
    plt.tight_layout()
    plt.ylabel('True label', fontsize=12)
    plt.xlabel('Predicted label', fontsize=12)
# Create the predictions for Y training data
preds = rfc.predict(X_test)
cm = confusion_matrix(y_test, preds)
np.set_printoptions(precision=2)
print('Confusion matrix')
print(cm)
plt.figure()
plot_confusion_matrix(cm)
# Assign True Neg, False Pos, False Neg, True Pos variables
cnf_matrix = np.array(cm).tolist()

tn_fp, fn_tp = cm

tn, fp = tn_fp
fn, tp = fn_tp
# Print Scores 
print("Accuracy:",round(metrics.accuracy_score(y_test, preds)*100, 2),'%')
print("Precision:",round(metrics.precision_score(y_test, preds)*100, 2), '%')
print("Recall:",round(metrics.recall_score(y_test, preds)*100, 2), '%')
print("Specificity:", round((tn/(tn+fp))*100, 2), '%')
print("Misclassification Rate:", round((fp+fn)/(tn+fp+fn+tn)*100, 2), '%')
pickle.dump(rfc, open("model/model_bot.pickle", "wb"))
model = pickle.load(open("model/model_bot.pickle", 'rb'))
result.head(1)
url_bot_test = 'data/Bot/test_data_4_students.csv'
df_bot_test = pd.read_csv(url_bot_test,sep='\t', encoding = "ISO-8859-1",nrows=4)
text_cols2=df_bot_test[['screen_name','location','description','url','created_at','lang','status','has_extended_profile','name']].copy()
text_cols2.head()
screen_name = "Nishan_CW"
location = "Western Sri Lanka"
description = "Social Innovator | System Designer | Realist | Humanitarian"
url = "https://t.co/zW9UUFzeWa"
created_at = "Mon Sep 19 04:59:39 +0000 2011"
lang = ""
status = "{ 'created_at': 'Wed Jan 08 12:34:56 +0000 2020', 'id': 1214888214726496300, 'id_str': '1214888214726496256', 'text': 'Advances in neural networks and other techniques promise to transform health care while raising profound questions… https://t.co/AWcRAgOXKM', 'truncated': true, 'entities': { 'hashtags': [], 'symbols': [], 'user_mentions': [], 'urls': [ { 'url': 'https://t.co/AWcRAgOXKM', 'expanded_url': 'https://twitter.com/i/web/status/1214888214726496256', 'display_url': 'twitter.com/i/web/status/1…', 'indices': [ 116, 139 ] } ] }, 'source': '<a href='http://twitter.com/download/android' rel='nofollow'>Twitter for Android</a>', 'in_reply_to_status_id': null, 'in_reply_to_status_id_str': null, 'in_reply_to_user_id': null, 'in_reply_to_user_id_str': null, 'in_reply_to_screen_name': null, 'geo': null, 'coordinates': null, 'place': { 'id': '173c2bb9d42baaa5', 'url': 'https://api.twitter.com/1.1/geo/id/173c2bb9d42baaa5.json', 'place_type': 'country', 'name': '<NAME>', 'full_name': '<NAME>', 'country_code': 'LK', 'country': 'Sri Lanka', 'contained_within': [], 'bounding_box': { 'type': 'Polygon', 'coordinates': [ [ [ 79.6505263, 5.9213829 ], [ 81.8787816, 5.9213829 ], [ 81.8787816, 9.8359375 ], [ 79.6505263, 9.8359375 ] ] ] }, 'attributes': {} }, 'contributors': null, 'is_quote_status': false, 'retweet_count': 0, 'favorite_count': 0, 'favorited': false, 'retweeted': false, 'possibly_sensitive': false, 'lang': 'en' }""{ 'created_at': 'Wed Jan 08 12:34:56 +0000 2020', 'id': 1214888214726496300, 'id_str': '1214888214726496256', 'text': 'Advances in neural networks and other techniques promise to transform health care while raising profound questions… https://t.co/AWcRAgOXKM', 'truncated': true, 'entities': { 'hashtags': [], 'symbols': [], 'user_mentions': [], 'urls': [ { 'url': 'https://t.co/AWcRAgOXKM', 'expanded_url': 'https://twitter.com/i/web/status/1214888214726496256', 'display_url': 'twitter.com/i/web/status/1…', 'indices': [ 116, 139 ] } ] }, 'source': '<a href='http://twitter.com/download/android' rel='nofollow'>Twitter for Android</a>', 'in_reply_to_status_id': null, 'in_reply_to_status_id_str': null, 'in_reply_to_user_id': null, 'in_reply_to_user_id_str': null, 'in_reply_to_screen_name': null, 'geo': null, 'coordinates': null, 'place': { 'id': '173c2bb9d42baaa5', 'url': 'https://api.twitter.com/1.1/geo/id/173c2bb9d42baaa5.json', 'place_type': 'country', 'name': 'Sri Lanka', 'full_name': '<NAME>', 'country_code': 'LK', 'country': 'Sri Lanka', 'contained_within': [], 'bounding_box': { 'type': 'Polygon', 'coordinates': [ [ [ 79.6505263, 5.9213829 ], [ 81.8787816, 5.9213829 ], [ 81.8787816, 9.8359375 ], [ 79.6505263, 9.8359375 ] ] ] }, 'attributes': {} }, 'contributors': null, 'is_quote_status': false, 'retweet_count': 0, 'favorite_count': 0, 'favorited': false, 'retweeted': false, 'possibly_sensitive': false, 'lang': 'en' }"
has_extended_profile = True
name = "<NAME>"
verified = False
followers_count = 248
friends_count = 167
statuses_count = 2560
listed_count = 6
df_bot_test = pd.DataFrame(columns=["screen_name", "location", "description","url","created_at","lang","status","has_extended_profile","name","verified","followers_count","friends_count","statuses_count","listed_count"], data=[[screen_name, location, description, url, created_at,lang,status,has_extended_profile,name,verified,followers_count,friends_count, statuses_count,listed_count]])
df_bot_test
text_cols2=df_bot_test[['screen_name','location','description','url','created_at','lang','status','has_extended_profile','name']].copy()
text_cols2.head()
point = []
text_cols2.rename(columns={'screen_name':'screen_name_processed'}, inplace=True)
text_cols2.rename(columns={'name':'name_processed'}, inplace=True)
def convert1(variable):
    stop_words = set(stopwords.words('english'))
    text_cols2[variable+'_processed_num_count'] = ""
    for i, row1 in text_cols2.iterrows():
        row1[variable+'_processed'] = row1[variable+'_processed'].lower() #Convert text to lowercase
        row1[variable+'_processed_num_count'] = sum(ch.isdigit() for ch in row1[variable+'_processed']) #create new column to get number of numbers
        row1[variable+'_processed'] = re.sub(r'\d+','', row1[variable+'_processed']) #Remove numbers
        row1[variable+'_processed']= row1[variable+'_processed'].translate(str.maketrans('','',string.punctuation)) #Remove punctuation
        row1[variable+'_processed'] = row1[variable+'_processed'].strip() #Remove whitespaces
        row1[variable+'_processed'] = [i for i in word_tokenize(row1[variable+'_processed']) if not i in stop_words] #Tokenization - REMOVE STOP WORDS
        for word in row1[variable+'_processed']:
            row1[variable+'_processed'] = lemmatizer.lemmatize(word)
        text_cols2.at[i, 'screen_name_processed'] = row1['screen_name_processed']
        text_cols2.at[i, 'screen_name_processed_num_count'] = row1['screen_name_processed_num_count']
text_cols2.head(4)
convert1('screen_name')
text_cols2['name_processed_num_count'] = ""
for index, row in text_cols2.iterrows():
    row['name_processed'] = row['name_processed'].lower() #Convert text to lowercase
    row['name_processed_num_count'] = (sum(c.isdigit() for c in row['name_processed'])) #create new column to get number of numbers
    row['name_processed'] = re.sub(r'\d+','', row['name_processed']) #Remove numbers
    row['name_processed']= row['name_processed'].translate(str.maketrans('','',string.punctuation)) #Remove punctuation
    row['name_processed'] = row['name_processed'].strip() #Remove whitespaces
    row['name_processed'] = [i for i in word_tokenize(row['name_processed']) if not i in stop_words] #Tokenization - REMOVE STOP WORDS
    text_cols2.at[index, 'name_processed'] = row['name_processed']
    text_cols2.at[index, 'name_processed_num_count'] = row['name_processed_num_count']
text_cols2.head()
word_list = r'bot|b0t|cannabis|tweet me|mishear|follow me|updates every|gorilla|yes_ofc|forget' \
                    r'expos|kill|clit|bbb|butt|fuck|XXX|sex|truthe|fake|anony|free|virus|funky|RNA|kuck|jargon' \
                    r'nerd|swag|jack|bang|bonsai|chick|prison|paper|pokem|xx|freak|ffd|dunia|clone|genie|bbb' \
                    r'ffd|onlyman|emoji|joke|troll|droop|free|every|wow|cheese|yeah|bio|magic|wizard|face'
listofwords = pickle.load(open("model/words_in_not_credible.pickle", 'rb'))
listofwords2 = pickle.load(open("model/word_couples_in_not_credible.pickle", 'rb'))
list_ofwords=list(listofwords)
str1 = '|'.join(str(e) for e in list_ofwords)
list_ofwords2=list(listofwords2)
str2 = '|'.join(str(e) for e in list_ofwords2)
word_list = str1 + '|' + str2 + '|' + word_list
word_list
text_cols2['screen_name_binary'] = df_bot_test.screen_name.str.contains(word_list, case=False, na=False)
text_cols2['name_binary'] = df_bot_test.name.str.contains(word_list, case=False, na=False)
text_cols2['description_binary'] = df_bot_test.description.str.contains(word_list, case=False, na=False)
text_cols2['status_binary'] = df_bot_test.status.str.contains(word_list, case=False, na=False)
text_cols2['listed_count_binary'] = (df_bot_test.listed_count>20000)==False
for column in df_bot_test:
    text_cols2[column+'_NA'] = np.where(df_bot_test[column].isnull(), 1, 0)
def impute_na(variable):
    for i, row in text_cols2[text_cols2[variable].isnull()].iterrows():
        obs_sample = text_cols2[variable].dropna().sample(1, random_state=int(row.screen_name_processed_num_count))
        obs_sample.index = [i]
        text_cols2.at[i, variable] = obs_sample

impute_na('has_extended_profile')
text_cols2['has_extended_profile'].head(20)
text_cols2.has_extended_profile = text_cols2.has_extended_profile.astype(int)
df_bot_test['des_hashtags'] = df_bot_test['description'].str.count('#')
df_bot_test['des_mentions'] = df_bot_test['description'].str.count('@')
df_bot_test['des_length'] = df_bot_test['description'].str.len()
df_bot_test['status_hashtags'] = df_bot_test['status'].str.count('#')
df_bot_test['status_mentions'] = df_bot_test['status'].str.count('@')
df_bot_test['status_length'] = df_bot_test['status'].str.len()
df_bot_test['des_link_count'] = df_bot_test['description'].str.count(':')

df_bot_test['status_punctuation'] = df_bot_test['status'].str.count('\.')
df_bot_test['des_punctuation'] = df_bot_test['description'].str.count('\.')
df_bot_test['status_quote'] = df_bot_test['status'].str.count('"')
df_bot_test['des_quote'] = df_bot_test['description'].str.count('"')
feature_set = df_bot_test[['status_punctuation','des_punctuation','status_quote','des_quote','des_link_count','des_hashtags', 'des_mentions', 'des_length', 'status_hashtags','status_mentions','status_length']].copy().fillna(0)
text_cols2.rename(columns={'has_extended_profile':'has_extended_profile_processed'}, inplace=True)
text_cols_features = text_cols2[['has_extended_profile_processed','name_processed_num_count','screen_name_processed_num_count','screen_name_binary', 'name_binary', 'description_binary', 'status_binary', 'listed_count_binary','location_NA','description_NA','url_NA','status_NA','has_extended_profile_NA']].copy()
test_data_features = df_bot_test[['verified', 'followers_count', 'friends_count', 'statuses_count']].copy()

result = pd.concat([feature_set,text_cols_features, test_data_features], axis=1, sort=False)
point = result.head(1).to_numpy()
# point = [1, 0, 0, False, False, True, False, True, 0, 0, 0, 0, 0, True,
#        1541514, 68, 14230]
point
# pred = model.predict(np.asarray(point).reshape(1, -1))
# pred[0]
pred = model.predict(point.reshape(1, -1))
pred[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15093342010005287, 'fraction_numerical': 0.024565827469801113, 'mean_word_length': 5.604351329572925, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 11, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 37, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '512933', 'license': 'MIT', 'n_tokens_mistral': 8971, 'n_tokens_neox': 8412, 'n_words': 1443, 'path': 'Automated_Account_Detection_Twitter_Data.ipynb', 'repo_name': 'AsyncWave/strainer-deception-and-bots-notebooks', 'size': '24608'}