<jupyter_start><jupyter_text>Data Pre-Processing<jupyter_code>engine = create_engine("sqlite:///../db/twitter_db.sqlite")
conn = engine.connect()
tweets_df = pd.read_sql("SELECT * FROM tweet_data", conn)
tweets_df.head(2)
tweets_df["created_at_datetime"] = tweets_df["created_at_datetime"].apply(lambda x: dt.datetime.strptime(x,
                                                                "%Y-%m-%d %H:%M:%S.%f"))
tweets_df = tweets_df.loc[tweets_df["created_at_datetime"] < dt.datetime(2019,8,15), :].reset_index(drop=True)
len(tweets_df)
grouped_df = tweets_df.groupby(tweets_df['user_name']).median()
grouped_df = grouped_df[['retweet_count']].sort_values('retweet_count', ascending = False).iloc[:7]
# grouped_df.sort_values('retweet_count', ascendingh = False)
grouped_df
top_candidates =[]
for i, r in grouped_df.iterrows():
    top_candidates.append(i)
# Remove <NAME> from list (dropped out)
top_candidates.pop(3)
top_candidates
tweets_df = tweets_df.loc[tweets_df['user_name'].isin(top_candidates), :].reset_index(drop=True)
len(tweets_df)
tweets_df["day"] = tweets_df["created_at_datetime"].apply(lambda x: dt.datetime.strftime(x, "%A"))
tweets_df["hour"] = tweets_df["created_at_datetime"].apply(lambda x: dt.datetime.strftime(x, "%H"))
tweets_df["month"] = tweets_df["created_at_datetime"].apply(lambda x: dt.datetime.strftime(x, "%B"))
tweets_df.drop(columns = ["created_at", "created_at_time", "created_at_date","created_at_datetime",
                    "tweet_id", "tweet_id_str", "in_reply_to_status_id",
                    "in_reply_to_status_id_str", "in_reply_to_user_id",
                    "in_reply_to_user_id_str", "in_reply_to_screen_name",
                    "user_id_str", "user_id", "user_screen_name", "id"], inplace = True)
tweets_df = tweets_df[["user_name", "month", "day", "hour", "retweet_count", "favorite_count", "full_text"]]
tweets_df.head(2)
len(tweets_df.groupby(tweets_df["user_name"]).count())
import nltk
import re
import string
pd.set_option('display.max_colwidth', 100) # To extend column width

stopwords = nltk.corpus.stopwords.words('english')
wn = nltk.WordNetLemmatizer()
def clean_text(text):
    text = text.replace('&amp;', '&')
    text = text.replace('\n', ' ')
    text = "".join([word.lower() for word in text if word not in string.punctuation])
    tokens = re.split('\W+', text)
    text = [wn.lemmatize(word) for word in tokens if word not in stopwords]
    return text<jupyter_output><empty_output><jupyter_text>Bag of Words<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
X_tdidf_df = tweets_df[["full_text", "month", "day", "hour", "retweet_count", "favorite_count"]]
X_tdidf_df = pd.get_dummies(X_tdidf_df, columns = ["month", "day", "hour"])
# TF-IDF
tfidf_vect = TfidfVectorizer(analyzer=clean_text)
X_tfidf_vect = tfidf_vect.fit_transform(X_tdidf_df['full_text'])
X_tdidf_df.drop(columns = ['full_text'], inplace = True)
X_tdidf_df = pd.concat([X_tdidf_df, pd.DataFrame(X_tfidf_vect.toarray(), columns=tfidf_vect.get_feature_names())], axis=1)
X_tdidf_df.shape
data_y_idf = tweets_df.values
data_x_idf = X_tdidf_df.values
X_count_idf = data_x_idf[:, 0:]
y_count_idf = data_y_idf[:,0]
X_count_idf_sparse = csr_matrix(X_count_idf)
y_count_idf<jupyter_output><empty_output><jupyter_text>Train-Test-Split/Label-Encoding (Grid Search)<jupyter_code>X_train_idf_cv, X_test_idf_cv, y_train_idf_cv, y_test_idf_cv = train_test_split(X_count_idf_sparse, 
                                                                y_count_idf, random_state=42,
                                                               train_size = 0.9,
                                                               shuffle = True)
X_scaler_idf_cv = MaxAbsScaler().fit(X_train_idf_cv)
X_train_scaled_idf_cv = X_scaler_idf_cv.transform(X_train_idf_cv)
X_test_scaled_idf_cv = X_scaler_idf_cv.transform(X_test_idf_cv)
# Step 1: Label-encode data set
label_encoder_idf_cv = LabelEncoder()
label_encoder_idf_cv.fit(y_train_idf_cv)

encoded_y_train_idf_cv = label_encoder_idf_cv.transform(y_train_idf_cv)
encoded_y_test_idf_cv = label_encoder_idf_cv.transform(y_test_idf_cv)
encoded_y_train_idf_cv = encoded_y_train_idf_cv.reshape(-1, 1)
encoded_y_test_idf_cv = encoded_y_test_idf_cv.reshape(-1, 1)<jupyter_output><empty_output><jupyter_text>Train-Test-Split/Label-Encoding (In-Depth Parameter Tuning)<jupyter_code>X_train_idf, X_test_idf, y_train_idf, y_test_idf = train_test_split(X_count_idf_sparse, y_count_idf, random_state=42,
                                                                   shuffle = True)
X_scaler_idf = MaxAbsScaler().fit(X_train_idf)
X_train_scaled_idf = X_scaler_idf.transform(X_train_idf)
X_test_scaled_idf = X_scaler_idf.transform(X_test_idf)
# Step 1: Label-encode data set
label_encoder_idf = LabelEncoder()
label_encoder_idf.fit(y_train_idf)

encoded_y_train_idf = label_encoder_idf.transform(y_train_idf)
encoded_y_test_idf = label_encoder_idf.transform(y_test_idf)
encoded_y_train_idf = encoded_y_train_idf.reshape(-1, 1)
encoded_y_test_idf = encoded_y_test_idf.reshape(-1, 1)<jupyter_output><empty_output><jupyter_text>Gradient Boosting Classifier  In-depth Parameter Tuning  GBC N_estimators (TF-IDF Vectorizer)<jupyter_code>n_estimators = [1, 2, 4, 8, 16, 32, 64, 100, 150, 200, 300]

train_results = []
test_results = []

for estimator in n_estimators:
    gbc = GradientBoostingClassifier(n_estimators=estimator)
    gbc.fit(X_train_scaled_idf, encoded_y_train_idf)

    train_results.append(gbc.score(X_train_scaled_idf, encoded_y_train_idf))
    test_results.append(gbc.score(X_test_scaled_idf, encoded_y_test_idf))
line1, = plt.plot(n_estimators, train_results, 'b', label="Train")
line2, = plt.plot(n_estimators, test_results, 'r', label="Test")

plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})

plt.ylabel('GBC score')
plt.xlabel('n_estimators')
plt.show()
n_estimators = [250, 300, 350, 400]

train_results = []
test_results = []

for estimator in n_estimators:
    gbc = GradientBoostingClassifier(n_estimators=estimator)
    gbc.fit(X_train_scaled_idf, encoded_y_train_idf)

    train_results.append(gbc.score(X_train_scaled_idf, encoded_y_train_idf))
    test_results.append(gbc.score(X_test_scaled_idf, encoded_y_test_idf))
line1, = plt.plot(n_estimators, train_results, 'b', label="Train")
line2, = plt.plot(n_estimators, test_results, 'r', label="Test")

plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})

plt.ylabel('GBC score')
plt.xlabel('n_estimators')
plt.show()
# Best:<jupyter_output><empty_output><jupyter_text>GBC Max_Depth (TF-IDF Vectorizer)<jupyter_code>max_depths = list(np.linspace(1, 90, 90, endpoint=True))
train_results = []
test_results = []
for max_depth in max_depths:
    gbc = GradientBoostingClassifier(max_depth=max_depth)
    gbc.fit(X_train_scaled_idf, encoded_y_train_idf)

    train_results.append(gbc.score(X_train_scaled_idf, encoded_y_train_idf))
    test_results.append(gbc.score(X_test_scaled_idf, encoded_y_test_idf))
line1, = plt.plot(max_depths, train_results, 'b', label="Train")
line2, = plt.plot(max_depths, test_results, 'r', label="Test")

plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})

plt.ylabel('RF score')
plt.xlabel('max_depths')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12868153041563446, 'fraction_numerical': 0.011973575557390587, 'mean_word_length': 5.529200359389039, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '585111', 'license': 'MIT', 'n_tokens_mistral': 2994, 'n_tokens_neox': 2875, 'n_words': 381, 'path': 'jupyter_notebook_code/Gradient Boosting Classifier.ipynb', 'repo_name': 'elanderos318/Tweetocracy', 'size': '7273'}