<jupyter_start><jupyter_text>Data<jupyter_code># url = 'https://raw.githubusercontent.com/k4t0mono/hashtag_analysis/master/data_log.csv'
dataset = pd.read_csv('data_log.csv')

with_std = [ 'std' in x for x in dataset.columns]
with_std = np.extract(with_std, dataset.columns)
print(with_std)
# dataset.drop(with_std, axis=1, inplace=True)
dataset.drop(['total_users'], axis=1, inplace=True)

train_dataset = dataset.sample(frac=0.8, random_state=0xe621)
test_dataset = dataset.drop(train_dataset.index)

train_labels = train_dataset.pop('total_tweets')
train_db = train_dataset.pop('db')
test_labels = test_dataset.pop('total_tweets')
test_db = test_dataset.pop('db')
train_stats = train_dataset.describe().transpose()
train_labels_stats = train_labels.describe().transpose()
def norm(x):
    return (x - train_stats['mean']) / train_stats['std']

def denorm(z):
    return z * train_labels_stats['std'] + train_labels_stats['mean']

normed_train_data = norm(train_dataset)
normed_test_data = norm(test_dataset)<jupyter_output><empty_output><jupyter_text>Graphs<jupyter_code>def plot_prediction(test_labels, test_predictions):
    plt.scatter(test_labels, test_predictions)
    plt.xlabel('True Values [log(x)]')
    plt.ylabel('Predictions [log(x)]')
    plt.axis('equal')
    plt.axis('square')
    plt.xlim([0,plt.xlim()[1]])
    plt.ylim([0,plt.ylim()[1]])
    _ = plt.plot([-5, 5], [-5, 5])
def plot_history(history):
    hist = pd.DataFrame(history.history)
    hist['epoch'] = history.epoch

    plt.figure()
    plt.xlabel('Epoch')
    plt.ylabel('Mean Abs Error [MPG]')
    plt.plot(hist['epoch'], hist['mae'],
           label='Train Error')
    plt.plot(hist['epoch'], hist['val_mae'],
           label = 'Val Error')
    #   plt.ylim([0,5])
    plt.legend()

    plt.figure()
    plt.xlabel('Epoch')
    plt.ylabel('Mean Square Error [$MPG^2$]')
    plt.plot(hist['epoch'], hist['mse'],
           label='Train Error')
    plt.plot(hist['epoch'], hist['val_mse'],
           label = 'Val Error')
    #   plt.ylim([0,20])
    plt.legend()

    plt.figure()
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.plot(hist['epoch'], hist['loss'],
           label='Loss')
    plt.plot(hist['epoch'], hist['val_loss'],
           label='Val Loss')
    plt.legend()

    plt.show()<jupyter_output><empty_output><jupyter_text>Neural Network<jupyter_code>def build_model(hidden_layers, std=False):
    shape = len(train_dataset.keys())
        
    layers = [
        tf.keras.layers.Dense(41, activation='relu', input_shape=[shape])
    ]
    
    for layer_config in hidden_layers:
        if len(layer_config) == 1:
            size, activation = layer_config[0], 'relu'
        else:
            size, activation = layer_config
        
        layers.append(tf.keras.layers.Dense(size, activation=activation))
    
    layers.append(tf.keras.layers.Dense(1))
    
    model = tf.keras.Sequential(layers)

    optimizer = tf.keras.optimizers.RMSprop(0.001)

    model.compile(
        loss='mse', optimizer=optimizer, metrics=['mae', 'mse']
    )
    
    return model
class PrintDot(tf.keras.callbacks.Callback):
  def on_epoch_end(self, epoch, logs):
    if epoch % 100 == 0: print('')
    print('.', end='')<jupyter_output><empty_output><jupyter_text>82 on the hidden<jupyter_code>nnr_82 = build_model([ (82, 'relu') ])

EPOCHS = 1000

early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)

history = nnr_82.fit(normed_train_data, train_labels, epochs=EPOCHS,
                    validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])<jupyter_output>WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'pandas.core.frame.DataFrame'>, <class 'NoneType'>

.................................................<jupyter_text>41 on the hidden<jupyter_code>nnr_41 = build_model([ (41, 'relu') ], True)

EPOCHS = 1000

early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)

history = nnr_41.fit(normed_train_data, train_labels, epochs=EPOCHS,
                    validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])<jupyter_output>WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'pandas.core.frame.DataFrame'>, <class 'NoneType'>

...........................................................................................<jupyter_text>20 on the hidden<jupyter_code>nnr_20 = build_model([ (20, 'relu') ], True)

EPOCHS = 1000

early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)

history = nnr_20.fit(normed_train_data, train_labels, epochs=EPOCHS,
                    validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])<jupyter_output>WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'pandas.core.frame.DataFrame'>, <class 'NoneType'>

.......................................................................................<jupyter_text>Regression Tree  Default<jupyter_code>tr_default = DecisionTreeRegressor(criterion='mse', max_features=1)
tr_default = tr_default.fit(normed_train_data, train_labels)<jupyter_output><empty_output><jupyter_text>Depth 3 Features 1<jupyter_code>tr_d3f1 = DecisionTreeRegressor(criterion='mse', max_depth=3, max_features=1)
tr_d3f1 = tr_d3f1.fit(normed_train_data, train_labels)<jupyter_output><empty_output><jupyter_text>Depth 3 Features 3<jupyter_code>tr_d3f3 = DecisionTreeRegressor(criterion='mse', max_depth=3, max_features=1)
tr_d3f3 = tr_d3f3.fit(normed_train_data, train_labels)<jupyter_output><empty_output><jupyter_text>Support Vector Regression  RBF<jupyter_code>svr_rbf = SVR(kernel='rbf', gamma='scale')
svr_rbf = svr_rbf.fit(normed_train_data, train_labels)<jupyter_output><empty_output><jupyter_text>Polynomial<jupyter_code>svr_poly = SVR(kernel='poly', gamma='scale')
svr_poly = svr_poly.fit(normed_train_data, train_labels)<jupyter_output><empty_output><jupyter_text>Linear<jupyter_code>svr_linear = SVR(kernel='linear', gamma='scale')
svr_linear = svr_linear.fit(normed_train_data, train_labels)<jupyter_output><empty_output><jupyter_text>Linear Model  Ordinary Least Squares<jupyter_code>linear_reg = LinearRegression()
linear_reg = linear_reg.fit(normed_train_data, train_labels)<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>predicts = {
    'nnr_82_test': nnr_82.predict(normed_test_data),
    'nnr_82_train': nnr_82.predict(normed_train_data),
    'nnr_41_test': nnr_41.predict(normed_test_data),
    'nnr_41_train': nnr_41.predict(normed_train_data),
    'nnr_20_test': nnr_20.predict(normed_test_data),
    'nnr_20_train': nnr_20.predict(normed_train_data),
    'tr_default_test': tr_default.predict(normed_test_data),
    'tr_default_train': tr_default.predict(normed_train_data),
    'tr_d3f1_test': tr_d3f1.predict(normed_test_data),
    'tr_d3f1_train': tr_d3f1.predict(normed_train_data),
    'tr_d3f3_test': tr_d3f3.predict(normed_test_data),
    'tr_d3f3_train': tr_d3f3.predict(normed_train_data),
    'svr_rbf_test': svr_rbf.predict(normed_test_data),
    'svr_rbf_train': svr_rbf.predict(normed_train_data),
    'svr_poly_test': svr_poly.predict(normed_test_data),
    'svr_poly_train': svr_poly.predict(normed_train_data),
    'svr_linear_test': svr_linear.predict(normed_test_data),
    'svr_linear_train': svr_linear.predict(normed_train_data),
    'linear_reg_test': linear_reg.predict(normed_test_data),
    'linear_reg_train': linear_reg.predict(normed_train_data),
}

labels = {
    'test': test_labels,
    'train': train_labels,
}
data = []
for c in [ 'nnr_82', 'nnr_41', 'nnr_20', 'tr_default', 'tr_d3f1', 'tr_d3f3', 'svr_rbf', 'svr_poly', 'svr_linear', 'linear_reg' ]:
    d = { 'config': c }
    for g in [ 'test', 'train' ]:
        k = '{}_{}'.format(c, g)
        d['{}_mse'.format(g)] = sklearn.metrics.mean_squared_error(labels[g], predicts[k])
        d['{}_mae'.format(g)] = sklearn.metrics.mean_absolute_error(labels[g], predicts[k])
        d['{}_r2'.format(g)] = sklearn.metrics.r2_score(labels[g], predicts[k])
        
    data.append(d)
results = pd.DataFrame(data)
results.to_csv('results_notu.csv', index=False)
results
plot_prediction(labels['test'], predicts['tr_d3f3_test'])
plot_prediction(labels['train'], predicts['tr_d3f3_train'])
results.sort_values('test_mse')
for i in linear_reg.coef_.argsort()[-5:][::-1]:
    print(i, normed_test_data.columns[i], linear_reg.coef_[i])<jupyter_output>31 tweet_rt_std_3 0.5002862648258279
27 tweet_per_user_std_3 0.3812269315989635
15 tweet_rt_std_1 0.37709398029293734
18 tweet_per_user_mean_2 0.3357138951180257
23 tweet_rt_std_2 0.32751776919933756
{'fraction_non_alphanumeric': 0.16609549920036554, 'fraction_numerical': 0.0310715101667809, 'mean_word_length': 4.707301173402868, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '321771', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 3427, 'n_tokens_neox': 3240, 'n_words': 463, 'path': 'regression.ipynb', 'repo_name': 'k4t0mono/hashtag_analysis', 'size': '8754'}