<jupyter_start><jupyter_text>Load Data<jupyter_code>X_train = read_file_train()
X_test = read_file_test()
Y_train = read_file_train_output()
set_buffer_nans_to_zero(X_train)
set_buffer_nans_to_zero(X_test)<jupyter_output><empty_output><jupyter_text>Change categorical variables to numerical<jupyter_code>cat_columns = ['zone_id', 'pollutant']
cats_to_drop = ['station_id']
X_train = pd.get_dummies(X_train, columns=cat_columns)
X_test = pd.get_dummies(X_test, columns=cat_columns)<jupyter_output><empty_output><jupyter_text>Normalize Data<jupyter_code>def normalize(X):
    return X.apply(lambda x: (x-x.mean()/x.std()))
X = pd.concat([X_train, X_test])
idx_train = X_train.index
idx_test = X_test.index
normalized_X = normalize(X)
X_train_norm = normalized_X.loc[idx_train]
X_test_norm = normalized_X.loc[idx_test]
X.drop(cats_to_drop, axis=1, inplace=True)
import sklearn.linear_model as lin_mods
def scorer(estimator, X, y):
    return score_function(estimator.predict(X), y)<jupyter_output><empty_output><jupyter_text>Define Regressor<jupyter_code>from sklearn.ensemble.forest import RandomForestRegressor

# lin_regressor = LinearRegression(fit_intercept=False, normalize=False, copy_X=True, n_jobs=1)
# model = Ridge(fit_intercept=False, normalize=False, copy_X=True)
# model = lin_mods.RidgeCV(cv=5, scoring=scorer)
model = RandomForestRegressor(n_estimators=3, n_jobs=5)<jupyter_output><empty_output><jupyter_text>Fit regression to train set<jupyter_code>model.fit(X_train_norm.drop(cats_to_drop, axis=1), Y_train['TARGET'])
pd.Series(model.feature_importances_, index=X.columns)
%matplotlib inline
pd.Series(model.coef_[0], index=X.columns)<jupyter_output><empty_output><jupyter_text>Predict on test set<jupyter_code>y_pred = model.predict(X_test_norm.drop(cats_to_drop, axis=1)).clip(0)

y_pred_train = model.predict(X_train_norm.drop(cats_to_drop, axis=1))
y_pred_train = pd.Series(y_pred_train, index=idx_train, name='TARGET')
y_pred_train
np.mean((np.array(Y_train) - np.array(y_pred_train))**2)
print_to_file_pred(pd.DataFrame(y_pred, index=idx_test, columns=["TARGET"]), 'linear_reg')
Y_train.std(), Y_train.mean(), y_pred_train.std(), y_pred_train.mean()
y_pred2 = np.maximum(y_pred_train, 0)
diff = (y_pred_train - Y_train)
Y_train.plot()
import matplotlib.pyplot as plt
plt.ylim([0, 500])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13142123287671234, 'fraction_numerical': 0.007277397260273972, 'mean_word_length': 10.344660194174757, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '76667', 'license': 'MIT', 'n_tokens_mistral': 959, 'n_tokens_neox': 926, 'n_words': 123, 'path': 'src/fumbling.ipynb', 'repo_name': 'GeoffNN/PlumeChallenge', 'size': '2336'}