<jupyter_start><jupyter_text>Variables<jupyter_code># Change this
date_col = 'Time'
predicted_col = 'day_close'
train_percentage = 1
add_regressors_to_prophet = True
shift_length = 100 # Only affects ML models
save_prediction_prophet = True # Starts predicting t=len(training_set)
# No need to change this
assert shift_length >= 0<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>df = load_data()
df.head(10)
df = preprocess_df(df, date_col, scale= False)
df
df.columns<jupyter_output><empty_output><jupyter_text>Analysis<jupyter_code># This can provide lots of functions related to rolling averages etc.
ewm = df.ewm(span=7)
ewm.mean()["day_close"]<jupyter_output><empty_output><jupyter_text>Prediction  Thetaforecaster uses simple exponential smoothing![Formula](https://i.imgur.com/5mTIBwo.png) More here: https://www.youtube.com/watch?v=k_HN0wOKDd0 Documentation: https://www.sktime.org/en/v0.4.2/modules/auto_generated/sktime.forecasting.theta.ThetaForecaster.html<jupyter_code>if train_percentage == 1: # Testing testing
    forecaster = ThetaForecaster()
    forecaster.fit(df[predicted_col])
else:
    y_train, y_test = temporal_train_test_split(df[predicted_col], train_size=train_percentage)
    fh = ForecastingHorizon(y_test.index, is_relative=False)
    forecaster = ThetaForecaster()
    forecaster.fit(y_train)
    y_pred = forecaster.predict(fh)
    mean_absolute_scaled_error(y_test, y_pred, y_train=y_train)
    print(f"""
        Mean Absolute Scaled Error: {mean_absolute_scaled_error(y_test, y_pred, y_train=y_train)}\n
        Mean Absolute Error: {mean_absolute_error(y_test, y_pred)}\n
        Mean Squared Error: {mean_squared_error(y_test, y_pred)}
    """)<jupyter_output><empty_output><jupyter_text>Prophet<jupyter_code>addtional_variables = [] if not add_regressors_to_prophet else list(filter(lambda x: x not in (date_col, predicted_col), df.columns)) 
pf_df = preprocess_for_prophet(df, date_col, predicted_col, extra_columns=addtional_variables)

if shift_length:
    shifted_df = pf_df[-shift_length:][addtional_variables] # Contains values dropped during shifting
    pf_df[addtional_variables] = pf_df[addtional_variables].shift(shift_length)
    pf_df = pf_df.dropna()

if train_percentage == 1:
    pf_train = pf_df
else:
    pf_train, pf_test = train_test_split(pf_df, train_size=train_percentage, shuffle=False)

pf = Prophet()
for variable in addtional_variables:
    pf.add_regressor(variable)
pf.fit(pf_train)
if train_percentage != 1:
    forecast = pf.predict(pf_test)
    print(forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']])
    pf.plot(forecast)
    pf.plot_components(forecast)
    print(f"""
        Mean Absolute Scaled Error: {mean_absolute_scaled_error(pf_test["y"], forecast["yhat"], y_train=pf_train["y"])}\n
        Mean Absolute Error: {mean_absolute_error(pf_test["y"], forecast["yhat"])}\n
        Mean Squared Error: {mean_squared_error(pf_test["y"], forecast["yhat"])}
    """)
if shift_length:
    shifted_df["ds"] = pf.make_future_dataframe(shift_length, include_history=False)["ds"].values
    prediction_df = pf.predict(shifted_df)
    print(prediction_df["yhat"])
    if save_prediction_prophet or True:
        prediction_df[["ds", "yhat", "yhat_lower", "yhat_upper", "trend", "trend_lower", "trend_upper"]].to_csv(\
            f"../data/results/prophet/predicted={predicted_col}_shift={shift_length}_start={prediction_df.iloc[0]['ds'].date()}_regressors={add_regressors_to_prophet}.csv", index=False)
with open(f'../saves/model for {shift_length}days.json', 'w') as fout:
    json.dump(model_to_json(pf), fout)  # Save model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12606193477665115, 'fraction_numerical': 0.004932858317347218, 'mean_word_length': 6.270916334661354, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '72449', 'license': 'MIT', 'n_tokens_mistral': 1361, 'n_tokens_neox': 1315, 'n_words': 215, 'path': 'model/forecasting.ipynb', 'repo_name': 'hazemkya/BitDictor', 'size': '3649'}