helloworld

app.py

@@ -0,0 +1,139 @@
+import os
+
+import numpy as np
+import pandas as pd
+import pinecone
+import streamlit as st
+
+from src.utils import (
+    tsfeatures_vector, get_closest_ids, 
+    plot_best_models_count, plot_closest_series,
+    get_catalogue
+)
+
+CATALOGUE = get_catalogue()
+pinecone.init(
+    api_key=os.environ['API_KEY'],
+    environment=os.environ['ENVIRONMENT'],
+)
+INDEX = pinecone.Index(os.environ['INDEX_NAME'])
+DATASETS = {
+    "Demand (AirPassengers)": "https://raw.githubusercontent.com/Nixtla/transfer-learning-time-series/main/datasets/air_passengers.csv",
+    "Electricity (Ercot COAST)": "https://raw.githubusercontent.com/Nixtla/transfer-learning-time-series/main/datasets/ercot_COAST.csv",
+    #"Electriciy (ERCOT, multiple markets)": "https://raw.githubusercontent.com/Nixtla/transfer-learning-time-series/main/datasets/ercot_multiple_ts.csv",
+    "Web Traffic (Peyton Manning)": "https://raw.githubusercontent.com/Nixtla/transfer-learning-time-series/main/datasets/peyton_manning.csv",
+    "Finance (Exchange USD-EUR)": "https://raw.githubusercontent.com/Nixtla/transfer-learning-time-series/main/datasets/usdeur.csv",
+}
+
+
+
+def st_timenet_features():
+    st.set_page_config(
+        page_title="TimeNet Insights",
+        page_icon="🔮",
+        layout="wide",
+        initial_sidebar_state="expanded",
+    )
+
+    st.title(
+        "TimeNet Insights: Revolutionizing Time Series by Nixtla"
+    )
+    st.write(
+        "<style>div.block-container{padding-top:2rem;}</style>", unsafe_allow_html=True
+    )
+
+    intro = """
+    This tool is designed to perform time series analysis through comparative studies with the TimeNet dataset, curated by Nixtla.
+
+    Beyond simply identifying the most similar series within our dataset to your uploaded data, this application provides insights into your time series, especially in terms of forecasting model performance.
+
+    This app allows you to determine which predictive models might perform optimally for your particular data. The comparison between your series and similar ones within the TimeNet dataset can enhance your understanding of the context of your time series and facilitate better-informed forecasting decisions.
+	"""
+    st.write(intro)
+
+    required_cols = ["ds", "y"]
+
+    with st.sidebar.expander("Dataset", expanded=True):
+        data_selection = st.selectbox("Select example dataset", DATASETS.keys())
+        data_url = DATASETS[data_selection]
+        url_json = st.text_input("Data (you can pass your own url here)", data_url)
+        st.write(
+            "You can also upload a CSV file like [this one](https://github.com/Nixtla/transfer-learning-time-series/blob/main/datasets/air_passengers.csv)."
+        )
+
+        uploaded_file = st.file_uploader("Upload CSV")
+        with st.form("Data"):
+
+            if uploaded_file is not None:
+                df = pd.read_csv(uploaded_file)
+                cols = df.columns
+                timestamp_col = st.selectbox("Timestamp column", options=cols)
+                value_col = st.selectbox("Value column", options=cols)
+            else:
+                timestamp_col = st.text_input("Timestamp column", value="timestamp")
+                value_col = st.text_input("Value column", value="value")
+            st.write("You must press Submit each time you want to forecast.")
+            submitted = st.form_submit_button("Submit")
+            if submitted:
+                if uploaded_file is None:
+                    st.write("Please provide a dataframe.")
+                    if url_json.endswith("json"):
+                        df = pd.read_json(url_json)
+                    else:
+                        df = pd.read_csv(url_json)
+                    df = df.rename(
+                        columns=dict(zip([timestamp_col, value_col], required_cols))
+                    )
+                else:
+                    # df = pd.read_csv(uploaded_file)
+                    df = df.rename(
+                        columns=dict(zip([timestamp_col, value_col], required_cols))
+                    )
+            else:
+                if url_json.endswith("json"):
+                    df = pd.read_json(url_json)
+                else:
+                    df = pd.read_csv(url_json)
+                cols = df.columns
+                if "unique_id" in cols:
+                    cols = cols[-2:]
+                df = df.rename(columns=dict(zip(cols, required_cols)))
+
+            if "unique_id" not in df:
+                df.insert(0, "unique_id", "ts_0")
+
+            df["ds"] = pd.to_datetime(df["ds"])
+            df = df.sort_values(["unique_id", "ds"])
+    with st.sidebar:
+        seasonality = st.number_input("Seasonality of your data:", value=1)
+        top_k = st.number_input('Number of closest series:', value=12)
+    
+    y_vector_feats = tsfeatures_vector(df.tail(100), seasonality)
+    closest_ids = get_closest_ids(y_vector_feats, top_k, INDEX)
+    st.header('Closest match from TimeNet')
+    st.write(
+        """
+        This side-by-side plot visualizes your uploaded time series (left) and its closest match from the TimeNet dataset (right). 
+
+        By comparing these two plots, you can see how your data's behavior aligns with the most similar series in the TimeNet dataset. Similarities in trends, cycles, or patterns can indicate shared underlying structures or influences between your series and the TimeNet series.
+        """
+
+    )
+    st.pyplot(
+        plot_closest_series(df, closest_ids[0]['id'], CATALOGUE)
+    )
+    st.header('Potential winner models')
+    st.write(
+        """
+        This plot showcases the "win rate" of various predictive models. 
+        Each model's win rate is based on how frequently it outperforms others when used to forecast the closest series to your own data.
+
+        This visualization allows you to compare the effectiveness of different models and identify which ones are more likely to provide accurate forecasts for your data.
+        """
+    )
+    st.pyplot(
+        plot_best_models_count(closest_ids, CATALOGUE)
+    )
+
+if __name__ == "__main__":
+    st_timenet_features()

requirements.txt

@@ -0,0 +1,13 @@
+fire
+numpy
+pandas
+pinecone-client
+plotly
+pyarrow
+python-dotenv
+s3fs
+seaborn
+statsforecast
+streamlit
+streamlit-aggrid
+tsfeatures

src/utils.py

@@ -0,0 +1,110 @@
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+
+from tsfeatures import (
+    tsfeatures, acf_features, arch_stat, crossing_points,
+    entropy, flat_spots, heterogeneity, holt_parameters,
+    lumpiness, nonlinearity, pacf_features, stl_features,
+    stability, hw_parameters, unitroot_kpss, unitroot_pp,
+    series_length, sparsity, hurst, statistics
+)
+
+
+FILE_CATALOGUE = os.environ['FILE_CATALOGUE']
+BUCKET_TIMENET = os.environ['BUCKET_TIMENET']
+KEY_TIMENET = os.environ['KEY_TIMENET']
+
+
+FEATS_COLS = ['hurst', 'series_length', 'unitroot_pp', 'unitroot_kpss', 'hw_alpha',
+       'hw_beta', 'hw_gamma', 'stability', 'nperiods', 'seasonal_period',
+       'trend_strength', 'spike', 'linearity', 'curvature', 'e_acf1',
+       'e_acf10', 'seasonal_strength', 'peak', 'trough', 'x_pacf5',
+       'diff1x_pacf5', 'diff2x_pacf5', 'seas_pacf', 'nonlinearity',
+       'lumpiness', 'alpha', 'beta', 'flat_spots', 'entropy',
+       'crossing_points', 'arch_lm', 'x_acf1', 'x_acf10', 'diff1_acf1',
+       'diff1_acf10', 'diff2_acf1', 'diff2_acf10', 'seas_acf1', 'sparsity',
+       'total_sum', 'mean', 'variance', 'median', 'p2point5', 'p5', 'p25',
+       'p75', 'p95', 'p97point5', 'max', 'min']
+
+def tsfeatures_vector(df:pd.DataFrame, seasonality: int) -> pd.DataFrame:
+    ts_df = tsfeatures(
+        ts=df[['unique_id', 'ds', 'y']],
+        freq=seasonality, 
+        features=[sparsity, acf_features, crossing_points,
+                  entropy, flat_spots, holt_parameters,
+                  lumpiness, nonlinearity, pacf_features, stl_features,
+                  stability, hw_parameters, unitroot_kpss, unitroot_pp,
+                  series_length, hurst, arch_stat, statistics], 
+        scale=False,
+    ).rename(columns={'trend': 'trend_strength'})
+    if seasonality == 1:
+        # add missing features when seasonality != 1
+        ts_df[['seasonal_strength', 'peak', 'trough', 'seas_pacf', 'seas_acf1']] = np.nan
+    ts_df[['trend_strength', 'seasonal_strength']] = ts_df[['trend_strength', 'seasonal_strength']].fillna(0)
+    vector = ts_df[FEATS_COLS].fillna(0).iloc[0].values
+    vector = (vector - vector.min()) / (vector.max() - vector.min())
+    return vector.tolist()
+
+def get_closest_ids(x: list, top_k: int, index_pinecone):
+    query_response = index_pinecone.query(
+        top_k=top_k,
+        include_values=False,
+        include_metadata=True,
+        vector=x,
+    )
+    return query_response['matches']
+
+def plot_best_models_count(ids, catalogue):
+    uids = [x['id'] for x in ids]
+    file_evaluations = catalogue['file_evaluation'].loc[uids].unique()
+    eval_df = [pd.read_parquet(f_eval) for f_eval in file_evaluations]
+    eval_df = pd.concat(eval_df).query('unique_id in @uids')
+    eval_df = pd.pivot(
+        eval_df,
+        index=['unique_id', 'metric'],
+        columns='model', 
+        values='value'
+    ).reset_index()
+    models = eval_df.drop(columns=['unique_id', 'metric']).columns
+    eval_df['BestModel'] = eval_df[models].idxmin(axis=1)
+    #eval_df = eval_df.groupby(['BestModel', 'metric']).size().rename('n').reset_index()
+    fig = sns.catplot(eval_df.query('metric != "mase"'), y='BestModel', kind='count', col='metric')
+    return fig
+
+def plot_closest_series(Y_df, id, catalogue):
+    # leer archivo de file_timenet y hacer el plot
+    uid_catalogue = catalogue.loc[id] 
+    closest_df = pd.read_parquet(uid_catalogue.file_timenet).query('unique_id == @id')
+    #Y_df['unique_id'] = 'ProvidedByUser'
+    
+    # Create a figure with 1 row and 2 columns
+    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(15,5))
+    
+    # Get the unique_id for each DataFrame
+    unique_id_Y_df = Y_df['unique_id'].unique()[0]
+    unique_id_closest_df = closest_df['unique_id'].unique()[0]
+    
+    # Plot the 'y' column for both dataframes, against 'ds', and label them with unique_id
+    sns.lineplot(x='ds', y='y', ax=axes[0], data=Y_df, label=unique_id_Y_df)
+    sns.lineplot(x='ds', y='y', ax=axes[1], data=closest_df)
+    
+    # Set the titles for the subplots
+    axes[0].set_title('Uploaded Dataset')
+    axes[1].set_title(f'TimenetTimeSeries:{uid_catalogue.dataset},{uid_catalogue.subdataset},{uid_catalogue.ts_name}')
+    
+    # Show legend on each subplot
+    axes[0].legend()
+    axes[1].legend()
+    
+    # Display the plot
+    plt.tight_layout()
+    plt.show()
+    return fig
+
+def get_catalogue():
+    return pd.read_parquet(FILE_CATALOGUE)
+