<jupyter_start><jupyter_text>Determine period size for benchmark datasetsThis notebook iterates over all datasets in our benchmark dataset collection and determines the dominant period size using [`forecast.findfrequency`](https://rdrr.io/cran/forecast/man/findfrequency.html) from R.<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import json
import rpy2

from pathlib import Path

plt.rcParams["figure.figsize"] = (20, 8)<jupyter_output><empty_output><jupyter_text>Prepare notebooks for R usage<jupyter_code>import os
os.environ["R_HOME"] = "/home/sebastian.schmidl/.conda/envs/python-r/lib/R"<jupyter_output><empty_output><jupyter_text>Load R libraries (install `forecast` if not present)<jupyter_code>import rpy2.robjects as robjects
from rpy2.robjects.packages import importr
try:
    from rpy2.rinterface import RRuntimeError
except ImportError:
    from rpy2.rinterface.embedded import RRuntimeError

try:
    forecast = importr("forecast")
except ImportError:
    utils = importr("utils")
    utils.chooseCRANmirror(ind=1)
    utils.install_packages(robjects.StrVector(["forecast"]))
    forecast = importr("forecast")<jupyter_output>/home/sebastian.schmidl/.conda/envs/python-r/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: During startup - 
  warnings.warn(x, RRuntimeWarning)
/home/sebastian.schmidl/.conda/envs/python-r/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Warning messages:

  warnings.warn(x, RRuntimeWarning)
/home/sebastian.schmidl/.conda/envs/python-r/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: 1: Setting LC_TIME failed, using "C" 

  warnings.warn(x, RRuntimeWarning)
/home/sebastian.schmidl/.conda/envs/python-r/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: 2: Setting LC_MONETARY failed, using "C" 

  warnings.warn(x, RRuntimeWarning)
/home/sebastian.schmidl/.conda/envs/python-r/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: 3: Setting LC_PAPER failed, using "C" 

  warnings.warn(x, RRuntimeWarning)
/home/sebastian.schmidl/.conda/envs/python-r/l[...]<jupyter_text>Iterate over datasets and determine their periodLoad dataset overview file<jupyter_code>root_path = Path("/home/projects/akita/data/benchmark-data/data-processed")
df_datasets = pd.read_csv(root_path / "datasets.csv")
df_datasets<jupyter_output><empty_output><jupyter_text>Perform period size extraction and store it in the metadata files and the dataset overview<jupyter_code>def find_frequency(values):
    try:
        x = robjects.FloatVector(values.tolist())
        return forecast.findfrequency(x)[0]
    except RRuntimeError as ex:
        if "zero-variance series" in str(ex) or "'order.max' must be >= 1" in str(ex) or "Error in lm.fit" in str(ex):
            return 1
        else:
            raise ex

def get_metadata_filename(dataset_path):
    filename_base = ".".join(dataset_path.stem.split(".")[:-1])
    return dataset_path.parent / f"{filename_base}.metadata.json"

def find_periods(dataset_path):
    df = pd.read_csv(dataset_path, parse_dates=True, infer_datetime_format=True)
    periods = []
    features = set(df.columns) - {"timestamp", "is_anomaly"}
    for f in features:
        period = find_frequency(df[f])
        periods.append(period)
    periods = [int(p) for p in periods]
    return list(zip(features, periods))

def aggregate_periods(periods):
    periods = np.array([p for (f, p) in periods])
    hist = np.bincount(periods)
    mode = hist.argmax()
    if hist[mode] < 2:
        mode = np.median(periods)
    return int(mode)
def process_dataset(dataset_path, metas, is_train = False):
    periods = find_periods(dataset_path)
    print(f"  ... {'train' if is_train else 'test'} dataset: {[p for (f, p) in periods]}")
    for meta in metas:
        if meta["is_train"] == is_train:
            meta["periods"] = dict(periods)
    if is_train == False:
        return aggregate_periods(periods)
    else:
        return None

current_collection = None
selected_collection = "NormA"
for i, row in df_datasets.iterrows():
    if selected_collection is not None and row["collection_name"] != selected_collection:
        continue
    if not np.isnan(row["period_size"]):
        print(f"Skipping {row['collection_name']}/{row['dataset_name']}, because it already has a period ({row['period_size']})")
        continue
    if row["collection_name"] == "Kitsune":
        print(f"Skipping {row['collection_name']}/{row['dataset_name']}, because it would take too long!")
        continue
    
    if current_collection != row["collection_name"]:
        # store dataset overview
        print("### Saving dataset overview!")
        df_datasets.to_csv(root_path / "datasets.csv", index=False)
    current_collection = row["collection_name"]

    print(f"Processing {row['collection_name']}/{row['dataset_name']} ...")
    dataset_path = root_path / row["test_path"]
    # load metadata
    meta_filename = get_metadata_filename(dataset_path)
    try:
        with meta_filename.open("r") as fh:
            metas = json.load(fh)
        skip_meta = False
    except FileNotFoundError:
        print("  ... skipping metadata files!")
        skip_meta = True
        metas = []
        
    # for test dataset
    period = process_dataset(dataset_path, metas, is_train=False)
    
    # for train dataset (if exist)
    if row["train_type"] != "unsupervised" and not skip_meta:
        dataset_path = root_path / row["train_path"]
        process_dataset(dataset_path, metas, is_train=True)
    
    print(f"  ... period = {period}")
    
    # store metadata
    if not skip_meta:
        print("  ... writing metadata file")
        with meta_filename.open("w") as fh:
            json.dump(metas, fh, indent=2, sort_keys=True)

    # adapt dataset overview
    df_datasets.loc[i, "period_size"] = period
    print("finished.")

# store dataset overview
print("\nSaving dataset overview!")
df_datasets.to_csv(root_path / "datasets.csv", index=False)<jupyter_output>### Saving dataset overview!
Processing NormA/Discords_annsgun ...
  ... test dataset: [111]
  ... period = 111
  ... writing metadata file
finished.
Processing NormA/Discords_dutch_power_demand ...
  ... test dataset: [111]
  ... period = 111
  ... writing metadata file
finished.
Processing NormA/Discords_marotta_valve_tek_14 ...
  ... test dataset: [1]
  ... period = 1
  ... writing metadata file
finished.
Processing NormA/Discords_marotta_valve_tek_16 ...
  ... test dataset: [1]
  ... period = 1
  ... writing metadata file
finished.
Processing NormA/Discords_marotta_valve_tek_17 ...
  ... test dataset: [1]
  ... period = 1
  ... writing metadata file
finished.
Processing NormA/Discords_patient_respiration1 ...
  ... test dataset: [22]
  ... period = 22
  ... writing metadata file
finished.
Processing NormA/Discords_patient_respiration2 ...
  ... test dataset: [40]
  ... period = 40
  ... writing metadata file
finished.
Processing NormA/SinusRW_Length_104000_AnomalyL_200_AnomalyN_20_[...]<jupyter_text>Replace periods of size `1` with `NaN` (because then our fallback values come into play).<jupyter_code>root_path = Path("/home/projects/akita/data/benchmark-data/data-processed")
df_datasets = pd.read_csv(root_path / "datasets.csv")

indexer = df_datasets["period_size"] == 1
df_datasets.loc[indexer, "period_size"] = np.nan

df_datasets.to_csv(root_path / "datasets.csv", index=False)
df_datasets[df_datasets["collection_name"] == "NormA"]<jupyter_output><empty_output><jupyter_text>Experimentation<jupyter_code>dataset_path = root_path / df_datasets.loc[1, "test_path"]
df = pd.read_csv(dataset_path, parse_dates=True, infer_datetime_format=True)
df
periods = find_periods(dataset_path)
period = aggregate_periods(periods)
df.plot()
indices = np.arange(0, df.shape[0], period)
plt.vlines(indices, df.min(axis=1).min()-5, df.max(axis=1).max()+5, color="black")
plt.show()
meta_filename = get_metadata_filename(root_path / df_datasets.loc[1, "test_path"])
with meta_filename.open("r") as fh:
    metas = json.load(fh)
metas
for meta in metas:
    if meta["is_train"] == False:
        meta["periods"] = dict(periods)
metas
periods = np.array(periods)
print(f"min={periods.min()},median={np.median(periods)},max={periods.max()},lcm={np.lcm.reduce(periods)},gcd={np.gcd.reduce(periods)},mode={np.bincount(periods).argmax()}")
aggregate_periods(periods)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13067513526229121, 'fraction_numerical': 0.012114796518466243, 'mean_word_length': 4.156458459672529, 'pattern_counts': {'":': 1, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '533088', 'license': 'MIT', 'n_tokens_mistral': 2928, 'n_tokens_neox': 2845, 'n_words': 588, 'path': 'notebooks/data-prep/Dataset Period Size Estimation.ipynb', 'repo_name': 'HPI-Information-Systems/TimeEval', 'size': '8502'}