<jupyter_start><jupyter_text>General definitions<jupyter_code># Get definitions
%run "00-definitions.ipynb"

MAX_GAP_TO_FILL = 4<jupyter_output><empty_output><jupyter_text>Clean raw data Fill in the gaps in the raw generation data by interpolating.<jupyter_code>def clean_data(category: str, technology: str = None):
    """
    Args:
        category: category of data, one of 'generation', 'load', etc.
        technology
    """
    
    entso_e_raw_filename = f"../data/raw/ENTSO-E_TP/ENTSO-E_TP_{category}{('_' + technology) if technology else ''}.csv"
    other_raw_filename_pattern = f"../data/raw/others/*_{category}{('_' + technology) if technology else ''}.csv"
    intermediate_filename = f"../data/intermediate/{category.capitalize()}{(' ' + technology) if technology else ''}.csv"
    statistics_filename = f"../data/intermediate/Statistics {category}{(' ' + technology) if technology else ''}.csv"
    
    # load ENTSO-E data
    print(f"Reading file {entso_e_raw_filename}...")
    raw_data = pd.read_csv(entso_e_raw_filename, 
                           index_col=0, parse_dates=True)
    
    # Load data from other sources and merge
    for other_src_path in glob(other_raw_filename_pattern):
        print(f"Reading file {other_src_path}...")
        other_source = pd.read_csv(other_src_path, 
                                   index_col=0, parse_dates=True)
        for col, ts in other_source.iteritems(): 
            raw_data[col] = ts

    stats = pd.DataFrame()
    cleaned_data = (
        raw_data.apply(
            expand_to_full_length, stats=stats, 
            start=START, end=END, tzone=TZ,
            max_gap=MAX_GAP_TO_FILL, min_timestep=MIN_TIMESTEP
        ).apply(remove_drops)
         .apply(remove_peaks)
    )
    
    stats['NRMSE of filter'] = rmse_of_filter(raw_data, cleaned_data) / raw_data.mean()

    # Write to disk
    cleaned_data.to_csv(intermediate_filename)
    print(f'Wrote {intermediate_filename}')
    stats.to_csv(statistics_filename)
    print(f'Wrote {statistics_filename}')
for tech in GEN_TYPES:
    clean_data('generation', tech)<jupyter_output>Reading file ../data/raw/ENTSO-E_TP/ENTSO-E_TP_generation_Solar.csv...
Reading file ../data/raw/others\REN_generation_Solar.csv...
Reading file ../data/raw/others\SVK_generation_Solar.csv...
Filling gaps for ES. . .
Filling gaps for FR. . .
Filling gaps for GB. . .
Filling gaps for IT. . .
Filling gaps for NL. . .
Filling gaps for PT. . .
Filling gaps for SE. . .<jupyter_text>Fill in gaps in the raw load data by interpolating.<jupyter_code>clean_data('load')
clean_data('load_forecast')
for tech in GEN_TYPES:
    clean_data('forecast', tech)<jupyter_output>Reading file ../data/raw/ENTSO-E_TP/ENTSO-E_TP_forecast_Solar.csv...
Filling gaps for DE. . .
Filling gaps for FR. . .
Filling gaps for NL. . .
Filling gaps for NO. . .
Filling gaps for PT. . .
Wrote ../data/intermediate/Forecast Solar.csv
Wrote ../data/intermediate/Statistics forecast Solar.csv
Reading file ../data/raw/ENTSO-E_TP/ENTSO-E_TP_forecast_Wind Onshore.csv...
Filling gaps for DE. . .
Filling gaps for FI. . .
Filling gaps for FR. . .
Filling gaps for GB-NIR. . .
Filling gaps for IE. . .
Filling gaps for NL. . .
Filling gaps for NO. . .
Filling gaps for PT. . .
Filling gaps for SE. . .
Wrote ../data/intermediate/Forecast Wind Onshore.csv
Wrote ../data/intermediate/Statistics forecast Wind Onshore.csv
Reading file ../data/raw/ENTSO-E_TP/ENTSO-E_TP_forecast_Wind Offshore.csv...
Filling gaps for NL. . .
Filling gaps for NO. . .
Wrote ../data/intermediate/Forecast Wind Offshore.csv
Wrote ../data/intermediate/Statistics forecast Wind Offshore.csv
{'fraction_non_alphanumeric': 0.12166987091458391, 'fraction_numerical': 0.0013732491073880802, 'mean_word_length': 3.1480637813211847, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '301422', 'license': 'MIT', 'n_tokens_mistral': 1264, 'n_tokens_neox': 1226, 'n_words': 287, 'path': 'notebooks/02a-clean-data.ipynb', 'repo_name': 'vttresearch/windtask25-timeseries', 'size': '3641'}