<jupyter_start><jupyter_text>Browse one file<jupyter_code>from pathlib import Path

import numpy as np
import pandas as pd

from dsgrid.dataformat.datafile import Datafile
from dsgrid.dataformat.datatable import Datatable

from ntbkhelp import OptionPresenter, show_enum, show_elements_with_data<jupyter_output><empty_output><jupyter_text>Choose data locationReview the choices in the first cell, select the path you want to use in the second cell, and run both cells.<jupyter_code># Publicly available
dsgrid_oedi_base_path = "s3://oedi-data-lake/dsgrid-2018-efs" # Uses h5py
dsgrid_hsds_base_path = "/nrel/dsgrid-2018-efs"               # Uses h5pyd
# Internal to NREL
dsgrid_nrel_base_path_windows = Path("//nrelnas01/PLEXOS/Projects/Load/dsgrid_v0.2.0/")
dsgrid_nrel_base_path_mac = Path("/Volumes/PLEXOS/Projects/Load/dsgrid_v0.2.0/")
dsgrid_base_path = dsgrid_oedi_base_path

is_hsds = str(dsgrid_base_path).startswith("/nrel/")
is_s3 = str(dsgrid_base_path).startswith("s3://")<jupyter_output><empty_output><jupyter_text>Choose data setSelect whether you want to look at county-level or state-level data in the second cell. Run both cells.<jupyter_code>if is_hsds or is_s3:
    dsgrid_county_dataset_path = f"{dsgrid_base_path}/raw_complete/"
    dsgrid_state_dataset_path = f"{dsgrid_base_path}/state_hourly_residuals/"
    if is_hsds:
        import h5pyd
    if is_s3:
        import s3fs
        s3 = s3fs.S3FileSystem(anon=True)
else:
    dsgrid_county_dataset_path = dsgrid_base_path / "products" / "raw_complete"
    dsgrid_state_dataset_path = dsgrid_base_path / "products" / "state_hourly_residuals"
dsgrid_dataset_path = dsgrid_state_dataset_path

if is_hsds:
    print("Will read datafiles with HSDS")
if is_s3:
    print("Will read datafiles with s3")<jupyter_output><empty_output><jupyter_text>Specify a local directory for any outputsReview and edit the specified path. (The default option should be available for most everyone, but feel to change as you see fit.) Then run the cell.<jupyter_code>output_dir = Path.home() / "Documents" / "dsgrid-legacy-efs"

if not output_dir.exists():
    input_str = input(f"{output_dir!r} does not exist. Would you like to create it? [Y/n] ")
    if input_str[0].lower() == "y":
        output_dir.mkdir()
        print(f"Created {output_dir!r}.")<jupyter_output><empty_output><jupyter_text>Choose a file<jupyter_code>if is_hsds:
    with h5pyd.Folder(dsgrid_dataset_path) as d:
        files = list(d)
elif is_s3:
    files = [p.split("/")[-1] for p in s3.glob(f"{dsgrid_dataset_path}/*.dsg")]
else:
    files = [p.name for p in dsgrid_dataset_path.glob("*.dsg")]
options = OptionPresenter(files)
options.present_options()
input_str = input("Which file would you like to examine (input number)? ")
if is_hsds:
    datafile = Datafile.load(f"{dsgrid_dataset_path}{options.get_option(input_str)}")
elif is_s3:
    datafile = Datafile.load(f"{dsgrid_dataset_path}{options.get_option(input_str)}")
else:
    datafile = Datafile.load(dsgrid_dataset_path / options.get_option(input_str))<jupyter_output><empty_output><jupyter_text>Examine Basic Information<jupyter_code>show_enum(datafile.sector_enum)
show_elements_with_data(datafile.sector_enum, list(datafile.keys()))
show_enum(datafile.geo_enum)<jupyter_output><empty_output><jupyter_text>⚠️ **WARNING** ⚠️ The next cell may take a few minutes to run if the geography is counties<jupyter_code>geo_ids = []
for _id in datafile.geo_enum.ids:
    for sector_id in datafile:
        if datafile[sector_id].has_data(_id):
            geo_ids.append(_id)
            break
show_elements_with_data(datafile.geo_enum, geo_ids)
show_enum(datafile.enduse_enum)
for sector_id in datafile:
    df, _geo_ids, _scaling_factors = datafile[sector_id].get_data(0)
    break
show_elements_with_data(datafile.enduse_enum, list(df.columns))
show_enum(datafile.time_enum)<jupyter_output><empty_output><jupyter_text>Get Total Energy per Sector and Enduse with Datatable⚠️ **WARNING** ⚠️ This cell may be slow and/or run out of memory for large files / county level data (e.g., commercial.dsg, residential.dsg).<jupyter_code>data_series = Datatable(datafile).data
total_energy = data_series.reset_index().pivot_table(values=0, index="sector", columns="enduse", aggfunc=np.sum)
total_energy<jupyter_output><empty_output><jupyter_text>Get Total Energy per Sector and Enduse by IteratingThis section runs for all files, but may take some time.<jupyter_code>total_energy = []; index = []
for sector_id in datafile:
    sector_dataset = datafile[sector_id]
    sector_energy = None
    for i in range(sector_dataset.n_geos):
        df, geos, scaling_factors = sector_dataset.get_data(i)
        tmp = df.sum()
        for factor in scaling_factors:
            if sector_energy is None:
                sector_energy = (tmp * factor)
            else:
                sector_energy += (tmp * factor)
    if sector_energy is not None:
        total_energy.append(sector_energy.to_frame().T)
        index.append(sector_id)
total_energy = pd.concat(total_energy)
total_energy.index = index
total_energy
total_energy.sum().sum() / 1.0E6<jupyter_output><empty_output><jupyter_text>Get Total Energy per Sector and Enduse by Aggregating and Converting UnitsReview user input in the first cell before running this section.As with the previous section, running this code may take some time.<jupyter_code># Applying a map requires making a new HDF5 file on disk
overwrite = False     # Choices: True, False
energy_units = "TWh"  # Choices: kWh, MWh, GWh, TWh
from dsgrid.dataformat.dimmap import FilterToSingleFuelMap, FullAggregationMap, UnitConversionMap
from dsgrid.dataformat.enumeration import MultiFuelEndUseEnumeration, allsectors, conus, allenduses, annual

mappings = [
    ('-annual', FullAggregationMap(datafile.time_enum, annual)),
    ('-conus', FullAggregationMap(datafile.geo_enum, conus)),
    ('-allsectors', FullAggregationMap(datafile.sector_enum, allsectors)),    
]

# handle multi-fuel enduse enums
enduse_enum = datafile.enduse_enum
if isinstance(enduse_enum, MultiFuelEndUseEnumeration):
    mappings.append(f'-electricity', FilterToSingleFuelMap(enduse_enum, 'Electricity'))
    enduse_enum = mappings[-1][1].to_enum

# handle different electricity units
allenduses_units = allenduses.units(allenduses.ids[0])
if not (enduse_enum.units(enduse_enum.ids[0]) == allenduses_units):
    mappings.append(f'-{allenduses_units}', UnitConversionMap(
        enduse_enum, 
        [enduse_enum.units(enduse_enum.ids[0])], 
        [allenduses_units]))
    enduse_enum = mappings[-1][1].to_enum

mappings.extend([
    ('-allenduses', FullAggregationMap(enduse_enum, allenduses)),
    (f'-{energy_units}', UnitConversionMap(allenduses, [allenduses.units(allenduses.ids[0])], [energy_units]))
])

new_datafile = datafile
for postfix, mapping in mappings:
    p = Path(new_datafile.filepath)
    new_path = output_dir / (p.stem + postfix + p.suffix)
    if new_path.exists() and overwrite:
        new_path.unlink()
    if new_path.exists():
        new_datafile = Datafile.load(new_path)
        continue
    print(f"Applying {mapping} and writing to {new_path}\n")
    new_datafile = new_datafile.map_dimension(new_path, mapping)
    
total_energy = Datatable(new_datafile).data.sum()
print(f"{p.name} describes {total_energy} {energy_units} of electricity")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10485621269669018, 'fraction_numerical': 0.006782419967444384, 'mean_word_length': 5.058340180772391, 'pattern_counts': {'":': 1, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '435865', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2580, 'n_tokens_neox': 2530, 'n_words': 550, 'path': 'notebooks/Browse one file.ipynb', 'repo_name': 'mhengg/DSgrid', 'size': '7372'}