<jupyter_start><jupyter_text>Text2Time Tutorial<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import shutil
import sys
import os<jupyter_output><empty_output><jupyter_text>Set sytem paths<jupyter_code>T2time_path = os.path.join('..', 'pvops', 'text2time')
timeseries_path = os.path.join('..', 'pvops', 'timeseries')
sys.path.append('..')
sys.path.append(T2time_path)
sys.path.append(timeseries_path)<jupyter_output><empty_output><jupyter_text>Import modules<jupyter_code>from pvops.text2time import visualize, utils, preprocess
from pvops.timeseries.models import linear, iec<jupyter_output><empty_output><jupyter_text>Define csv paths<jupyter_code>example_OMpath = os.path.join('example_data', 'example_om_data2.csv')
example_prodpath = os.path.join('example_data', 'example_prod_data_cumE2.csv')
example_metapath = os.path.join('example_data', 'example_metadata2.csv')<jupyter_output><empty_output><jupyter_text>Read data<jupyter_code>prod_data = pd.read_csv(example_prodpath, error_bad_lines=False, engine='python')
om_data = pd.read_csv(example_OMpath, error_bad_lines=False, engine='python')
metadata = pd.read_csv(example_metapath, error_bad_lines=False, engine='python')<jupyter_output><empty_output><jupyter_text>Explore production data<jupyter_code>prod_data<jupyter_output><empty_output><jupyter_text>Explore OM data<jupyter_code>om_data<jupyter_output><empty_output><jupyter_text>Explore metadata<jupyter_code>metadata<jupyter_output><empty_output><jupyter_text>2. Prepare data for analysisAssigning dictionaries to connect pvOps variables with User's column names<jupyter_code>#Format for dictionaries is {pvops variable: user-specific column names}
prod_col_dict = {'siteid': 'randid', 
                 'timestamp': 'Date', 
                 'energyprod': 'Energy',
                 'irradiance':'Irradiance',
                 'baseline': 'IEC_pstep', #user's name choice for new column (baseline expected energy defined by user or calculated based on IEC)
                 'dcsize': 'dcsize', #user's name choice for new column (System DC-size, extracted from meta-data)
                 'compared': 'Compared',#user's name choice for new column
                 'energy_pstep': 'Energy_pstep'} #user's name choice for new column

om_col_dict = {'siteid': 'randid', 
               'datestart': 'date_start',
               'dateend': 'date_end',
               'workID': 'WONumber',
               'worktype': 'WOType',
               'asset': 'Asset',
               'eventdur': 'EventDur', #user's name choice for new column (Repair Duration)
               'modatestart': 'MonthStart', #user's name choice for new column (Month when an event begins)
               'agedatestart': 'AgeStart'} #user's name choice for new column (Age of system when event begins)

metad_col_dict = {'siteid': 'randid',
                  'dcsize': 'DC_Size_kW',
                  'COD': 'COD'}<jupyter_output><empty_output><jupyter_text>2.1 Convert date strings to date-time objects  O&M Data<jupyter_code>#Note:  NaNs are converted to NaTs
om_data_converted = preprocess.om_date_convert(om_data, om_col_dict)

print('----------Original data types')
print(om_data.dtypes)
print('\n----------Post-processed data types')
print(om_data_converted.dtypes)
print('\n----------Converted data frame')
om_data_converted<jupyter_output>----------Original data types
randid         object
Asset          object
date_start     object
date_end       object
WONumber        int64
WOType         object
GeneralDesc    object
dtype: object

----------Post-processed data types
randid                 object
Asset                  object
date_start     datetime64[ns]
date_end       datetime64[ns]
WONumber                int64
WOType                 object
GeneralDesc            object
dtype: object

----------Converted data frame<jupyter_text>Production data<jupyter_code>prod_data_converted = preprocess.prod_date_convert(prod_data, prod_col_dict)

print('Pre-processing')
print(prod_data.dtypes)
print('\nPost-processed')
print(prod_data_converted.dtypes)
prod_data_converted<jupyter_output>Pre-processing
randid         object
Date           object
Energy        float64
Irradiance    float64
dtype: object

Post-processed
randid                object
Date          datetime64[ns]
Energy               float64
Irradiance           float64
dtype: object<jupyter_text>3. Handling issues with data quality  3.1 Missing site-IDsDrop rows where site-ID is NAN in OM-data (helpful when multiple sites are in O&M data frame and a NAN doesn't identify a specific site)<jupyter_code>om_data_sitena, addressed = preprocess.data_site_na(om_data_converted, om_col_dict)
addressed #printing row that was addressed<jupyter_output><empty_output><jupyter_text>Print post-processed data frame<jupyter_code>om_data_sitena<jupyter_output><empty_output><jupyter_text>3.2 O&M Start-dates that occur after concluding date (inverted dates) Addressing issue by swapping dates<jupyter_code>om_data_checked_s, addressed = preprocess.om_datelogic_check(om_data_sitena, om_col_dict, 'swap')
addressed<jupyter_output><empty_output><jupyter_text>Print post-processed data frame<jupyter_code>om_data_checked_s<jupyter_output><empty_output><jupyter_text>Addressing issue by dropping rows instead of swapping dates<jupyter_code>om_data_checked_d, date_error = preprocess.om_datelogic_check(om_data_sitena, om_col_dict, 'drop')
om_data_checked_d #post-processed data frame<jupyter_output><empty_output><jupyter_text>3.3 Unexpected drops in energy delivered (when collected on cumulative basis) Quick visualization pre-processed data<jupyter_code>plotvar = 'Energy'
for sid in prod_data_converted.loc[:, 'randid'].unique():
    mask = prod_data_converted.loc[:, 'randid']== sid
    prod_data_converted.loc[mask].head(n=15)
    prod_data_converted.loc[mask].plot(x='Date',y=plotvar, title=sid)
    plt.ylabel('Joules')<jupyter_output><empty_output><jupyter_text>Addressing issue by forward-filling<jupyter_code>prod_data_anom, addressed = utils.prod_anomalies(prod_data_converted, prod_col_dict, 1.0, np.nan, ffill=True)
addressed<jupyter_output><empty_output><jupyter_text>Print post-processed data frame<jupyter_code>#post-processed data frame
prod_data_anom<jupyter_output><empty_output><jupyter_text>Quick visualization of post-processed data<jupyter_code>plotvar = 'Energy'
for sid in prod_data_anom.loc[:, 'randid'].unique():
    mask = prod_data_anom.loc[:, 'randid']== sid
    prod_data_anom.loc[mask].head(n=15)
    prod_data_anom.loc[mask].plot(x='Date',y=plotvar, title=sid)
    plt.ylabel('Joules')<jupyter_output><empty_output><jupyter_text>3.4 Missing time-stamps in data  Production DataDropping rows with missing time-stamps<jupyter_code>prod_data_datena_d, addressed = preprocess.prod_nadate_process(prod_data_anom, prod_col_dict, pnadrop=True)
addressed<jupyter_output><empty_output><jupyter_text>Print post-processed data frame<jupyter_code>prod_data_datena_d<jupyter_output><empty_output><jupyter_text>Identifying rows with missing time-stamps but not dropping them<jupyter_code>prod_data_datena_id, addressed = preprocess.prod_nadate_process(prod_data_anom, prod_col_dict, pnadrop=False)
prod_data_datena_id<jupyter_output><empty_output><jupyter_text>O&M Data Dropping rows when end-date of an event is missing<jupyter_code>om_data_datena_d, addressed = preprocess.om_nadate_process(om_data_checked_s, om_col_dict, om_dendflag='drop')
addressed<jupyter_output><empty_output><jupyter_text>Print post-processed data frame<jupyter_code>om_data_datena_d<jupyter_output><empty_output><jupyter_text>Rather than dropping rows, assigning "today's" time-stamp for missing end-dates to consider an open ticket<jupyter_code>om_data_datena_t, addressed = preprocess.om_nadate_process(om_data_checked_s, om_col_dict, om_dendflag='today')
om_data_datena_t<jupyter_output><empty_output><jupyter_text>4. Pre-visualizing preparation 4.1 Print out overview of data frames  Production<jupyter_code>prod_summary, om_summary = utils.summarize_overlaps(prod_data_datena_d, om_data_datena_t, prod_col_dict, om_col_dict)
prod_summary<jupyter_output><empty_output><jupyter_text>O&M<jupyter_code>om_summary<jupyter_output><empty_output><jupyter_text>4.2 Extract overlapping data<jupyter_code>prod_data_clean, om_data_clean = utils.overlapping_data(prod_data_datena_d, om_data_datena_d, prod_col_dict, om_col_dict)<jupyter_output><empty_output><jupyter_text>Print post-processed production data frame<jupyter_code>prod_data_clean<jupyter_output><empty_output><jupyter_text>Print post-processed O&M data frame<jupyter_code>om_data_clean<jupyter_output><empty_output><jupyter_text>4.3 Calculate reference production data (IEC in this case)<jupyter_code>prod_data_clean_iec = iec.iec_calc(prod_data_clean, prod_col_dict, metadata, metad_col_dict, gi_ref=1000.)<jupyter_output><empty_output><jupyter_text>Print head of post-processed data frame<jupyter_code>prod_data_clean_iec.head(n=15)<jupyter_output><empty_output><jupyter_text>4.4 Calculating a comparison of production data relative to baseline Calculate as a ratio (actual/baseline)<jupyter_code>prod_data_quant = utils.prod_quant(prod_data_clean_iec, prod_col_dict, comp_type='norm', ecumu=True)
prod_data_quant #Quick look at post-processed data<jupyter_output><empty_output><jupyter_text>Calculate as a difference (baseline-actual)<jupyter_code>prod_data_quant = utils.prod_quant(prod_data_clean_iec, prod_col_dict, comp_type='diff', ecumu=True)
prod_data_quant #Quick look at post-processed data<jupyter_output><empty_output><jupyter_text>5. Visualizations 5.1 Visualizing overlapping production and O&M data Making directories to store generated visualizations.<jupyter_code>#User should modify paths as needed
main_fldr = 'analysis'
if os.path.isdir(main_fldr):
    shutil.rmtree(main_fldr)
prod_fldr = os.path.join(main_fldr, 'perf_plots')
site_fldr = os.path.join(main_fldr, 'site_plots')
os.makedirs(prod_fldr)
os.makedirs(site_fldr)<jupyter_output><empty_output><jupyter_text>Making visualizations<jupyter_code>figs = visualize.visualize_om_prod_overlap(prod_data_quant, om_data_clean, prod_col_dict, om_col_dict, prod_fldr=prod_fldr, e_cumu=True, be_cumu=False, samp_freq='H', pshift=0.0, baselineflag=True)<jupyter_output><empty_output><jupyter_text>Display figure handles of overlapping data ("figs" contains one figure per site in overlapping data frames)<jupyter_code>for i in range(len(figs)):
    figs[i].show()<jupyter_output><empty_output><jupyter_text>5.2 Calculate additional O&M metrics and generate relevant plots Calculating individual event duration and age of system at time of event occurence<jupyter_code>om_data_update = utils.om_summary_stats(om_data_clean, metadata, om_col_dict, metad_col_dict)
om_data_update #Quick look at post-processed data<jupyter_output><empty_output><jupyter_text>Setting Seaborn fig and font settings (inputs to count_fig and catscat_fig below)<jupyter_code>my_figsize = (12,10)
my_fontsize = 20
my_savedpi = 300
fig_sets = {'figsize': my_figsize, 
            'fontsize': my_fontsize 
            }<jupyter_output><empty_output><jupyter_text>Creating scatter-plot of system age at beginning of each event, per site<jupyter_code>cat_varx = om_col_dict['siteid']
cat_vary= om_col_dict['agedatestart']
sv_nm = 'system_age.png'
myfig = visualize.visualize_categorical_scatter(om_data_update, om_col_dict, cat_varx, cat_vary, fig_sets)
myfig.savefig(os.path.join(site_fldr,sv_nm), dpi=my_savedpi)<jupyter_output><empty_output><jupyter_text>Creating scatter-plot of each event-duration, per site<jupyter_code>cat_varx = om_col_dict['siteid']
cat_vary= om_col_dict['eventdur']
sv_nm = 'event_dur.png'
myfig = visualize.visualize_categorical_scatter(om_data_update, om_col_dict, cat_varx, cat_vary, fig_sets)
myfig.savefig(os.path.join(site_fldr,sv_nm), dpi=my_savedpi)<jupyter_output><empty_output><jupyter_text>Count-plot of  of events, per site<jupyter_code>count_var = om_col_dict['siteid']
sv_nm = 'event_count_per_site.png'
myfig = visualize.visualize_counts(om_data_update, om_col_dict, count_var, fig_sets)
myfig.savefig(os.path.join(site_fldr,sv_nm), dpi=my_savedpi)<jupyter_output><empty_output><jupyter_text>Count-plot of  of events, per month<jupyter_code>count_var = om_col_dict['modatestart']
sv_nm = 'event_count_per_month.png'
myfig = visualize.visualize_counts(om_data_update, om_col_dict, count_var, fig_sets)
myfig.savefig(os.path.join(site_fldr,sv_nm), dpi=my_savedpi)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10130327016870552, 'fraction_numerical': 0.005996641880546894, 'mean_word_length': 6.6783302639656235, 'pattern_counts': {'":': 0, '<': 175, '<?xml version=': 0, '>': 175, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '44', 'dir': 'jupyter-structured-clean-dedup', 'id': '295669', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 4521, 'n_tokens_neox': 4372, 'n_words': 787, 'path': 'examples/tutorial_text2time_module.ipynb', 'repo_name': 'bfemery-sandia/pvOps', 'size': '12507'}