<jupyter_start><jupyter_text>Months<jupyter_code>MONTHS = '' 

def months(df):
    global MONTHS
    MIN_DATE = df['Date'].min()
    MAX_DATE = df['Date'].max()
    MONTHS = str(int((MAX_DATE - MIN_DATE).days / 30))
     
    print('Min Date: ', MIN_DATE)
    print('Max Date: ', MAX_DATE)
    print('Months:   ', MONTHS)
months(df)<jupyter_output><empty_output><jupyter_text>Create TS Settings<jupyter_code>DATE      = """ Enter Code """
TARGET    = """ Enter Code """

PROJECT_NAME = 'Lab_2'

VERSION = '1'
MODE    = 'Q'
    
FDWS = """ Enter Code """

FDS  = """ Enter Code """


BASE   = 'L2_Base_V:'

PREFIX = BASE + VERSION + '_Mnths:' + MONTHS + '_Mode:' + MODE
DATASET_FILENAME = 'Months_' + MONTHS
MAX_WAIT = 14400
READ_TIMEOUT = 14400

HOLDOUT_START_DATE  = None 
VALIDATION_DURATION = None 
HOLDOUT_DURATION    = None    
NUMBER_BACKTESTS    = None 
GAP_DURATION        = None 

FEATURE_SETTINGS = []

CAL_ID = None

print(FEATURE_SETTINGS)
print(CAL_ID)<jupyter_output><empty_output><jupyter_text>Create Project  Function to create projects<jupyter_code>def create_dr_project(df, project_name, fw_start=None, fw_end=None, fdw_start=None, fdw_end=None):
    
    ###############################
    # Create Datetime Specification
    ###############################
    # SERIES_COL = [SERIES]
    time_partition = dr.DatetimePartitioningSpecification(
        datetime_partition_column = DATE,
        forecast_window_start     = fw_start, 
        forecast_window_end       = fw_end,
        feature_derivation_window_start = fdw_start,
        feature_derivation_window_end   = fdw_end,
        holdout_start_date        = HOLDOUT_START_DATE ,
        validation_duration       = VALIDATION_DURATION,  
        holdout_duration          = HOLDOUT_DURATION,
        gap_duration              = GAP_DURATION,
        number_of_backtests       = NUMBER_BACKTESTS, 
        feature_settings          = FEATURE_SETTINGS,
        use_time_series           = True,
        calendar_id               = CAL_ID
      )
     

    ################
    # Create Project
    ################
    project = dr.Project.create(
        project_name = project_name, 
        sourcedata   = df, 
        max_wait     = MAX_WAIT, 
        read_timeout = READ_TIMEOUT,
        dataset_filename = DATASET_FILENAME
    )
    print("Post-Project MB: ", (df.memory_usage(index=True).sum()/1024/1024).round(2))
    print("Post-Project Records: {:,}".format(len(df)))
    print(f'Project {project_name} Created...')

    #################
    # Start Autopilot
    #################
    project.set_target(
        target = TARGET,   
        metric = None,      
        mode   = dr.AUTOPILOT_MODE.QUICK , # dr.AUTOPILOT_MODE.FULL_AUTO,
        #advanced_options = opts,
        worker_count = -1,
        partitioning_method = time_partition,
        max_wait = MAX_WAIT
    )
    return project<jupyter_output><empty_output><jupyter_text>Model Factory<jupyter_code>projects = []  # Keep List of all projects<jupyter_output><empty_output><jupyter_text>Function to iterate through various FDWs & FDs<jupyter_code>def model_factory(df, FDWS, FDS):
    PREFIX = BASE + str(VERSION) + '_Mnths:' + MONTHS + '_Mode:' + MODE
    
    for fdw in FDWS:
        for fd in FDS:
            fd_start  = fd[0] 
            fd_end    = fd[1]
            fdw_start = fdw[0]
            fdw_end   = fdw[1]

            # Name project
            project_name = f"{PREFIX}_FDW:{fdw_start}-{fdw_end}_FD:{fd_start}-{fd_end}"  
            print(project_name)

            data = df.copy() 

            # Create project
            project = create_dr_project(data, project_name, 
                                        fw_start=fd_start, fw_end=fd_end, 
                                        fdw_start=fdw_start, fdw_end=fdw_end)

            projects.append(project)<jupyter_output><empty_output><jupyter_text>Default Model<jupyter_code>model_factory(df, FDWS, FDS)<jupyter_output><empty_output><jupyter_text>Forecast Distance Breakout<jupyter_code># Update Version
VERSION = 2
FDS  = """ Enter Code """
model_factory(df, FDWS, FDS)<jupyter_output><empty_output><jupyter_text>Pull Results  Get Project Names in a List<jupyter_code>projects = dr.Project.list(search_params={'project_name': BASE}) 
projects<jupyter_output><empty_output><jupyter_text>Get Project Names and PIDs in a List<jupyter_code>lst = []

for p in projects:
    r = ((p, p.id))
    lst.append(r)
lst<jupyter_output><empty_output><jupyter_text>Unlock Holdouts<jupyter_code>for i in lst:
    project = Project.get(i[1])
    project.unlock_holdout()<jupyter_output><empty_output><jupyter_text>Compute Backtests for Blenders<jupyter_code>for i in lst:
    project = Project.get(i[1])
    lb = project.get_datetime_models()
    for model in lb:
        
        if 'Blender' in model.model_type:
            try:
                print(project.project_name, model)  
                dr.DatetimeModel.score_backtests(model) 
                print(f'Computing backtests for model {model.id} in Project {project.project_name}')
            except dr.errors.ClientError:
                pass
            print(f'All available backtests have been submitted for scoring for project {project.project_name}')
            print(' ')
        else:
            None<jupyter_output><empty_output><jupyter_text>Compute All Backtests for Top Models in Backtest 1 and Holdout groups<jupyter_code>OPTIMIZE_GROUP = ['validation', 'holdout']
PROJECT_METRIC = project.metric
METRICS = list(set([PROJECT_METRIC, 'MASE', 'RMSE']))
for p in lst :
    for met in METRICS:
        for o in OPTIMIZE_GROUP:
            project = Project.get(p[1])
            lb = project.get_datetime_models()

            best_models = sorted(
                                [model for model in lb if model.metrics[met][o]],  
                                key=lambda m: m.metrics[met][o],
                                )[0:3]
            
            for mod in best_models:

                if mod.metrics[met]["backtesting"] == None:
                    try:
                        print(project.project_name, mod)  
                        dr.DatetimeModel.score_backtests(mod) 
                        print(f'Computing backtests for model {mod.model_type} in Project {project.project_name}')
                    except dr.errors.ClientError:
                        pass
                    print(f'All available backtests have been submitted for scoring for project {project.project_name}')
                    print(' ')
                else:
                    print(project.project_name)
                    print(f'{mod.model_type} All Backtests Already Computed')
                    print(' ')<jupyter_output><empty_output><jupyter_text>Get Project and Model Scores<jupyter_code>OPTIMIZATION_PERIOD = 'backtesting'  # BackTest 1: validation  All Backtest: backtesting  Holdout: holdout<jupyter_output><empty_output><jupyter_text>Get scores in a DataFrame<jupyter_code>models = []
scores = pd.DataFrame()

for p in lst:
    project = Project.get(p[1])
    lb = project.get_datetime_models()
    best_model = sorted(
                        [model for model in lb if model.metrics[project.metric][OPTIMIZATION_PERIOD]],  
                        key=lambda m: m.metrics[project.metric][OPTIMIZATION_PERIOD],
                        )[0]

    backtest_scores = pd.DataFrame(
        [
            {
                'Project_Name': project.project_name,
                'Project_ID': project.id,
                'Model_ID': best_model.id,
                'Model_Type': best_model.model_type,
                'Featurelist': best_model.featurelist_name,
                'Optimization_Metric': project.metric,
                'Scores': best_model.metrics,
            }
        ]
    )
    scores = scores.append(backtest_scores, sort=False).reset_index(drop=True)  


print(f'Scores for all {len(projects)} projects have been computed')
print('')

scores = scores.join(json_normalize(scores["Scores"].tolist())).drop(labels=['Scores'], axis=1) 

# Drop Empty Columns
scores = scores[scores.columns.drop(list(scores.filter(regex='crossValidation$')))]

# Rename Columns
scores.columns = scores.columns.str.replace(".backtesting", "_All_BT")
scores.columns = scores.columns.str.replace(".holdout", "_Holdout")
scores.columns = scores.columns.str.replace(".validation", "_BT_1")
scores.columns = scores.columns.str.replace(' ', '_')

scores = scores[scores.columns.drop(list(scores.filter(regex='_All_BTScores$')))]

scores.head(2)<jupyter_output><empty_output><jupyter_text>Put column names into variables for easy reference<jupyter_code>METRICS = scores.filter(regex='MASE|RMSE').columns.to_list()
PROJECT = ['Project_Name', 'Project_ID', 'Model_ID', 'Model_Type', 'Featurelist']
COLS = PROJECT + METRICS
scores[COLS]<jupyter_output><empty_output><jupyter_text>Sort dataframe by score column<jupyter_code>scores['FDW_Start'] = scores['Project_Name'].str.extract(r'FDW:(-\d{1,2})')
scores['FDW_End']   = scores['Project_Name'].str.extract(r'FDW:-\d{1,2}-(\d{1,2})_')
scores['FD_Start']  = scores['Project_Name'].str.extract(r'FD:(\d{1,2})')
scores['FD_End']    = scores['Project_Name'].str.extract(r'FD:\d{1,2}-(\d{1,2})')
scores['Months']    = scores['Project_Name'].str.extract(r'_Mnths:(\d{1,2})_')

scores.rename(columns={'All_Backtests_Poisson Deviance':'All_Backtests_Poisson_Deviance', 
                       'Backtest_1_Poisson Deviance':'Backtest_1_Poisson_Deviance',
                       'Holdout_Poisson Deviance':'Holdout_Poisson_Deviance',
                       'Holdout_Tweedie Deviance':'Holdout_Tweedie_Deviance',
                       'All_Backtests_Tweedie Deviance':'All_Backtests_Tweedie_Deviance',
                       'Backtest_1_Tweedie Deviance':'Backtest_1_Tweedie_Deviance',
                       'Holdout_Tweedie Deviance':'Holdout_Tweedie_Deviance'}, inplace=True)


META = ['FDW_Start', 'FDW_End', 'FD_Start', 'FD_End', 'Months']
MORE = PROJECT + META + METRICS 
  
scores[MORE].sort_values(by=[""" Enter Code """], ascending=True)<jupyter_output><empty_output><jupyter_text>Get Best Model<jupyter_code>hrmse = scores.loc[scores[""" Enter Code """].notnull()]

# Take the Single Best model
# hrmse_best = pd.DataFrame(hrmse.loc[hrmse.MASE_All_BT.idxmin()]).transpose()

# Take the Best model by Project Name
hrmse_best = hrmse.loc[hrmse.groupby('Project_Name').MASE_All_BT.idxmin()]

best_models = pd.DataFrame(hrmse_best) 
best_models<jupyter_output><empty_output><jupyter_text>Generate Predictions  Select the project and model from the dataframe<jupyter_code>RECORD = 0

# Verify correct project and model
PID = best_models['Project_ID'].values[RECORD]
MID = best_models['Model_ID'].values[RECORD]

project = dr.Project.get(PID)
model   = dr.Model.get(PID, MID)
print(project, model)
PID = best_models['Project_ID'].values[RECORD]
MID = best_models['Model_ID'].values[RECORD]

project = dr.Project.get(PID)
model   = dr.Model.get(PID, MID)

dataset = project.upload_dataset(""" Enter Code """)

pred_job = model.request_predictions(dataset_id = dataset.id)

preds = pred_job.get_result_when_complete()

preds.head(5)<jupyter_output><empty_output><jupyter_text>Generate Predictions with 90% Prediction Interval<jupyter_code>PID = best_models['Project_ID'].values[RECORD]
MID = best_models['Model_ID'].values[RECORD]

project = dr.Project.get(PID)
model   = dr.Model.get(PID, MID)

dataset = project.upload_dataset(""" Enter Code """)

pred_job = model.request_predictions(dataset_id = dataset.id,
                                     """ Enter Code """)

preds = pred_job.get_result_when_complete()

preds.head(5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1271483539974348, 'fraction_numerical': 0.0055579307396323215, 'mean_word_length': 3.066759388038943, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '38488', 'license': 'Apache-2.0', 'n_tokens_mistral': 3993, 'n_tokens_neox': 3788, 'n_words': 647, 'path': 'DRU/Time_Series/Time_Series_1/Session_2/S2_Lab_1_Start.ipynb', 'repo_name': 'harunpehlivan/tutorials-for-data-scientists', 'size': '11695'}