<commit_before>#generate_warning_points_from_return_periods.py
import netCDF4 as nc
import numpy as np
import os
from json import dumps

def generate_warning_points(ecmwf_prediction_folder, return_period_file, out_directory, threshold=1):
    """
    Create warning points from return periods and ECMWD prediction data

    """

    #Get list of prediciton files

    prediction_files = sorted([os.path.join(ecmwf_prediction_folder,f) for f in os.listdir(ecmwf_prediction_folder) \
                              if not os.path.isdir(os.path.join(ecmwf_prediction_folder, f)) and f.lower().endswith('.nc')],
                              reverse=True)

    #get the comids in ECMWF files
    data_nc = nc.Dataset(prediction_files[0], mode="r")
    prediction_comids = data_nc.variables['COMID'][:]
    comid_list_length = len(prediction_comids)
    time_length = len(data_nc.variables['time'][:])
    data_nc.close()

    first_half_size = 40 #run 6-hr resolution for all
    if time_length == 41 or time_length == 61:
        #run at full or 6-hr resolution for high res and 6-hr for low res
        first_half_size = 41
    elif time_length == 85 or time_length == 125:
        #run at full resolution for all
        first_half_size = 65

    print "Extracting Forecast Data ..."
    #get information from datasets
    reach_prediciton_array_first_half = np.zeros((comid_list_length,len(prediction_files),first_half_size))
    reach_prediciton_array_second_half = np.zeros((comid_list_length,len(prediction_files),20))
    for file_index, prediction_file in enumerate(prediction_files):
        data_values_2d_array = []
        try:
            ensemble_index = int(os.path.basename(prediction_file)[:-3].split("_")[-1])
            #Get hydrograph data from ECMWF Ensemble
            data_nc = nc.Dataset(prediction_file, mode="r")
            qout_dimensions = data_nc.variables['Qout'].dimensions
            if qout_dimensions[0].lower() == 'time' and qout_dimensions[1].lower() == 'comid':
                data_values_2d_array = data_nc.variables['Qout'][:].transpose()
            elif qout_dimensions[0].lower() == 'comid' and qout_dimensions[1].lower() == 'time':
                data_values_2d_array = data_nc.variables['Qout'][:]
            else:
                print "Invalid ECMWF forecast file", prediction_file
                data_nc.close()

        except Exception, e:
            print e
            #pass
        #add data to main arrays and order in order of interim comids
        if len(data_values_2d_array) > 0:
            for comid_index, comid in enumerate(prediction_comids):
                if(ensemble_index < 52):
                    reach_prediciton_array_first_half[comid_index][file_index] = data_values_2d_array[comid_index][:first_half_size]
                    reach_prediciton_array_second_half[comid_index][file_index] = data_values_2d_array[comid_index][first_half_size:]
                if(ensemble_index == 52):
                    if first_half_size == 65:
                        #convert to 3hr-6hr
                        streamflow_1hr = data_values_2d_array[comid_index][:90:3]
                        # get the time series of 3 hr/6 hr data
                        streamflow_3hr_6hr = data_values_2d_array[comid_index][90:]
                        # concatenate all time series
                        reach_prediciton_array_first_half[comid_index][file_index] = np.concatenate([streamflow_1hr, streamflow_3hr_6hr])
                    elif time_length == 125:
                        #convert to 6hr
                        streamflow_1hr = data_values_2d_array[comid_index][:90:6]
                        # calculate time series of 6 hr data from 3 hr data
                        streamflow_3hr = data_values_2d_array[comid_index][90:109:2]
                        # get the time series of 6 hr data
                        streamflow_6hr = data_values_2d_array[comid_index][109:]
                        # concatenate all time series
                        reach_prediciton_array_first_half[comid_index][file_index] = np.concatenate([streamflow_1hr, streamflow_3hr, streamflow_6hr])
                    else:
                        reach_prediciton_array_first_half[comid_index][file_index] = data_values_2d_array[comid_index][:]<commit_msg>Updated script for backwards compatibility, bug fixes<commit_after>#generate_warning_points_from_return_periods.py
import netCDF4 as nc
import numpy as np
import os
from json import dumps

def generate_warning_points(ecmwf_prediction_folder, return_period_file, out_directory, threshold=1):
    """
    Create warning points from return periods and ECMWD prediction data

    """

    #Get list of prediciton files

    prediction_files = sorted([os.path.join(ecmwf_prediction_folder,f) for f in os.listdir(ecmwf_prediction_folder) \
                              if not os.path.isdir(os.path.join(ecmwf_prediction_folder, f)) and f.lower().endswith('.nc')],
                              reverse=True)

    #get the comids in ECMWF files
    data_nc = nc.Dataset(prediction_files[0], mode="r")
    prediction_comids = data_nc.variables['COMID'][:]
    comid_list_length = len(prediction_comids)

    first_half_size = 40 #run 6-hr resolution for all
    if 'time' in data_nc.variables.keys():
        time_length = len(data_nc.variables['time'][:])
        if time_length == 41 or time_length == 61:
            #run at full or 6-hr resolution for high res and 6-hr for low res
            first_half_size = 41
        elif time_length == 85 or time_length == 125:
            #run at full resolution for all
            first_half_size = 65
    data_nc.close()

    print "Extracting Forecast Data ..."
    #get information from datasets
    reach_prediciton_array_first_half = np.zeros((comid_list_length,len(prediction_files),first_half_size))
    reach_prediciton_array_second_half = np.zeros((comid_list_length,len(prediction_files),20))
    for file_index, prediction_file in enumerate(prediction_files):
        data_values_2d_array = []
        try:
            ensemble_index = int(os.path.basename(prediction_file)[:-3].split("_")[-1])
            #Get hydrograph data from ECMWF Ensemble
            data_nc = nc.Dataset(prediction_file, mode="r")
            qout_dimensions = data_nc.variables['Qout'].dimensions
            if qout_dimensions[0].lower() == 'time' and qout_dimensions[1].lower() == 'comid':
                data_values_2d_array = data_nc.variables['Qout'][:].transpose()
            elif qout_dimensions[0].lower() == 'comid' and qout_dimensions[1].lower() == 'time':
                data_values_2d_array = data_nc.variables['Qout'][:]
            else:
                print "Invalid ECMWF forecast file", prediction_file
                data_nc.close()

        except Exception, e:
            print e
            #pass
        #add data to main arrays and order in order of interim comids
        if len(data_values_2d_array) > 0:
            for comid_index, comid in enumerate(prediction_comids):
                if(ensemble_index < 52):
                    reach_prediciton_array_first_half[comid_index][file_index] = data_values_2d_array[comid_index][:first_half_size]
                    reach_prediciton_array_second_half[comid_index][file_index] = data_values_2d_array[comid_index][first_half_size:]
                if(ensemble_index == 52):
                    if first_half_size == 65:
                        #convert to 3hr-6hr
                        streamflow_1hr = data_values_2d_array[comid_index][:90:3]
                        # get the time series of 3 hr/6 hr data
                        streamflow_3hr_6hr = data_values_2d_array[comid_index][90:]
                        # concatenate all time series
                        reach_prediciton_array_first_half[comid_index][file_index] = np.concatenate([streamflow_1hr, streamflow_3hr_6hr])
                    elif len(data_values_2d_array[comid_index]) == 125:
                        #convert to 6hr
                        streamflow_1hr = data_values_2d_array[comid_index][:90:6]
                        # calculate time series of 6 hr data from 3 hr data
                        streamflow_3hr = data_values_2d_array[comid_index][90:109:2]
                        # get the time series of 6 hr data
                        streamflow_6hr = data_values_2d_array[comid_index][109:]
                        # concatenate all time series
                        reach_prediciton_array_first_half[comid_index][file_index] = np.concatenate([streamflow_1hr, streamflow_3hr, streamflow_6hr])
                    else:
                        reach_prediciton_array_first_half[comid_index][file_index] = data_values_2d_array[comid_index][:]
{'fraction_non_alphanumeric': 0.08529613537617196, 'fraction_numerical': 0.020237823004802195, 'mean_word_length': 2.4396382225717654, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '9691', 'n_tokens_mistral': 2770, 'n_tokens_neox': 2589, 'n_words': 539}