<commit_before>import numpy as np
import pandas as pd
import random
import datetime

from ..configuration.power_curve_configuration import PowerCurveConfiguration
from ..configuration.dataset_configuration import DatasetConfiguration
from ..configuration.power_deviation_matrix_configuration import PowerDeviationMatrixConfiguration

import dataset
import binning
import turbine

from power_deviation_matrix import RewsDeviationMatrixDefinition
from power_deviation_matrix import DeviationMatrixDefinition
from power_deviation_matrix import PowerDeviationMatrixDimension

import corrections

from rotor_wind_speed_ratio import RotorWindSpeedRatio
from web_service import WebService

from ..reporting import reporting
from ..core.status import Status

class RandomizeYear:
    
    def __init__(self, time_stamp_column):
        self.time_stamp_column = time_stamp_column
        
    def __call__(self, row):
        
        date_time = row[self.time_stamp_column]
        
        year = random.randint(1980, 2020)
        
        return datetime.datetime(year = year,
                                 month = date_time.month,
                                 day = date_time.day, 
                                 hour = date_time.hour,
                                 minute = date_time.minute,
                                 second = date_time.second)

class SubPower:
            
    def __init__(self, unfiltered_data_frame, filtered_data_frame, aggregations, wind_speed_column, power_polumn, wind_speed_bins, sub_divisions = 4):

        self.sub_divisions = sub_divisions
        self.aggregations = aggregations
        
        self.wind_speed_column = wind_speed_column
        self.power_polumn = power_polumn
        
        self.data_count = "Data Count"
        self.wind_speed_sub_bin_col = "Wind Speed Sub Bin"
             
        Status.add("Creating sub-power bins", verbosity=2)

        self.wind_speed_sub_bins = binning.Bins(self.center_of_first_sub_bin(wind_speed_bins), \
                            self.sub_width(wind_speed_bins), \
                            self.center_of_last_sub_bin(wind_speed_bins))

        self.unfiltered_sub_power = self.calculate_sub_power(unfiltered_data_frame)   
        self.filtered_sub_power = self.calculate_sub_power(filtered_data_frame)

        Status.add("Creating cut-in wind speed", verbosity=2)
        self.cut_in_wind_speed = self.calculate_cut_in_speed(self.unfiltered_sub_power)
    
    def calculate_sub_power(self, data_frame):
        
        # TODO this line generates the following pandas warning
        # data_frame[self.wind_speed_sub_bin_col] = data_frame[self.wind_speed_column].map(self.wind_speed_sub_bins.binCenter)
        # SettingWithCopyWarning: 
        # A value is trying to be set on a copy of a slice from a DataFrame.
        # Try using .loc[row_indexer,col_indexer] = value instead
        # See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy

        data_frame.loc[:, self.wind_speed_sub_bin_col] = data_frame[self.wind_speed_column].map(self.wind_speed_sub_bins.binCenter)

        Status.add("Creating sub-power distribution", verbosity=2)

        sub_distribution = data_frame[self.power_polumn].groupby(data_frame[self.wind_speed_sub_bin_col]).agg({self.data_count:'count'})
        sub_power = data_frame[[self.power_polumn]].groupby(data_frame[self.wind_speed_sub_bin_col]).agg({self.power_polumn:'mean'})
                
        sub_power = sub_power.join(sub_distribution, how = 'inner')
        sub_power.dropna(inplace = True)                           

        return sub_power
        
    def sub_width(self, bins):
        return bins.binWidth / float(self.sub_divisions)

    def center_of_first_sub_bin(self, bins):
        start_of_first_bin =  bins.centerOfFirstBin - 0.5 * bins.binWidth
        return start_of_first_bin + 0.5 * self.sub_width(bins)

    def center_of_last_sub_bin(self, bins):
        return bins.centerOfLastBin + 0.5 * self.sub_width(bins)

    def sub_limit(self, sub_index, start):

        sub_start = start + sub_index * self.wind_speed_sub_bins.binWidth
        sub_end = sub_start + self.wind_speed_sub_bins.binWidth

        return (sub_start, sub_end)
        
    def get_count_for_range(self, start, end):
        
        width = end - start
        
        if width != self.wind_speed_sub_bins.binWidth:
            raise Exception("Unexpected implied bin width for range {0} to {1}. Implied width = {2} vs Expected Width = {3}".format(start, end, width, self.wind_speed_sub_bins.binWidth))
            
        center = 0.5 * (start + end)

        try:

            sub_distribution =  self.filtered_sub_power[self.data_count]
            
            if center in sub_distribution:
                return sub_distribution[center]
            else:
                return 0.0
        
        except Exception as e:
           
           raise Exception("Cannot calculate weight for center {0}: {1}".format(center, e))
         
    def calculate_cut_in_speed(self, sub_power):
            
        first_center = None
        powers = sub_power[self.power_polumn]
        
        for speed in powers.index:    
            
            if powers[speed] > 0:
                if first_center == None or speed < first_center:
                    first_center = speed
        
        if first_center == None:
            raise Exception("Could not determine cut-in")

        cut_in = first_center - 0.5 * self.wind_speed_sub_bins.binWidth
        
        Status.add("Cut-in: {0}".format(cut_in), verbosity=2)
        
        return cut_in

class Analysis(object):

    def __init__(self, config):

        self.define_columns()

        self.apply_settings(config)

        Status.add("Interpolation Mode: %s" % self.interpolationMode)
        Status.add("Power Curve Mode: %s" % self.powerCurveMode)

        self.load_data()

        self.load_specified_power_curve()

        self.calculate_meteorlogical_data_stats()

        self.calculate_baseline_wind_speed()
        self.define_wind_speed_bins()
        
        self.apply_derived_filters()
        self.apply_negative_power_period_treatment()           
        self.calculate_dataset_hours()

        self.calculate_actual_power_curves()         

        self.powerCurve = self.selectPowerCurve(self.powerCurveMode)
        
        self.calculate_power_coefficient()      

        self.calculate_normalised_parameters()
        self.create_calculated_power_deviation_matrix_bins()

        self.calculate_baseline_power()

        self.corrections = {}
        self.calculate_corrections()

        self.calculate_measured_turbulence_power()

        self.calculate_power_deviation_matrices()

        self.calculate_aep()
        
        Status.add("Complete")

    def calculate_meteorlogical_data_stats(self):

        if self.hasDensity:
            self.meanMeasuredSiteDensity = self.dataFrame[self.hubDensity].dropna().mean()     
            Status.add("Mean measured density is %.4f kg/m^3" % self.meanMeasuredSiteDensity)
        else:
            self.meanMeasuredSiteDensity = None

    def calculate_normalised_parameters(self):
        
        self.calculate_normalised_wind_speed()
        self.calculate_rotor_wind_speed_ratio()
        self.calculate_normalised_power()

    def calculate_dataset_hours(self):

        seconds_per_hour = 60.0 * 60.0
        hours_per_time_step = float(self.timeStepInSeconds) / seconds_per_hour

        self.hours = len(self.dataFrame.index) * hours_per_time_step

        Status.add("Total of %.3f hours of data used in analysis." % self.hours)

    def define_wind_speed_bins(self):

        self.windSpeedBins = binning.Bins(self.powerCurveFirstBin, self.powerCurveBinSize, self.powerCurveLastBin)
        self.dataFrame[self.windSpeedBin] = self.dataFrame[self.baseline.wind_speed_column].map(self.windSpeedBins.binCenter)        

    def apply_negative_power_period_treatment(self):

        if not self.hasActualPower:
            return

        negative_powers = (self.dataFrame[self.actualPower] < 0)
        negative_powers_count = np.sum(negative_powers)
        
        if negative_powers_count > 0:
            
            if self.negative_power_period_treatment == 'Remove': 
                Status.add("Removing {0} negative power periods".format(negative_powers_count))
                self.dataFrame = self.dataFrame[~negative_powers]
            elif self.negative_power_period_treatment == 'Set to Zero': 
                Status.add("Setting {0} negative power periods to zero".format(negative_powers_count))
                self.dataFrame.loc[negative_powers, self.actualPower] = 0.0
            elif self.negative_power_period_treatment == 'Keep':
                Status.add("Keeping {0} negative power periods".format(negative_powers_count))
            else:
                raise Exception('Unknown negative power period treatment: {0}'.format(self.negative_power_period_treatment))

    def calculate_normalised_wind_speed(self):

        self.zero_ti_rated_power = self.powerCurve.zeroTurbulencePowerCurve.initialZeroTurbulencePowerCurve.selectedStats.ratedPower
        self.zero_ti_rated_wind_speed = self.powerCurve.zeroTurbulencePowerCurve.initialZeroTurbulencePowerCurve.ratedWindSpeed
        self.zero_to_cut_in_wind_speed = self.powerCurve.zeroTurbulencePowerCurve.initialZeroTurbulencePowerCurve.selectedStats.cutInWindSpeed

        Status.add("Zero TI Wind Speeds", verbosity=2)
        Status.add("Zero TI Rated Wind Speed: {0}".format(self.zero_ti_rated_wind_speed), verbosity=2)
        Status.add("Zero TI Cut-In Wind Speed: {0}".format(self.zero_to_cut_in_wind_speed), verbosity=2)
        
        self.normalisedWS = 'Normalised Wind Speed'

        self.dataFrame[self.normalisedWS] = (self.dataFrame[self.baseline.wind_speed_column] - self.zero_to_cut_in_wind_speed) / (self.zero_ti_rated_wind_speed - self.zero_to_cut_in_wind_speed)

    def calculate_power_coefficient(self):

        if self.density_pre_correction_active:
            return 

        if not self.hasActualPower:
            return 

        if not self.hasDensity:
            return 

        if not self.densityCorrectionActive:
            return

        if self.reference_density is None:
            raise Exception("Reference Density not Defined") 

        area = np.pi*(self.rotorGeometry.diameter/2.0)**2
        a = 1000*self.dataFrame[self.actualPower]/(0.5*self.dataFrame[self.hubDensity] *area*np.power(self.dataFrame[self.hubWindSpeed],3))
        b = 1000*self.dataFrame[self.actualPower]/(0.5*self.reference_density*area*np.power(self.dataFrame[self.baseline.wind_speed_column],3))
        
        fraction_of_data_points_exceeding_betz_limit = (len(a[a>16.0/27])*100.0)/len(a)
        
        if fraction_of_data_points_exceeding_betz_limit > 0.001:
            Status.add("{0:.02}% data points slightly exceed Betz limit - if this number is high, investigate...".format(fraction_of_data_points_exceeding_betz_limit), verbosity=2)

        if (abs(a-b) > 0.005).any():
            raise Exception("Density correction has not been applied consistently.")
        
        self.dataFrame[self.powerCoeff] = a

    def load_specified_power_curve(self):

        if self.powerCurveMode != 'Specified':
            self.specified_power_curve = None
            return

        if self.specified_power_curve.absolute_path is None:
            raise Exception("Power Curve Mode is set to Specified, but Path to specified power curve not defined")

        power_curve_config = PowerCurveConfiguration(self.specified_power_curve.absolute_path)            
        
        self.specified_power_curve = turbine.PowerCurve(rotor_geometry=self.rotorGeometry, 
                                                          reference_density=power_curve_config.density, 
                                                          data_frame=power_curve_config.data_frame,
                                                          wind_speed_column=power_curve_config.speed_column, 
                                                          turbulence_column=power_curve_config.turbulence_column, 
                                                          power_column=power_curve_config.power_column, 
                                                          name = 'Specified', 
                                                          interpolation_mode = self.interpolationMode, 
                                                          zero_ti_pc_required = (self.powerCurveMode == 'Specified'))
            
    def load_specified_power_deviation_matrix(self):
        
        Status.add("Loading power deviation matrix...")
        
        if self.specified_power_deviation_matrix.absolute_path is None:
            raise Exception("Power deviation matrix path not set.")

        self.specifiedPowerDeviationMatrix = PowerDeviationMatrixConfiguration(self.specified_power_deviation_matrix.absolute_path)

    def calculate_actual_power_curves(self):

        if not self.hasActualPower:
            return

        Status.add("Calculating actual power curves...")

        self.aggregations = binning.Aggregations(self.powerCurveMinimumCount)

        self.calculate_all_measured_power_curves()        
        self.calculate_day_night_power_curves()
        self.calculate_inner_outer_measured_power_curves()

        Status.add("Actual Power Curves Complete.")

    def calculate_all_measured_power_curves(self):
        self.allMeasuredPowerCurve = self.calculateMeasuredPowerCurve(self.get_base_filter, self.cutInWindSpeed, self.cutOutWindSpeed, self.ratedPower, self.actualPower, 'All Measured', zero_ti_pc_required = (self.powerCurveMode == 'AllMeasured'))

    def calculate_day_night_power_curves(self):
        self.dayTimePowerCurve = self.calculateMeasuredPowerCurve(self.get_day_filter, self.cutInWindSpeed, self.cutOutWindSpeed, self.ratedPower, self.actualPower, 'Day Time', zero_ti_pc_required = False)
        self.nightTimePowerCurve = self.calculateMeasuredPowerCurve(self.get_night_filter, self.cutInWindSpeed, self.cutOutWindSpeed, self.ratedPower, self.actualPower, 'Night Time', zero_ti_pc_required = False)

    def calculate_inner_outer_measured_power_curves(self):

        if self.hasShear:
            self.innerMeasuredPowerCurve = self.calculate_inner_measured_power_curve()            
            self.outerMeasuredPowerCurve = self.calculate_outer_measured_power_curve()            
        else:
            self.innerMeasuredPowerCurve = None
            self.outerMeasuredPowerCurve = None

    def calculate_inner_measured_power_curve(self, supress_zero_turbulence_curve_creation=False):
            
            if supress_zero_turbulence_curve_creation:
                zero_ti_pc_required = False
            else:
                zero_ti_pc_required = (self.powerCurveMode == 'InnerMeasured')

            return self.calculateMeasuredPowerCurve(self.get_inner_range_filter, self.cutInWindSpeed, self.cutOutWindSpeed, self.ratedPower, self.actualPower, 'Inner Range', zero_ti_pc_required = zero_ti_pc_required)            

    def calculate_outer_measured_power_curve(self):
            return self.calculateMeasuredPowerCurve(self.get_outer_range_filter, self.cutInWindSpeed, self.cutOutWindSpeed, self.ratedPower, self.actualPower, 'Outer Range', zero_ti_pc_required = (self.powerCurveMode == 'OuterMeasured'))

    def calculate_rotor_wind_speed_ratio(self):

        if self.hasShear:
            self.rotor_wind_speed_ratio = 'Rotor Wind Speed Ratio'
            self.dataFrame[self.rotor_wind_speed_ratio] = self.dataFrame[self.shearExponent].map(RotorWindSpeedRatio(self.rotorGeometry.diameter, self.rotorGeometry.hubHeight))
        else:
            self.rotor_wind_speed_ratio = None

    def calculate_normalised_power(self):

        if self.hasActualPower:
            self.normalisedPower = 'Normalised Power'
            self.dataFrame[self.normalisedPower] = self.dataFrame[self.actualPower] / self.ratedPower

    def apply_settings(self, config):

        Status.add("Applying settings...")            

        self.densityCorrectionActive = config.densityCorrectionActive
        
        self.rewsActive = config.rewsActive
        self.rewsVeer = config.rewsVeer
        self.rewsUpflow = config.rewsUpflow
        self.rewsExponent = config.rewsExponent

        self.turbRenormActive = config.turbRenormActive
        self.powerDeviationMatrixActive = config.powerDeviationMatrixActive
        self.productionByHeightActive = config.productionByHeightActive
        self.web_service_active = config.web_service_active

        self.web_service_url = config.web_service_url
        self.specified_power_deviation_matrix = config.specified_power_deviation_matrix

        self.powerCurveMinimumCount = config.powerCurveMinimumCount
        self.powerCurvePaddingMode = config.powerCurvePaddingMode

        self.interpolationMode = config.interpolationMode
        self.powerCurveMode = config.powerCurveMode
        
        self.negative_power_period_treatment = config.negative_power_period_treatment
        self.negative_power_bin_average_treatment = config.negative_power_bin_average_treatment

        self.powerCurveFirstBin = config.powerCurveFirstBin
        self.powerCurveBinSize = config.powerCurveBinSize
        self.powerCurveLastBin = config.powerCurveLastBin

        self.specified_power_curve = config.specified_power_curve

        self.calculated_power_deviation_matrix_definition = DeviationMatrixDefinition(
                        method=config.power_deviation_matrix_method,
                        dimensions=config.calculated_power_deviation_matrix_dimensions,
                        minimum_count=config.power_deviation_matrix_minimum_count)

        self.rews_deviation_matrix_definition = RewsDeviationMatrixDefinition(
                        method=config.power_deviation_matrix_method,
                        dimensions=config.calculated_power_deviation_matrix_dimensions,
                        minimum_count=config.power_deviation_matrix_minimum_count)

        self.nominal_wind_speed_distribution = config.nominal_wind_speed_distribution

        self.inner_range_dimensions = config.inner_range_dimensions

        self.datasetConfigs = []

        for dataset in config.datasets:

            if not isinstance(dataset, DatasetConfiguration):
                self.datasetConfigs.append(DatasetConfiguration(dataset.absolute_path))
            else:
                self.datasetConfigs.append(datasets[i])

    def define_columns(self):

        self.nameColumn = "Dataset Name"
        self.turbulencePower = "Simulated TI Corrected Power"
        self.windSpeedBin = "Wind Speed Bin"
        self.dataCount = "Data Count"
        self.powerStandDev = "Power Standard Deviation"
        self.powerCoeff = "Power Coefficient"
        self.measuredTurbulencePower = 'Measured TI Corrected Power'
        self.measuredTurbPowerCurveInterp = 'Measured TI Corrected Power Curve Interp'
        self.measuredPowerCurveInterp = 'All Measured Power Curve Interp'

    def calculate_power_deviation_matrices(self):<commit_msg>fix appending dataset in analysis.py<commit_after>import numpy as np
import pandas as pd
import random
import datetime

from ..configuration.power_curve_configuration import PowerCurveConfiguration
from ..configuration.dataset_configuration import DatasetConfiguration
from ..configuration.power_deviation_matrix_configuration import PowerDeviationMatrixConfiguration

import dataset
import binning
import turbine

from power_deviation_matrix import RewsDeviationMatrixDefinition
from power_deviation_matrix import DeviationMatrixDefinition
from power_deviation_matrix import PowerDeviationMatrixDimension

import corrections

from rotor_wind_speed_ratio import RotorWindSpeedRatio
from web_service import WebService

from ..reporting import reporting
from ..core.status import Status

class RandomizeYear:
    
    def __init__(self, time_stamp_column):
        self.time_stamp_column = time_stamp_column
        
    def __call__(self, row):
        
        date_time = row[self.time_stamp_column]
        
        year = random.randint(1980, 2020)
        
        return datetime.datetime(year = year,
                                 month = date_time.month,
                                 day = date_time.day, 
                                 hour = date_time.hour,
                                 minute = date_time.minute,
                                 second = date_time.second)

class SubPower:
            
    def __init__(self, unfiltered_data_frame, filtered_data_frame, aggregations, wind_speed_column, power_polumn, wind_speed_bins, sub_divisions = 4):

        self.sub_divisions = sub_divisions
        self.aggregations = aggregations
        
        self.wind_speed_column = wind_speed_column
        self.power_polumn = power_polumn
        
        self.data_count = "Data Count"
        self.wind_speed_sub_bin_col = "Wind Speed Sub Bin"
             
        Status.add("Creating sub-power bins", verbosity=2)

        self.wind_speed_sub_bins = binning.Bins(self.center_of_first_sub_bin(wind_speed_bins), \
                            self.sub_width(wind_speed_bins), \
                            self.center_of_last_sub_bin(wind_speed_bins))

        self.unfiltered_sub_power = self.calculate_sub_power(unfiltered_data_frame)   
        self.filtered_sub_power = self.calculate_sub_power(filtered_data_frame)

        Status.add("Creating cut-in wind speed", verbosity=2)
        self.cut_in_wind_speed = self.calculate_cut_in_speed(self.unfiltered_sub_power)
    
    def calculate_sub_power(self, data_frame):
        
        # TODO this line generates the following pandas warning
        # data_frame[self.wind_speed_sub_bin_col] = data_frame[self.wind_speed_column].map(self.wind_speed_sub_bins.binCenter)
        # SettingWithCopyWarning: 
        # A value is trying to be set on a copy of a slice from a DataFrame.
        # Try using .loc[row_indexer,col_indexer] = value instead
        # See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy

        data_frame.loc[:, self.wind_speed_sub_bin_col] = data_frame[self.wind_speed_column].map(self.wind_speed_sub_bins.binCenter)

        Status.add("Creating sub-power distribution", verbosity=2)

        sub_distribution = data_frame[self.power_polumn].groupby(data_frame[self.wind_speed_sub_bin_col]).agg({self.data_count:'count'})
        sub_power = data_frame[[self.power_polumn]].groupby(data_frame[self.wind_speed_sub_bin_col]).agg({self.power_polumn:'mean'})
                
        sub_power = sub_power.join(sub_distribution, how = 'inner')
        sub_power.dropna(inplace = True)                           

        return sub_power
        
    def sub_width(self, bins):
        return bins.binWidth / float(self.sub_divisions)

    def center_of_first_sub_bin(self, bins):
        start_of_first_bin =  bins.centerOfFirstBin - 0.5 * bins.binWidth
        return start_of_first_bin + 0.5 * self.sub_width(bins)

    def center_of_last_sub_bin(self, bins):
        return bins.centerOfLastBin + 0.5 * self.sub_width(bins)

    def sub_limit(self, sub_index, start):

        sub_start = start + sub_index * self.wind_speed_sub_bins.binWidth
        sub_end = sub_start + self.wind_speed_sub_bins.binWidth

        return (sub_start, sub_end)
        
    def get_count_for_range(self, start, end):
        
        width = end - start
        
        if width != self.wind_speed_sub_bins.binWidth:
            raise Exception("Unexpected implied bin width for range {0} to {1}. Implied width = {2} vs Expected Width = {3}".format(start, end, width, self.wind_speed_sub_bins.binWidth))
            
        center = 0.5 * (start + end)

        try:

            sub_distribution =  self.filtered_sub_power[self.data_count]
            
            if center in sub_distribution:
                return sub_distribution[center]
            else:
                return 0.0
        
        except Exception as e:
           
           raise Exception("Cannot calculate weight for center {0}: {1}".format(center, e))
         
    def calculate_cut_in_speed(self, sub_power):
            
        first_center = None
        powers = sub_power[self.power_polumn]
        
        for speed in powers.index:    
            
            if powers[speed] > 0:
                if first_center == None or speed < first_center:
                    first_center = speed
        
        if first_center == None:
            raise Exception("Could not determine cut-in")

        cut_in = first_center - 0.5 * self.wind_speed_sub_bins.binWidth
        
        Status.add("Cut-in: {0}".format(cut_in), verbosity=2)
        
        return cut_in

class Analysis(object):

    def __init__(self, config):

        self.define_columns()

        self.apply_settings(config)

        Status.add("Interpolation Mode: %s" % self.interpolationMode)
        Status.add("Power Curve Mode: %s" % self.powerCurveMode)

        self.load_data()

        self.load_specified_power_curve()

        self.calculate_meteorlogical_data_stats()

        self.calculate_baseline_wind_speed()
        self.define_wind_speed_bins()
        
        self.apply_derived_filters()
        self.apply_negative_power_period_treatment()           
        self.calculate_dataset_hours()

        self.calculate_actual_power_curves()         

        self.powerCurve = self.selectPowerCurve(self.powerCurveMode)
        
        self.calculate_power_coefficient()      

        self.calculate_normalised_parameters()
        self.create_calculated_power_deviation_matrix_bins()

        self.calculate_baseline_power()

        self.corrections = {}
        self.calculate_corrections()

        self.calculate_measured_turbulence_power()

        self.calculate_power_deviation_matrices()

        self.calculate_aep()
        
        Status.add("Complete")

    def calculate_meteorlogical_data_stats(self):

        if self.hasDensity:
            self.meanMeasuredSiteDensity = self.dataFrame[self.hubDensity].dropna().mean()     
            Status.add("Mean measured density is %.4f kg/m^3" % self.meanMeasuredSiteDensity)
        else:
            self.meanMeasuredSiteDensity = None

    def calculate_normalised_parameters(self):
        
        self.calculate_normalised_wind_speed()
        self.calculate_rotor_wind_speed_ratio()
        self.calculate_normalised_power()

    def calculate_dataset_hours(self):

        seconds_per_hour = 60.0 * 60.0
        hours_per_time_step = float(self.timeStepInSeconds) / seconds_per_hour

        self.hours = len(self.dataFrame.index) * hours_per_time_step

        Status.add("Total of %.3f hours of data used in analysis." % self.hours)

    def define_wind_speed_bins(self):

        self.windSpeedBins = binning.Bins(self.powerCurveFirstBin, self.powerCurveBinSize, self.powerCurveLastBin)
        self.dataFrame[self.windSpeedBin] = self.dataFrame[self.baseline.wind_speed_column].map(self.windSpeedBins.binCenter)        

    def apply_negative_power_period_treatment(self):

        if not self.hasActualPower:
            return

        negative_powers = (self.dataFrame[self.actualPower] < 0)
        negative_powers_count = np.sum(negative_powers)
        
        if negative_powers_count > 0:
            
            if self.negative_power_period_treatment == 'Remove': 
                Status.add("Removing {0} negative power periods".format(negative_powers_count))
                self.dataFrame = self.dataFrame[~negative_powers]
            elif self.negative_power_period_treatment == 'Set to Zero': 
                Status.add("Setting {0} negative power periods to zero".format(negative_powers_count))
                self.dataFrame.loc[negative_powers, self.actualPower] = 0.0
            elif self.negative_power_period_treatment == 'Keep':
                Status.add("Keeping {0} negative power periods".format(negative_powers_count))
            else:
                raise Exception('Unknown negative power period treatment: {0}'.format(self.negative_power_period_treatment))

    def calculate_normalised_wind_speed(self):

        self.zero_ti_rated_power = self.powerCurve.zeroTurbulencePowerCurve.initialZeroTurbulencePowerCurve.selectedStats.ratedPower
        self.zero_ti_rated_wind_speed = self.powerCurve.zeroTurbulencePowerCurve.initialZeroTurbulencePowerCurve.ratedWindSpeed
        self.zero_to_cut_in_wind_speed = self.powerCurve.zeroTurbulencePowerCurve.initialZeroTurbulencePowerCurve.selectedStats.cutInWindSpeed

        Status.add("Zero TI Wind Speeds", verbosity=2)
        Status.add("Zero TI Rated Wind Speed: {0}".format(self.zero_ti_rated_wind_speed), verbosity=2)
        Status.add("Zero TI Cut-In Wind Speed: {0}".format(self.zero_to_cut_in_wind_speed), verbosity=2)
        
        self.normalisedWS = 'Normalised Wind Speed'

        self.dataFrame[self.normalisedWS] = (self.dataFrame[self.baseline.wind_speed_column] - self.zero_to_cut_in_wind_speed) / (self.zero_ti_rated_wind_speed - self.zero_to_cut_in_wind_speed)

    def calculate_power_coefficient(self):

        if self.density_pre_correction_active:
            return 

        if not self.hasActualPower:
            return 

        if not self.hasDensity:
            return 

        if not self.densityCorrectionActive:
            return

        if self.reference_density is None:
            raise Exception("Reference Density not Defined") 

        area = np.pi*(self.rotorGeometry.diameter/2.0)**2
        a = 1000*self.dataFrame[self.actualPower]/(0.5*self.dataFrame[self.hubDensity] *area*np.power(self.dataFrame[self.hubWindSpeed],3))
        b = 1000*self.dataFrame[self.actualPower]/(0.5*self.reference_density*area*np.power(self.dataFrame[self.baseline.wind_speed_column],3))
        
        fraction_of_data_points_exceeding_betz_limit = (len(a[a>16.0/27])*100.0)/len(a)
        
        if fraction_of_data_points_exceeding_betz_limit > 0.001:
            Status.add("{0:.02}% data points slightly exceed Betz limit - if this number is high, investigate...".format(fraction_of_data_points_exceeding_betz_limit), verbosity=2)

        if (abs(a-b) > 0.005).any():
            raise Exception("Density correction has not been applied consistently.")
        
        self.dataFrame[self.powerCoeff] = a

    def load_specified_power_curve(self):

        if self.powerCurveMode != 'Specified':
            self.specified_power_curve = None
            return

        if self.specified_power_curve.absolute_path is None:
            raise Exception("Power Curve Mode is set to Specified, but Path to specified power curve not defined")

        power_curve_config = PowerCurveConfiguration(self.specified_power_curve.absolute_path)            
        
        self.specified_power_curve = turbine.PowerCurve(rotor_geometry=self.rotorGeometry, 
                                                          reference_density=power_curve_config.density, 
                                                          data_frame=power_curve_config.data_frame,
                                                          wind_speed_column=power_curve_config.speed_column, 
                                                          turbulence_column=power_curve_config.turbulence_column, 
                                                          power_column=power_curve_config.power_column, 
                                                          name = 'Specified', 
                                                          interpolation_mode = self.interpolationMode, 
                                                          zero_ti_pc_required = (self.powerCurveMode == 'Specified'))
            
    def load_specified_power_deviation_matrix(self):
        
        Status.add("Loading power deviation matrix...")
        
        if self.specified_power_deviation_matrix.absolute_path is None:
            raise Exception("Power deviation matrix path not set.")

        self.specifiedPowerDeviationMatrix = PowerDeviationMatrixConfiguration(self.specified_power_deviation_matrix.absolute_path)

    def calculate_actual_power_curves(self):

        if not self.hasActualPower:
            return

        Status.add("Calculating actual power curves...")

        self.aggregations = binning.Aggregations(self.powerCurveMinimumCount)

        self.calculate_all_measured_power_curves()        
        self.calculate_day_night_power_curves()
        self.calculate_inner_outer_measured_power_curves()

        Status.add("Actual Power Curves Complete.")

    def calculate_all_measured_power_curves(self):
        self.allMeasuredPowerCurve = self.calculateMeasuredPowerCurve(self.get_base_filter, self.cutInWindSpeed, self.cutOutWindSpeed, self.ratedPower, self.actualPower, 'All Measured', zero_ti_pc_required = (self.powerCurveMode == 'AllMeasured'))

    def calculate_day_night_power_curves(self):
        self.dayTimePowerCurve = self.calculateMeasuredPowerCurve(self.get_day_filter, self.cutInWindSpeed, self.cutOutWindSpeed, self.ratedPower, self.actualPower, 'Day Time', zero_ti_pc_required = False)
        self.nightTimePowerCurve = self.calculateMeasuredPowerCurve(self.get_night_filter, self.cutInWindSpeed, self.cutOutWindSpeed, self.ratedPower, self.actualPower, 'Night Time', zero_ti_pc_required = False)

    def calculate_inner_outer_measured_power_curves(self):

        if self.hasShear:
            self.innerMeasuredPowerCurve = self.calculate_inner_measured_power_curve()            
            self.outerMeasuredPowerCurve = self.calculate_outer_measured_power_curve()            
        else:
            self.innerMeasuredPowerCurve = None
            self.outerMeasuredPowerCurve = None

    def calculate_inner_measured_power_curve(self, supress_zero_turbulence_curve_creation=False):
            
            if supress_zero_turbulence_curve_creation:
                zero_ti_pc_required = False
            else:
                zero_ti_pc_required = (self.powerCurveMode == 'InnerMeasured')

            return self.calculateMeasuredPowerCurve(self.get_inner_range_filter, self.cutInWindSpeed, self.cutOutWindSpeed, self.ratedPower, self.actualPower, 'Inner Range', zero_ti_pc_required = zero_ti_pc_required)            

    def calculate_outer_measured_power_curve(self):
            return self.calculateMeasuredPowerCurve(self.get_outer_range_filter, self.cutInWindSpeed, self.cutOutWindSpeed, self.ratedPower, self.actualPower, 'Outer Range', zero_ti_pc_required = (self.powerCurveMode == 'OuterMeasured'))

    def calculate_rotor_wind_speed_ratio(self):

        if self.hasShear:
            self.rotor_wind_speed_ratio = 'Rotor Wind Speed Ratio'
            self.dataFrame[self.rotor_wind_speed_ratio] = self.dataFrame[self.shearExponent].map(RotorWindSpeedRatio(self.rotorGeometry.diameter, self.rotorGeometry.hubHeight))
        else:
            self.rotor_wind_speed_ratio = None

    def calculate_normalised_power(self):

        if self.hasActualPower:
            self.normalisedPower = 'Normalised Power'
            self.dataFrame[self.normalisedPower] = self.dataFrame[self.actualPower] / self.ratedPower

    def apply_settings(self, config):

        Status.add("Applying settings...")            

        self.densityCorrectionActive = config.densityCorrectionActive
        
        self.rewsActive = config.rewsActive
        self.rewsVeer = config.rewsVeer
        self.rewsUpflow = config.rewsUpflow
        self.rewsExponent = config.rewsExponent

        self.turbRenormActive = config.turbRenormActive
        self.powerDeviationMatrixActive = config.powerDeviationMatrixActive
        self.productionByHeightActive = config.productionByHeightActive
        self.web_service_active = config.web_service_active

        self.web_service_url = config.web_service_url
        self.specified_power_deviation_matrix = config.specified_power_deviation_matrix

        self.powerCurveMinimumCount = config.powerCurveMinimumCount
        self.powerCurvePaddingMode = config.powerCurvePaddingMode

        self.interpolationMode = config.interpolationMode
        self.powerCurveMode = config.powerCurveMode
        
        self.negative_power_period_treatment = config.negative_power_period_treatment
        self.negative_power_bin_average_treatment = config.negative_power_bin_average_treatment

        self.powerCurveFirstBin = config.powerCurveFirstBin
        self.powerCurveBinSize = config.powerCurveBinSize
        self.powerCurveLastBin = config.powerCurveLastBin

        self.specified_power_curve = config.specified_power_curve

        self.calculated_power_deviation_matrix_definition = DeviationMatrixDefinition(
                        method=config.power_deviation_matrix_method,
                        dimensions=config.calculated_power_deviation_matrix_dimensions,
                        minimum_count=config.power_deviation_matrix_minimum_count)

        self.rews_deviation_matrix_definition = RewsDeviationMatrixDefinition(
                        method=config.power_deviation_matrix_method,
                        dimensions=config.calculated_power_deviation_matrix_dimensions,
                        minimum_count=config.power_deviation_matrix_minimum_count)

        self.nominal_wind_speed_distribution = config.nominal_wind_speed_distribution

        self.inner_range_dimensions = config.inner_range_dimensions

        self.datasetConfigs = []

        for dataset in config.datasets:

            if not isinstance(dataset, DatasetConfiguration):
                self.datasetConfigs.append(DatasetConfiguration(dataset.absolute_path))
            else:
                self.datasetConfigs.append(dataset)

    def define_columns(self):

        self.nameColumn = "Dataset Name"
        self.turbulencePower = "Simulated TI Corrected Power"
        self.windSpeedBin = "Wind Speed Bin"
        self.dataCount = "Data Count"
        self.powerStandDev = "Power Standard Deviation"
        self.powerCoeff = "Power Coefficient"
        self.measuredTurbulencePower = 'Measured TI Corrected Power'
        self.measuredTurbPowerCurveInterp = 'Measured TI Corrected Power Curve Interp'
        self.measuredPowerCurveInterp = 'All Measured Power Curve Interp'

    def calculate_power_deviation_matrices(self):
{'fraction_non_alphanumeric': 0.07349251105879918, 'fraction_numerical': 0.0048115477145148355, 'mean_word_length': 2.6852240228789324, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3671345', 'n_tokens_mistral': 11427, 'n_tokens_neox': 10712, 'n_words': 1765}