<jupyter_start><jupyter_text>Load directories<jupyter_code>root_directory = 'D:/github/ekostat_calculator'#"../" #os.getcwd()
workspace_directory = root_directory + '/workspaces' 
resource_directory = root_directory + '/resources'
#alias = 'lena'
user_id = 'test_user' #kanske ska vara off_line user?
workspace_alias = 'lena_indicator'<jupyter_output><empty_output><jupyter_text>Initiate EventHandler<jupyter_code>print(root_directory)
paths = {'user_id': user_id, 
         'workspace_directory': root_directory + '/workspaces', 
         'resource_directory': root_directory + '/resources', 
         'log_directory': 'D:/github' + '/log', 
         'test_data_directory': 'D:/github' + '/test_data'}
t0 = time.time()
ekos = EventHandler(**paths)
#request = ekos.test_requests['request_workspace_list']
#response = ekos.request_workspace_list(request) 
#ekos.write_test_response('request_workspace_list', response)
print('-'*50)
print('Time for request: {}'.format(time.time()-t0))
# OLD: ekos = EventHandler(root_directory)<jupyter_output>2018-06-01 14:43:32,592	event_handler.py	74	__init__	DEBUG	Start EventHandler: event_handler<jupyter_text>Load existing workspace<jupyter_code>#ekos.copy_workspace(source_uuid='default_workspace', target_alias='lena_1')
ekos.print_workspaces()
workspace_uuid = ekos.get_unique_id_for_alias(workspace_alias = 'lena_indicator')
print(workspace_uuid)
workspace_alias = ekos.get_alias_for_unique_id(workspace_unique_id = workspace_uuid)
ekos.load_workspace(unique_id = workspace_uuid)
# Här får jag ofta felmeddelande:
# AttributeError: module 'core' has no attribute 'ParameterMapping'
ekos.import_default_data(workspace_alias = workspace_alias)<jupyter_output>2018-06-01 13:43:03,753	event_handler.py	1024	import_default_data	DEBUG	Trying to load default data in workspace "6fe05a39-2018-4cba-9dd4-27057578ff23" with alias "lena_1"
2018-06-01 13:43:03,770	workspaces.py	1006	import_default_data	DEBUG	Default data file has been copied to workspace raw data folder: chlorophyll_integrated_2015_2016_row_format.txt
2018-06-01 13:43:03,792	workspaces.py	1006	import_default_data	DEBUG	Default data file has been copied to workspace raw data folder: chlorophyll_sharkweb_data_chlorophyll_wb_2007-2017_20180320.txt
2018-06-01 13:43:03,805	workspaces.py	1006	import_default_data	DEBUG	Default data file has been copied to workspace raw data folder: physicalchemicalmodel_110001_PROFILER_alldepths_SE652400-223501_toolbox.dat
2018-06-01 13:43:03,829	workspaces.py	1006	import_default_data	DEBUG	Default data file has been copied to workspace raw data folder: physicalchemicalmodel_120004_PROFILER_alldepths_SE612520-172080_toolbox.dat<jupyter_text>Load all data in workspace<jupyter_code>#ekos.get_workspace(unique_id = workspace_uuid, alias = workspace_alias).delete_alldata_export()
#%%timeit
ekos.load_data(unique_id = workspace_uuid)
w = ekos.get_workspace(unique_id = workspace_uuid)
len(w.data_handler.get_all_column_data_df())<jupyter_output>2018-06-01 14:44:07,667	event_handler.py	1003	get_workspace	DEBUG	Getting workspace "6fe05a39-2018-4cba-9dd4-27057578ff23" with alias "None"<jupyter_text>Step 0   Apply first data filter<jupyter_code>w.apply_data_filter(step = 0) # This sets the first level of data filter in the IndexHandler<jupyter_output><empty_output><jupyter_text>Step 1 Set subset filter<jupyter_code>#w.copy_subset(source_uuid='default_subset', target_alias='A')
subset_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias, subset_alias = 'A')
print(w.get_subset_list(), subset_uuid)

f1 = w.get_data_filter_object(subset = subset_uuid, step=1) 
print(f1.include_list_filter)

w.apply_data_filter(subset = subset_uuid, step = 1)

df_step1 = w.get_filtered_data(step = 1, subset = subset_uuid)
#print(df_step1.columns)
#df_step1[['SDATE', 'YEAR', 'MONTH', 'POSITION', 'VISS_EU_CD', 'WATER_TYPE_AREA', 'DEPH', 'MNDEP', 'MXDEP','BQIm']].dropna(subset = ['BQIm'])<jupyter_output>['default_subset', 'fb1db7b1-b6bd-4ef9-9ee7-84520b9be8c1'] fb1db7b1-b6bd-4ef9-9ee7-84520b9be8c1
{'MYEAR': ['2013', '2014', '2015', '2016', '2017', '2018', '2019'], 'STATN': [], 'WATER_BODY': []}<jupyter_text>Step 2<jupyter_code>water_body = 'SE574000-114230' # type_area 1-s,
w.get_step_object(step = 2, subset = subset_uuid).load_indicator_settings_filters()

temp = w.get_step_object(step = 2, subset = subset_uuid).indicator_data_filter_settings['oxygen']
temp.get_value(variable='REF_VALUE', type_area='1', water_body = water_body)
#temp.settings.df
w.get_step_object(step = 2, subset = subset_uuid).get_indicator_data_filter_settings('oxygen')
w.get_step_object(step = 2, subset = subset_uuid).indicator_ref_settings
w.get_step_object(step = 2, subset = subset_uuid)._indicator_setting_files['oxygen']
indicator_list = w.get_available_indicators(subset= 'A', step=2)<jupyter_output>2018-06-01 14:44:29,443	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,464	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,478	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,485	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,492	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,500	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,509	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,517	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,526	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,538	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,551	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2
2018-06-01 14:44:29,566	workspaces.py	1410	get_filtered_data	DEBUG	STEP: step_2<jupyter_text>Apply indicator filter<jupyter_code>wb_list = df_step1.VISS_EU_CD.unique()
print('number of waterbodies in step 1: {}'.format(len(wb_list)))
typeA_list = [row.split('-')[0].strip().lstrip('0') for row in df_step1.WATER_TYPE_AREA.unique()]
print('number of type areas in step 1: {}'.format(len(typeA_list)))
wb_list
#list(zip(typeA_list, df_step1.WATER_TYPE_AREA.unique()))
indicator_list = ['oxygen','bqi','din_winter','ntot_summer', 'ntot_winter', 'dip_winter', 'ptot_summer', 'ptot_winter', 'biov', 'chl', 'secchi']
for indicator in indicator_list:
    w.apply_indicator_data_filter(step = 2, 
                          subset = subset_uuid, 
                          indicator = indicator)
w.index_handler.booleans['step_0'][subset_uuid]['step_1']['step_2']['SE584340-174401'].keys()
wb = 'SE583926-161744' #typomr 22
#wb = 'SE654470-222700' #typomr 13
type_area = '2'#'01s - Västkustens inre kustvatten'
indicator = 'din_winter'
#w.index_handler.booleans['step_0'][subset_uuid]['step_1']['step_2'][type_area]['din_winter']['boolean']
print(w.get_filtered_data(step = 2, subset = subset_uuid, water_body = wb, indicator = indicator).MONTH.unique())
print(w.get_filtered_data(step = 2, subset = subset_uuid, water_body = wb, indicator = indicator).DEPH.min(),
        w.get_filtered_data(step = 2, subset = subset_uuid, water_body = wb, indicator = indicator).DEPH.max())
print(w.get_filtered_data(step = 2, subset = subset_uuid, water_body = wb, indicator = indicator).VISS_EU_CD.unique())
w.get_filtered_data(step = 2, subset = subset_uuid, water_body = wb).WATER_TYPE_AREA.unique()
w.mapping_objects['quality_element'].cfg['indicators']
[item.strip() for item in w.mapping_objects['quality_element'].cfg['indicators'].loc[indicator][0].split(', ')]<jupyter_output><empty_output><jupyter_text>Step 3 Load Indicator objects step 3....<jupyter_code>w.get_step_object(step = 3, subset = subset_uuid).indicator_setup(subset_unique_id = subset_uuid) 
#, indicator_list = ['din_winter', 'dip_winter']
w.get_step_object(step = 3, subset = subset_uuid).indicator_objects[indicator].get_ref_value_type(water_body = wb)
dw_obj = w.get_step_object(step = 3, subset = subset_uuid).indicator_objects[indicator]
dw_obj.set_water_body_indicator_df(water_body = wb)
dw_obj.get_water_body_indicator_df(wb)
dw_obj.column_list
temp = dw_obj.water_body_indicator_df[wb].dropna(subset = ['DIN', 'REFERENCE_VALUE']).copy(deep = True)
temp['ek'] = np.divide(temp.REFERENCE_VALUE,temp.DIN)
temp
#dw_obj.ref_settings.get_value(variable = 'HG_EQR_LIMIT', type_area = type_area)
#dw_obj.ref_settings.get_value(variable = 'PB_EQR_LIMIT', type_area = type_area)
#dw_obj.tolerance_settings.get_value(variable = 'MIN_NR_YEARS', type_area = '22')
#dw_obj.tolerance_settings.get_min_nr_years(type_area = '22')
#dw_obj.mapping_objects['water_body'].keys()
dw_obj.calculate_ek_value(water_body = wb)
dw_obj = w.get_step_object(step = 3, subset = subset_uuid).indicator_objects[indicator]
dw_obj.water_body_indicator_df[wb]
print(dw_obj.classification_results[wb].keys())
for key, value in dw_obj.classification_results[wb].items():
    if key == 'all_data':
        pass
    else:
        print('**************************************')
        print(key)
        print(value)

dw_obj.classification_results[wb]['mean_EQR_by_year']#.dropna(subset = ['mean_ek_value'])
dw_obj.water_body_indicator_df[wb].dropna(subset = ['REFERENCE_VALUE'])
def get_EK(x):
    y = x.DIN/x.REFERENCE_VALUE
    if y > 1:
        return 1
    else:
        return y

df = dw_obj.water_body_indicator_df[wb]
df['ek_value'] = df.apply(get_EK, axis = 1)
df[dw_obj.indicator_parameter]/df.REFERENCE_VALUE
by_date = df.groupby(['SDATE', 'YEAR'],).ek_value.agg(['count', 'min', 'max', 'mean']).reset_index()
# by_date.to_csv(self.paths['results'] +'/' + self.name + water_body +'by_occation.txt', sep='\t')
by_date.rename(columns={'mean':'mean_ek_value', 'count': 'number_of_values'}, inplace=True) # Cant use "mean" below
by_date
# Remove occations with not enough samples
# Or use count as a flag for what to display for the user?
by_date['all_ok'] = True
ix = by_date.loc[by_date['number_of_values'] < 1, 'all_ok'].index
by_date.set_value(ix, 'all_ok', False)
"""
2) Medelvärdet av EK för varje parameter beräknas för varje år.
"""
by_year = by_date.groupby('YEAR').mean_ek_value.agg(['count', 'min', 'max', 'mean']).reset_index()
by_year.rename(columns={'mean':'mean_ek_value', 'count': 'number_of_dates'}, inplace=True)
by_year['all_ok'] = True
by_year.loc[by_year['number_of_dates'] < 1, 'all_ok'] = False
# by_year.to_csv(self.paths['results'] +'/' + self.name + water_body + 'by_year.txt', sep='\t')
by_year
by_period = by_year[['mean_ek_value']].describe()
by_period
"""
3) Medelvärdet av EK för varje parameter och vattenförekomst (beräknas för minst
en treårsperiod)
"""
by_period = by_year[['mean_ek_value']].describe()#.agg(['count', 'min', 'max', 'mean'])
by_period = by_period.transpose()
#by_period#.loc['mean', 'mean_ek_value']
#
#by_period['count'].get_value('mean_ek_value')
by_period['all_ok']  = True
if by_period['count'].get_value('mean_ek_value') < 3:
    by_period['all_ok'] = False

by_period
by_period['mean'].get_value('mean_ek_value')
temp_df = pd.DataFrame(np.random.randn(10, 3), columns=['A', 'B', 'C'],
...                   index=pd.date_range('1/1/2000', periods=10))
temp_df.iloc[3:7] = np.nan
temp_df.describe()
def set_above_one_value(x):
    if x > 1:
        return 1
    else:
        return x
dw_obj.water_body_indicator_df[wb]['EK'] = dw_obj.water_body_indicator_df[wb]['DIN']/dw_obj.water_body_indicator_df[wb]['REFERENCE_VALUE']
dw_obj.water_body_indicator_df[wb]['EK'] = dw_obj.water_body_indicator_df[wb]['EK'].apply(set_above_one_value)
dw_obj.water_body_indicator_df[wb]['EK']
dw_obj.get_filtered_data(subset = subset_uuid, step = 'step_2', type_area = 22, indicator = 'din_winter')[['DIN']].dropna()
B2_NTOT_WINTER_SETTINGS = lv_workspace.get_subset_object('B').get_step_object('step_2').indicator_ref_settings['ntot_winter']
lv_workspace.get_subset_object('B').get_step_object('step_2').indicator_ref_settings['ntot_winter'].allowed_variables
# gör om till
# lv_workspace.get_indicator_ref_settings(step = , subset = , indicator = , waterbody/type)
# ger samma resultat som:
#lv_workspace.get_subset_object('B').get_step_object('step_2').indicator_ref_settings['ntot_winter'].settings.ref_columns

lv_workspace.get_subset_object('B').get_step_object('step_2').indicator_ref_settings['ntot_winter'].settings.get_value('EK G/M', 22)
#print(B2_NTOT_WINTER_SETTINGS)
#B2_NTOT_WINTER_SETTINGS.get_value('2', 'DEPTH_INTERVAL')
av = lv_workspace.get_subset_object('B').get_step_object('step_2').indicator_data_filter_settings['ntot_winter'].allowed_variables
lv_workspace.get_subset_object('B').get_step_object('step_2').indicator_data_filter_settings['ntot_winter'].settings.df[av]
lv_workspace.get_subset_object('B').get_step_object('step_2').indicator_data_filter_settings['ntot_winter'].settings.df

B2_NTOT_WINTER_SETTINGS.settings.mapping_water_body['N m Bottenvikens kustvatten']<jupyter_output><empty_output><jupyter_text>Set subset time and area filter<jupyter_code>f1_A = lv_workspace.get_data_filter_object(step=1, subset='A') 
f1_A.include_list_filter
lv_workspace.get_data_filter_info(step=1, subset='A') 
f1_A.exclude_list_filter
f0.include_list_filter<jupyter_output><empty_output><jupyter_text>Apply subset filter<jupyter_code>lv_workspace.apply_subset_filter(subset='A') # Not handled properly by the IndexHandler<jupyter_output><empty_output><jupyter_text>Extract filtered data<jupyter_code>data_after_subset_filter = lv_workspace.get_filtered_data(level=1, subset='A') # level=0 means first filter 
print('{} rows mathing the filter criteria'.format(len(data_after_subset_filter)))
data_after_subset_filter.head()
# show available waterbodies
lst = data_after_subset_filter.SEA_AREA_NAME.unique()
print('Waterbodies in subset:\n{}'.format('\n'.join(lst)))
import numpy as np
np.where(lv_workspace.index_handler.subset_filter)
f = lv_workspace.get_data_filter_object(step=1, subset='A') 

f.all_filters
f.exclude_list_filter
f.include_list_filter
s = lv_workspace.get_step_1_object('A')
s.data_filter.all_filters
f0 = lv_workspace.get_data_filter_object(step=0) 
f0.exclude_list_filter
f0.include_list_filter<jupyter_output><empty_output><jupyter_text>Quality factor Nutrients<jupyter_code>lv_workspace.initiate_quality_factors()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1367713004484305, 'fraction_numerical': 0.05170964125560538, 'mean_word_length': 7.756441717791411, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '337618', 'license': 'MIT', 'n_tokens_mistral': 5927, 'n_tokens_neox': 5409, 'n_words': 844, 'path': 'notebooks/lv_notebook_Indicator-Copy1.ipynb', 'repo_name': 'lvikt/ekostat_calculator', 'size': '14272'}