<jupyter_start><jupyter_text>Load directories<jupyter_code>root_directory = 'D:/github/w_vattenstatus/ekostat_calculator'#"../" #os.getcwd()
workspace_directory = root_directory + '/workspaces' 
resource_directory = root_directory + '/resources'

user_id = 'test_user'<jupyter_output><empty_output><jupyter_text>Initiate EventHandler<jupyter_code>print(root_directory)
paths = {'user_id': user_id, 
         'workspace_directory': root_directory + '/workspaces', 
         'resource_directory': root_directory + '/resources', 
         'log_directory': 'D:/github' + '/log', 
         'test_data_directory': 'D:/github' + '/test_data',
         'cache_directory': 'D:/github/w_vattenstatus/cache'}
t0 = time.time()
ekos = EventHandler(**paths)
print('-'*50)
print('Time for request: {}'.format(time.time()-t0))<jupyter_output>2018-12-12 10:30:41,530	logger.py	85	add_log	DEBUG	
2018-12-12 10:30:41,530	logger.py	86	add_log	DEBUG	========================================================================================================================
2018-12-12 10:30:41,530	logger.py	87	add_log	DEBUG	### Log added for log_id "event_handler" at locaton: D:\github\log\main_event_handler.log
2018-12-12 10:30:41,530	logger.py	88	add_log	DEBUG	------------------------------------------------------------------------------------------------------------------------
2018-12-12 10:30:41,530	event_handler.py	117	__init__	DEBUG	Start EventHandler: event_handler
2018-12-12 10:30:41,531	event_handler.py	157	_load_mapping_objects	DEBUG	Loading mapping files from original files.<jupyter_text>Load existing workspace<jupyter_code>workspace_alias = 'kustzon_selection'
ekos.print_workspaces()
workspace_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias)
print(workspace_uuid)
workspace_alias = ekos.get_alias_for_unique_id(workspace_uuid = workspace_uuid)
ekos.load_workspace(unique_id = workspace_uuid)<jupyter_output>2018-12-12 10:31:02,098	event_handler.py	3071	load_workspace	DEBUG	Trying to load new workspace "62b3b0e7-67b8-41f6-b8c1-8e50073480b9" with alias "kustzon_selection"
2018-12-12 10:31:02,235	logger.py	85	add_log	DEBUG	
2018-12-12 10:31:02,235	logger.py	86	add_log	DEBUG	========================================================================================================================
2018-12-12 10:31:02,250	logger.py	87	add_log	DEBUG	### Log added for log_id "7512aff0-aa3a-44c4-83d1-cb24596c4d50" at locaton: D:\github\w_vattenstatus\ekostat_calculator\workspaces\62b3b0e7-67b8-41f6-b8c1-8e50073480b9\log\subset_7512aff0-aa3a-44c4-83d1-cb24596c4d50.log
2018-12-12 10:31:02,250	logger.py	88	add_log	DEBUG	------------------------------------------------------------------------------------------------------------------------
2018-12-12 10:31:02,385	logger.py	85	add_log	DEBUG	
2018-12-12 10:31:02,385	logger.py	86	add_log	DEBUG	================================================================[...]<jupyter_text>QUALITY ELEMENTS<jupyter_code>w = ekos.get_workspace(workspace_uuid = workspace_uuid)
len(w.data_handler.get_all_column_data_df())
subset_alias = 'SE1_selection'
subset_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias, subset_alias = subset_alias)
w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(quality_element = 'nutrients_sw')
w.get_step_object(step = 3, subset = subset_uuid).quality_element['Nutrients summer-winter'].results.head()
w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(subset_unique_id = subset_uuid, quality_element = 'Phytoplankton', class_name = 'QualityElementPhytoplankton')
merge_on = ['VISS_EU_CD', 'WATER_BODY_NAME', 'WATER_TYPE_AREA']
indicator_name = 'qe_nutrients'
# current
df = w.get_step_object(step = 3, subset = subset_uuid).quality_element['Nutrients'].sld.load_df(indicator_name)
kolumner = df.columns
kolumner
import re
these_cols = [col for col in kolumner if re.search(indicator_name + r'$', col)]
df2 = df[these_cols + merge_on]
df2.rename(columns = {col: col.strip(indicator_name) for col in these_cols})
df
{col: col.strip(indicator_name) for col in these_cols}
tolset = w.get_step_object(step = 2, subset = subset_uuid).get_indicator_tolerance_settings('indicator_din_winter')
boolean_list = df['STATUS'] == 'HIGH'
df.loc[boolean_list, 'STATUS'] = 'HIGH2'
w.get_step_object(step = 3, subset = subset_uuid).quality_element['Nutrients'].results[['VISS_EU_CD','WATER_BODY_NAME','WATER_TYPE_AREA','STATUS_NUTRIENTS','mean_EQR','MEAN_N_EQR','EQR_N_winter_mean','global_EQR_ntot_summer','MEAN_P_EQR','EQR_P_winter_mean','global_EQR_ptot_summer'
]].to_csv('D:/Nutrients'+subset_alias+'.txt', float_format='%.3f', header = True, index = None, sep = '\t')<jupyter_output><empty_output><jupyter_text>Load all data in workspace<jupyter_code>#ekos.get_workspace(unique_id = workspace_uuid, alias = workspace_alias).delete_alldata_export()
#%%timeit
ekos.load_data(workspace_uuid = workspace_uuid)
print('subsetlist', w.get_subset_list())<jupyter_output><empty_output><jupyter_text>Step 0<jupyter_code>w.data_handler.all_data.head()<jupyter_output><empty_output><jupyter_text>Apply first data filter<jupyter_code>w.apply_data_filter(step = 0) # This sets the first level of data filter in the IndexHandler<jupyter_output><empty_output><jupyter_text>Step 1  Set subset filter<jupyter_code>#w.copy_subset(source_uuid='default_subset', target_alias='period_2007-2012_refvalues_2017')
subset_alias = 'period_2007-2012_refvalues_2017'
subset_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias, subset_alias = subset_alias)
w.set_data_filter(subset = subset_uuid, step=1, 
                         filter_type='include_list', 
                         filter_name='MYEAR', 
                         data=['2007', '2008', '2009', '2010', '2011', '2012']) 
w.set_data_filter(subset = subset_uuid, step=1, 
                         filter_type='include_list', 
                         filter_name='viss_eu_cd', data = ['SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655'])
                         #data=['SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655']) 
                         #wb with no data for din 'SE591400-182320'


f1 = w.get_data_filter_object(subset = subset_uuid, step=1) 
print(f1.include_list_filter)
print('subset_alias:', subset_alias, '\nsubset uuid:', subset_uuid)
f1 = w.get_data_filter_object(subset = subset_uuid, step=1) 
print(f1.include_list_filter)<jupyter_output><empty_output><jupyter_text>Apply step 1 datafilter to subset<jupyter_code>w.apply_data_filter(subset = subset_uuid, step = 1)<jupyter_output><empty_output><jupyter_text>Step 2 Load indicator settings filter<jupyter_code>w.get_step_object(step = 2, subset = subset_uuid).load_indicator_settings_filters()<jupyter_output><empty_output><jupyter_text>set available indicators<jupyter_code>w.get_available_indicators(subset= subset_uuid, step=2)<jupyter_output><empty_output><jupyter_text>Apply indicator data filter<jupyter_code>#list(zip(typeA_list, df_step1.WATER_TYPE_AREA.unique()))
#indicator_list = w.get_available_indicators(subset= subset_uuid, step=2)
indicator_list = ['oxygen','din_winter','ntot_summer', 'ntot_winter', 'dip_winter', 'ptot_summer', 'ptot_winter','bqi', 'biov', 'chl', 'secchi']
#indicator_list = ['din_winter','ntot_summer', 'ntot_winter', 'dip_winter', 'ptot_summer', 'ptot_winter']
#indicator_list = ['biov', 'chl']
#indicator_list = ['bqi', 'secchi']
#indicator_list = ['bqi', 'secchi'] + ['biov', 'chl'] + ['din_winter']
indicator_list.remove('din_winter')
# indicator_list = ['dip_winter', 'ntot_winter', 'ptot_winter']
indicator_list = ['indicator_' + indicator for indicator in indicator_list]

print('apply indicator data filter to {}'.format(indicator_list))
for indicator in indicator_list:
    w.apply_indicator_data_filter(step = 2, 
                          subset = subset_uuid, 
                          indicator = indicator)#,
                        # water_body_list = test_wb)
    #print(w.mapping_objects['water_body'][wb])
    #print('*************************************')

#df = w.get_filtered_data(subset = subset_uuid, step = 'step_2', water_body = 'SE625180-181655', indicator = 'indicator_din_winter').dropna(subset = ['DIN'])<jupyter_output><empty_output><jupyter_text>Step 3  Set up indicator objects<jupyter_code>w.get_step_object(step = 3, subset = subset_uuid).indicator_setup(subset_unique_id = subset_uuid, indicator_list = indicator_list) 
def concat_df(df, save_df, filename, water_body, indicator_object):
            #concatenate results
            if type(save_df) is pd.DataFrame:
                save_df = pd.concat([save_df, df])
            elif os.path.exists(indicator_object.result_directory + filename + '.txt'):
                save_df = indicator_object.sld.load_df(file_name = filename)
                
                save_df = pd.concat([save_df, df])
            else:
                save_df = df
            #save_df['new_index'] = [str(ix) +'_' + wb for ix, ind, wb in zip(save_df.index, indicator_object.name, save_df.VISS_EU_CD)]
            #save_df.set_index(keys = 'new_index')
            return save_df
w.get_step_object(step = 3, subset = subset_uuid).indicator_objects['indicator_biov'].\
    get_water_body_indicator_df(water_body = 'SE584340-174401')

save_df = False
for ind in indicator_list:
    for wb in ['SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655']:
        ind_obj = w.get_step_object(step = 3, subset = subset_uuid).indicator_objects[ind]
        df = ind_obj.get_water_body_indicator_df(water_body = wb)
        save_df = concat_df(df, save_df, 'test', wb, ind_obj)
save_df['period'] = str(np.min(save_df['YEAR'])) +'-'+ str(np.max(save_df['YEAR']))
save_df.head()
str(np.min(save_df['YEAR'])) +'-'+ str(np.max(save_df['YEAR']))
save_df.to_csv('D:/github/ekostat_calcR-master/data/'+subset_alias+'.txt', float_format='%.3f', header = True, index = None, sep = '\t')<jupyter_output><empty_output><jupyter_text>CALCULATE STATUS<jupyter_code>#w.get_step_object(step = 3, subset = subset_uuid).calculate_status(indicator_list = indicator_list)<jupyter_output><empty_output><jupyter_text>CALCULATE QUALITY ELEMENTS<jupyter_code>subset_alias<jupyter_output><empty_output><jupyter_text>Plotting results<jupyter_code>#%matplotlib inline
#import seaborn as sns
#for name, group in ind_obj.classification_results['status_by_date'].groupby('VISS_EU_CD'):
    #group['date'] = pd.to_datetime(group.SDATE)
    #group.dropna(subset = ['date', 'DIN'])
    #sns.tsplot(data = group.to_dict(), time = 'SDATE', value = 'DIN', condition = 'STATUS', legend = True)
#    group.plot('SDATE', ['DIN', 'REFERENCE_VALUE'], title = name + group.WATER_TYPE_AREA.values[0], marker ='*')
#name + group.WATER_TYPE_AREA.values[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18003842283414143, 'fraction_numerical': 0.05891501235019669, 'mean_word_length': 5.197278911564626, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '180252', 'license': 'MIT', 'n_tokens_mistral': 4323, 'n_tokens_neox': 3848, 'n_words': 590, 'path': 'notebooks/.ipynb_checkpoints/lv_notebook_calculate_qualityelements-checkpoint.ipynb', 'repo_name': 'lvikt/ekostat_calculator', 'size': '10931'}