<jupyter_start><jupyter_text>install requirements import boto3import osfrom botocore import UNSIGNEDfrom botocore.config import Config bucket name is h4g-hague4th and folder for challenges is added in Prefix='H4G_Hague_4thEdition/'def sync_s3_folder(bucket_name, folder, destination_path):    s3 = boto3.resource('s3', config=Config(signature_version=UNSIGNED))    your_bucket = s3.Bucket(bucket_name)    print(os.path.join(destination_path, folder).replace("/", "\\"))    for s3_file in your_bucket.objects.all():        if folder in s3_file.key:            file=os.path.join(destination_path, s3_file.key.replace("/", "\\"))            print(file).replace('\\', '/'))            if not os.path.exists(os.path.dirname(file)):                os.makedirs(os.path.dirname(file))            if file.rstrip('\\') != os.path.join(destination_path, folder).replace("/", "\\"):                your_bucket.download_file(s3_file.key, file)sync_s3_folder('h4g-hague4th', 'H4G_Hague_4thEdition/Water_and_Circular_greenhouse_economy/', 'h4g-hague4th')<jupyter_code>import os
os.getcwd()
import pandas as pd
os.chdir('h4g-hague4th/H4G_Hague_4thEdition\Water_and_Circular_greenhouse_economy\IM_Metingen_2020/')
pd.options.display.max_columns = None
#pd.set_option('display.max_rows', None)
data1.head()
data1.pivot_table?
data1 = pd.read_csv('IM-Metingen_2020_1_mnd01tm-04.csv', sep=';', low_memory=False)
#data2 = pd.read_csv('IM-Metingen_2020_3_mnd08tm-09.csv', sep=';', low_memory=False)
#data3 = pd.read_csv('IM-Metingen_2020_2_mnd05tm-07.csv', sep=';', low_memory=False)
#data4 = pd.read_csv('IM-Metingen_2020_4_mnd10tm-12.csv', sep=';', low_memory=False)
data1.head()
pd.concat?
pd.merge?
pd.DataFrame(data1["Parameter.omschrijving"].value_counts()[:50]).reset_index()
subset_params = pd.DataFrame(data1["Parameter.omschrijving"].value_counts()[:25].reset_index()["index"])
subset_params.columns=["Parameter.omschrijving"]
top25 = pd.merge(left=data1,right=subset_params,how="right")
top25
top25[(top25["Parameter.omschrijving"]=="nitraat")]["Numeriekewaarde"]
pd.DataFrame.plot.hist?
top25[(top25["Parameter.omschrijving"]=="nitraat")]["Numeriekewaarde"].plot.hist(grid=True,bins=100,rwidth=0.9)
pvt25 = top25.pivot_table(index="Monster.Identificatie",columns="Parameter.omschrijving", values="Numeriekewaarde")
pvt25.describe(include="all")
corr25 = pvt25.corr()
corr25
sns.heatmap?
import matplotlib.pyplot as plt
plt.figure(figsize = (25,25))

import seaborn as sns
import numpy as np
mask = np.triu(np.ones_like(corr25, dtype=bool))
# plot the heatmap
sns.heatmap(corr25, mask = mask,
        xticklabels=corr25.columns,
        yticklabels=corr25.columns)
pvt25.head()
cmap=sns.diverging_palette(5, 250, as_cmap=True)

def magnify():
    return [dict(selector="th",
                 props=[("font-size", "7pt")]),
            dict(selector="td",
                 props=[('padding', "0em 0em")]),
            dict(selector="th:hover",
                 props=[("font-size", "12pt")]),
            dict(selector="tr:hover td:hover",
                 props=[('max-width', '200px'),
                        ('font-size', '12pt')])
]

corr25.style.background_gradient(cmap, axis=1)\
    .set_properties(**{'max-width': '80px', 'font-size': '10pt'})\
    .set_caption("Hover to magify")\
    .set_precision(2)\
    .set_table_styles(magnify())
bio_data = pd.read_csv('/home/u749352/h4g-hague4th/Biologische_monsterkenmerken_HHSK.csv', sep=';', low_memory=False, encoding = "ISO-8859-1",)
data1.head()
data1.describe()
data1.columns.tolist()
data1.columns = ['Wassermanager.Beschreibung',
 'Messobjekt.Namespace',
 'Messobjekt.code',
 'Messobjekt.localID',
 'Namensraum',
 'Sample.Identifier',
 'Sample.localID',
 'SampleCompartment.code',
 'Probenfach.Beschreibung',
 'Organ.code',
 'Orgel.Beschreibung',
 'organismus.name',
 'sampler.code',
 'Proben.Beschreibung',
 'Beispiel-Abholdatum',
 'Abholzeit der Probe',
 'GeometriePunkt.X',
 'GeometriePunkt.Y',
 'mess.localID',
 'Ergebnisdatum',
 'Anfangsdatum',
 'Startzeit',
 'Endtermin',
 'Endzeit',
 'typing.code',
 'Eingabe.Beschreibung',
 'Größe.code',
 'Betrag.Beschreibung',
 'Parametergruppe',
 'Parameter.code',
 'Parameter.Beschreibung',
 'Parameter.CASnummer',
 'Biotaxon.name',
 'Einheitencode',
 'Einheit.Beschreibung',
 'Kapazitätscode',
 'Kapazitätsbeschreibung',
 'Lifestage.code',
 'Lebensstadium.Beschreibung',
 'Längenklassen.code',
 'Längenklasse.Beschreibung',
 'geschlechter.code',
 'Geschlecht.Beschreibung',
 'Aussehen.Code',
 'Aussehen.Beschreibung',
 'Lebensform.code',
 'Lebensform.Beschreibung',
 'Verhaltenscode',
 'Verhalten.Beschreibung',
 'AnalysisCompartment.code',
 'AnalysisCompartment.description',
 'Fach.Code',
 'Fach.Beschreibung',
 'Wert edit method.code',
 'Wertbearbeitungsmethode.Beschreibung',
 'Grenzwertsymbol',
 'Zahlenwert',
 'Alphanumerischer Wert',
 'Qualitätsurteil.code',
 'Qualitätsurteil.Beschreibung',
 'Berichtsjahr']
test = pd.DataFrame(data1['Parameter.code'].value_counts())
data1 = pd.DataFrame(data1)
test = data1[['Parameter.code','Zahlenwert']][data1['Zahlenwert'] > 0]
data1[['Parameter.code','Zahlenwert']][data1['Zahlenwert'] > 0]
test = pd.DataFrame(test[['Parameter.code']].value_counts())
test.head()
test.to_csv("Häufigkeit_Chemie_noNull.csv")
data1.head(30)
test = pd.DataFrame(data1[['Parameter.code','Parameter.Beschreibung']])
data1[['Parameter.code','Parameter.Beschreibung']].set_index('Parameter.code').head()
data1[['Parameter.code','Parameter.Beschreibung']].set_index('Parameter.code').join(test,how='right',rsuffix="_").head()
data1.head()
data1.describe(include='all')
data1.shape
import numpy as np
data4.shape
data1.shape[0]+data2.shape[0]+data3.shape[0]+data4.shape[0]
bio_data.shape
bio_data.columns.tolist()
bio_data.columns = ['Messpunkt',
  'Ortsbeschreibung',
  'X',
  'J',
  'Datum',
  'Zeit',
  'Taxatyp',
  'Methodencode',
  'Methodenname',
  'Attributcode',
  'Attributname',
  'Attributwert',
  'class_name_orig']
df = bio_data[bio_data["Methodencode"]=="VEGSTO"]
df.head()
df1 = df[["Messpunkt","Attributcode","Attributname","Attributwert"]]
df1.head()
df.shape
df1[40:58]
df1[40:58].T
pd.DataFrame.pivot?
df1[0:55].pivot(index='Messpunkt', columns=["Attributname","Attributwert"], values="Attributwert")
bio_data.describe(include='all')
bio_data.groupby('Attributcode').count()
bio_data['Attributcode'].value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1772449459332393, 'fraction_numerical': 0.031969910672308414, 'mean_word_length': 5.959651035986914, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '337846', 'license': 'MIT', 'n_tokens_mistral': 2538, 'n_tokens_neox': 2468, 'n_words': 300, 'path': 'code/HackatonForGood.ipynb', 'repo_name': 'lookaheads/hackathon4good2021', 'size': '6381'}