<jupyter_start><jupyter_text>Exploring data by pollutant<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

%matplotlib inline
path_raw_original = '../data/raw/original/'
filename_pollutants = 'Measurement_item_info.csv'
filename_instruments = 'Measurement_info.csv'

df_pollutants = pd.read_csv(path_raw_original + filename_pollutants)
df = pd.read_csv(path_raw_original + filename_instruments)

df_pollutants.head(6)
# replace Item code with Item name & shorten column names

pollutants_dict = pd.Series(df_pollutants['Item name'].values,index=df_pollutants['Item code']).to_dict()
df['Item code'] = df['Item code'].replace(pollutants_dict)
df.rename(columns={"Measurement date": "datetime",
                   "Station code": "station",
                   "Item code": "pollutant",
                  "Average value": "value",
                  "Instrument status": "operational"}, inplace = True)
df['datetime'] = pd.to_datetime(df['datetime'])
df
# Are there any NaNs?

df.isnull().values.any()
# we will want to delete data where values are negative

df = df[(df['value']>0)]
df
# split dataframes by pollutant
# within pollutants split by instrument status code and pickle

pollutants = {}

for p in ['SO2', 'NO2', 'CO', 'O3', 'PM10', 'PM2.5']:
    sub = df.loc[df['pollutant'] == p]
    pollutants[p]={}
    pollutants[p]['all'] = sub
    pollutants[p]['not0'] = sub.loc[sub['operational'] != 0]
    for c in [0,1,2,4,8,9]:
        pollutants[p][c] = sub.loc[sub['operational'] == c]
    
    for k in pollutants[p].keys():
        pollutants[p][k].to_pickle('../data/processed/{}_{}.pkl'.format(p,k))<jupyter_output><empty_output><jupyter_text>Plotting some example pollutants and instrument status codes<jupyter_code>pollutants['SO2'][0].plot.scatter(x='datetime', y='value', figsize=(12, 6),
                                  title='SO2 at normal instrument operation accross all stations')
ax1 = pollutants['SO2'][0].plot(kind='scatter',x='datetime', y='value',
                                figsize=(12, 6), color='grey', label='0 - normal operation',
                                title='Non-normal operational data points for SO2 pollutant')
ax2 = pollutants['SO2'][9].plot(kind='scatter',x='datetime', y='value',
                                figsize=(12, 6), color='r', ax=ax1, label='9 - abnormal data')
ax3 = pollutants['SO2'][8].plot(kind='scatter',x='datetime', y='value',
                                figsize=(12, 6), color='g', ax=ax1, label='8 - under repair')
ax4 = pollutants['SO2'][4].plot(kind='scatter',x='datetime', y='value',
                                figsize=(12, 6), color='y', ax=ax1, label='4 - power cut off')
ax5 = pollutants['SO2'][2].plot(kind='scatter',x='datetime', y='value',
                                figsize=(12, 6), color='orange', ax=ax1, label='2 - abnormal')

# let's zoom in on the y-range that is important

ax1 = pollutants['SO2'][0].plot(kind='scatter',x='datetime', y='value', ylim=(0,0.4),
                                figsize=(12, 6), color='grey', label='0 - normal operation',
                                title='Non-normal operational data points for SO2 pollutant')
ax2 = pollutants['SO2'][9].plot(kind='scatter',x='datetime', y='value',
                                figsize=(12, 6), color='r', ax=ax1, label='9 - abnormal data')
ax3 = pollutants['SO2'][8].plot(kind='scatter',x='datetime', y='value',
                                figsize=(12, 6), color='g', ax=ax1, label='8 - under repair')
ax4 = pollutants['SO2'][4].plot(kind='scatter',x='datetime', y='value',
                                figsize=(12, 6), color='y', ax=ax1, label='4 - power cut off')
ax5 = pollutants['SO2'][2].plot(kind='scatter',x='datetime', y='value',
                                figsize=(12, 6), color='orange', ax=ax1, label='2 - abnormal')

pollutants['SO2']['all'].operational.value_counts(normalize=True) * 100
pollutants['SO2']['all'].describe()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1758076634109692, 'fraction_numerical': 0.02779864763335838, 'mean_word_length': 3.2854077253218885, 'pattern_counts': {'":': 5, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '540655', 'license': 'MIT', 'n_tokens_mistral': 1353, 'n_tokens_neox': 1275, 'n_words': 272, 'path': 'notebooks/explore_data_by_pollutants.ipynb', 'repo_name': 'ggantos/atmos', 'size': '3993'}