<jupyter_start><jupyter_text>Data downloaded from Purple Air, around Milan, Italy and Walnut Creek, California. Selected a few sensor to average out and filtered out the *inside* sensors.North of Italyhttps://www.purpleair.com/map?opt=1/mAQI/a10/cC08.68/45.2634/8.8042https://www.purpleair.com/sensorlist?exclude=true&nwlat=45.787619691362494&selat=44.73429255594439&nwlng=7.160429912659254&selng=10.447989925169566Walnut Creek, Californiahttps://www.purpleair.com/map?opt=1/i/mAQI/a10/cC0&select=5351713.41/37.90262/-122.04889https://www.purpleair.com/sensorlist?exclude=true&nwlat=37.92492052460308&selat=37.88031769472657&nwlng=-122.11097636966711&selng=-121.98679844265672<jupyter_code>import os
import pandas as pd
import datetime
def AQI_calc(x) :
    if float(x) <= 50 :
        return 'Good'
    elif float(x) <= 100 :
        return 'Moderate'
    elif float(x) <= 150 : 
        return 'USG'
    elif float(x) <= 200 :
        return 'Unhealthy'
    elif float(x) <= 300 :
        return 'Very Unhealthy'
    else :
        return 'Hazardous'

# lat = 37
# long = -122
# city='concord'

lat = 45
long = 7.6
city='milano'


directory = '../data/'+city
my_data=pd.DataFrame(columns=['date','PM10'])

for filename in os.listdir(directory):
    if filename.endswith(".csv") :
        #rint(os.path.join(directory, filename))
        # open file import relative colums and append to dataframe
        with open(os.path.join(directory, filename), 'r') as file_object :
            my_data = my_data.append(pd.read_csv(file_object, usecols=[0,3], names=['date','PM10'],header=0))
    else:
        continue

my_data['date'] = my_data['date'].apply(lambda x : datetime.datetime.strptime(x, '%Y-%m-%d %H:%M:%S %Z').strftime("%d/%m/%Y"))
my_data.dropna(axis='rows', inplace=True)
my_grouped_data = my_data.groupby(by='date').mean()
my_grouped_data['AQI']=0
my_grouped_data['AQI'] = my_grouped_data['PM10'].apply(lambda x : AQI_calc(x))
my_grouped_data['lat']=lat
my_grouped_data['long']=long
my_grouped_data.to_csv('../data/'+city+'.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1595744680851064, 'fraction_numerical': 0.10203094777562863, 'mean_word_length': 4.561827956989247, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '544983', 'license': 'MIT', 'n_tokens_mistral': 928, 'n_tokens_neox': 783, 'n_words': 128, 'path': 'notebooks/clean_data.ipynb', 'repo_name': 'andrea-fantini/AQI_viz_data_processing', 'size': '2068'}