<jupyter_start><jupyter_text>AQI data is collected from a paid third party app , it have AQI of 2013 to 2018 for every hourwe have take PM 2.5 feature from it Also, the data we have collected is on daily basis, so we have to convert data from hours to day we have to combine this data and data from html into one csv file We have to handle the missing values , invalid data and power fails<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
def avg_data_2013():
    temp_i=0
    average=[]
    for rows in pd.read_csv('Data/AQI/aqi2013.csv',chunksize=24): 
        #chunksize=24 -> represents how many records we are taking at a time, here 24 because of 24 hours a day.
        add_var=0
        avg=0.0
        data=[]
        df=pd.DataFrame(data=rows) #converting list to dataframe
        for index,row in df.iterrows(): #iterating through rows
            data.append(row['PM2.5']) #taking/appending pm2.5 data to data
        #now data have all the values in pm2.5, also it have missing values and invalid datas
        for i in data:
            if type(i) is float or type(i) is int:
                add_var=add_var+i
            elif type(i) is str:
                if i!='NoData' and i!='PwrFail' and i!='---' and i!='InVld':
                    temp=float(i)
                    add_var=add_var+temp
        avg=add_var/24
        temp_i=temp_i+1
        average.append(avg)
    return average

def avg_data_2014():
    temp_i=0
    average=[]
    for rows in pd.read_csv('Data/AQI/aqi2014.csv',chunksize=24): 
        #chunksize=24 -> represents how many records we are taking at a time, here 24 because of 24 hours a day.
        add_var=0
        avg=0.0
        data=[]
        df=pd.DataFrame(data=rows) #converting list to dataframe
        for index,row in df.iterrows(): #iterating through rows
            data.append(row['PM2.5']) #taking/appending pm2.5 data to data
        #now data have all the values in pm2.5, also it have missing values and invalid datas
        for i in data:
            if type(i) is float or type(i) is int:
                add_var=add_var+i
            elif type(i) is str:
                if i!='NoData' and i!='PwrFail' and i!='---' and i!='InVld':
                    temp=float(i)
                    add_var=add_var+temp
        avg=add_var/24
        temp_i=temp_i+1
        average.append(avg)
    return average

def avg_data_2015():
    temp_i=0
    average=[]
    for rows in pd.read_csv('Data/AQI/aqi2015.csv',chunksize=24): 
        #chunksize=24 -> represents how many records we are taking at a time, here 24 because of 24 hours a day.
        add_var=0
        avg=0.0
        data=[]
        df=pd.DataFrame(data=rows) #converting list to dataframe
        for index,row in df.iterrows(): #iterating through rows
            data.append(row['PM2.5']) #taking/appending pm2.5 data to data
        #now data have all the values in pm2.5, also it have missing values and invalid datas
        for i in data:
            if type(i) is float or type(i) is int:
                add_var=add_var+i
            elif type(i) is str:
                if i!='NoData' and i!='PwrFail' and i!='---' and i!='InVld':
                    temp=float(i)
                    add_var=add_var+temp
        avg=add_var/24
        temp_i=temp_i+1
        average.append(avg)
    return average

def avg_data_2016():
    temp_i=0
    average=[]
    for rows in pd.read_csv('Data/AQI/aqi2016.csv',chunksize=24): 
        #chunksize=24 -> represents how many records we are taking at a time, here 24 because of 24 hours a day.
        add_var=0
        avg=0.0
        data=[]
        df=pd.DataFrame(data=rows) #converting list to dataframe
        for index,row in df.iterrows(): #iterating through rows
            data.append(row['PM2.5']) #taking/appending pm2.5 data to data
        #now data have all the values in pm2.5, also it have missing values and invalid datas
        for i in data:
            if type(i) is float or type(i) is int:
                add_var=add_var+i
            elif type(i) is str:
                if i!='NoData' and i!='PwrFail' and i!='---' and i!='InVld':
                    temp=float(i)
                    add_var=add_var+temp
        avg=add_var/24
        temp_i=temp_i+1
        average.append(avg)
    return average

def avg_data_2017():
    temp_i=0
    average=[]
    for rows in pd.read_csv('Data/AQI/aqi2017.csv',chunksize=24): 
        #chunksize=24 -> represents how many records we are taking at a time, here 24 because of 24 hours a day.
        add_var=0
        avg=0.0
        data=[]
        df=pd.DataFrame(data=rows) #converting list to dataframe
        for index,row in df.iterrows(): #iterating through rows
            data.append(row['PM2.5']) #taking/appending pm2.5 data to data
        #now data have all the values in pm2.5, also it have missing values and invalid datas
        for i in data:
            if type(i) is float or type(i) is int:
                add_var=add_var+i
            elif type(i) is str:
                if i!='NoData' and i!='PwrFail' and i!='---' and i!='InVld':
                    temp=float(i)
                    add_var=add_var+temp
        avg=add_var/24
        temp_i=temp_i+1
        average.append(avg)
    return average

def avg_data_2018():
    temp_i=0
    average=[]
    for rows in pd.read_csv('Data/AQI/aqi2018.csv',chunksize=24): 
        #chunksize=24 -> represents how many records we are taking at a time, here 24 because of 24 hours a day.
        add_var=0
        avg=0.0
        data=[]
        df=pd.DataFrame(data=rows) #converting list to dataframe
        for index,row in df.iterrows(): #iterating through rows
            data.append(row['PM2.5']) #taking/appending pm2.5 data to data
        #now data have all the values in pm2.5, also it have missing values and invalid datas
        for i in data:
            if type(i) is float or type(i) is int:
                add_var=add_var+i
            elif type(i) is str:
                if i!='NoData' and i!='PwrFail' and i!='---' and i!='InVld':
                    temp=float(i)
                    add_var=add_var+temp
        avg=add_var/24
        temp_i=temp_i+1
        average.append(avg)
    return average
if __name__=='__main__':
    lst2013=avg_data_2013()
    lst2014=avg_data_2014()
    lst2015=avg_data_2015()
    lst2016=avg_data_2016()
    lst2017=avg_data_2017()
    lst2018=avg_data_2018()
    plt.plot(range(0,365),lst2013,label='2013 data')
    plt.plot(range(0,364),lst2014,label='2014 data')
    plt.plot(range(0,365),lst2015,label='2015 data')
    #plt.plot(range(0,121),lst2016,label="2016 data")
    plt.xlabel('Day')
    plt.ylabel('PM 2.5')
    plt.legend(loc='upper right')
    plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10302583025830259, 'fraction_numerical': 0.04162361623616236, 'mean_word_length': 2.1516279069767443, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '617702', 'license': 'MIT', 'n_tokens_mistral': 2415, 'n_tokens_neox': 2186, 'n_words': 678, 'path': 'plot_AQI.ipynb', 'repo_name': 'sanchittapdiya/Air_Quality_Index_Prediction', 'size': '6775'}