<jupyter_start><jupyter_text>Techniques to measure the chemical composition of the atmospheremeasurement of PM2  with R & P Model 2025 PM-2.5 Sequential Air Sampler w/VSCC - GravimetricPM10_total with HI-VOL SA/GMW-1200 - GRAVIMETRICPM25 with Met-one BAM-1020 W/PM2.5 SCC - Beta Attenuation (optical method)Pressure (mb) withINSTRUMENTAL - BAROMETRIC SENSORVOC_ppb_C = 6L Pressurized Canister - Precon Saturn GC/MSO3_ppb, 8-HR RUN AVG BEGIN HOURNO2 ppb, INSTRUMENTAL - CHEMILUMINESCENCElead (ng/m3), Lo-Vol-Xontech 920 or 924, Teflon - ICP/Mass SpectrometerCO ppb,  INSTRUMENTAL - Gas Filter Correlation Thermo Electron 48i-TLEAQI, R & P Model 2025 PM-2.5 Sequential Air Sampler w/VSCC - Gravimetric<jupyter_code># Files to download for further analysis
# Define relative path to files
file_path1 = os.path.join("data","output_figures", "sandiego_2014_fires", "SD_weather_aq2014" ,                       
                         "SD_temp.csv")
file_path2 = os.path.join("data","output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                        
                         "SD_pres.csv" )
file_path3 = os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                        
                         "SD_RH.csv")
file_path4 = os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                        
                         "SD_wind.csv")
file_path5 = os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                       
                         "SD_O3.csv")
file_path6 = os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                       
                         "SD_CO.csv")
file_path7 = os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                       
                         "SD_NO2.csv")
file_path8= os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                       
                         "SD_PM25.csv")
file_path9 = os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                       
                         "SD_PM10.csv")
file_path10 = os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                       
                         "SD_PM2.csv")
file_path11 = os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                       
                         "SD_lead.csv")
file_path12 = os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                       
                         "SD_VOC.csv")
file_path13 = os.path.join("data", "output_figures", "sandiego_2014_fires", "SD_weather_aq2014",                       
                         "SD_AQI.csv")

# To check if path is created
os.path.exists(file_path8)
sd_temp = pd.read_csv(file_path1, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_pres = pd.read_csv(file_path2, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_RH = pd.read_csv(file_path3, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_wind = pd.read_csv(file_path4, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_O3 = pd.read_csv(file_path5, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_CO = pd.read_csv(file_path6, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_NO2 = pd.read_csv(file_path7, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_PM25 = pd.read_csv(file_path8, parse_dates=['Date Local'],
                                index_col=['Date Local'])

sd_PM10 = pd.read_csv(file_path9, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_lead = pd.read_csv(file_path11, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_VOC = pd.read_csv(file_path12, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_AQI = pd.read_csv(file_path13, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_O3.head(3)
sd_O3['City Name'].unique()
sd_O3.columns
# frames = [sd_O3, sd_RH, sd_temp]
# O3_RH_t = pd.concat(frames, sort= False)
# result = df1.append([df2, df3])
# sd_combined_df = sd_O3.append([sd_RH, sd_temp, sd_pres, sd_wind, sd_CO, sd_NO2, sd_PM25], sort = False)
# sd_combined_df.set_index('Date Local', inplace=True)

# pd.DataFrame.to_csv(sd_combined_df)

sd_combined_df = pd.concat([sd_O3, sd_RH, sd_temp, sd_pres, sd_wind, sd_CO, sd_NO2], axis= 0, join = 'outer').sort_index()
# sd_combined_df.set_index('Date Local', inplace=True)
# finaldf = pd.concat([df1, df2, df3], axis=1, join='inner').sort_index()
# result.head(2)
sd_combined_df.to_csv('data/output_figures/sandiego_2014_fires/air_quality_csv/sd_cumulative_df.csv')
# print(sd_combined_df['City Name'].unique())
print(sd_combined_df.columns)
sd_combined_df.head()
# To check empty columns
sd_combined_df.isnull().any()
# # data_frames = [sd_temp, sd_O3, sd_NO2, sd_CO,]
# # df_merged = reduce(lambda  left,right: pd.merge(left,right,on=['Date Local'],
# #                                             how='outer'), data_frames)
# # df_merged.head()

# dfs= [sd_RH, sd_temp, sd_pres, sd_wind]
# def mergefiles(dfs, countfiles, i=0):
#     if i == (countfiles - 2): # it gets to the second to last and merges it with the last
#         return

#     dfm = dfs[i].merge(mergefiles(dfs[i+1], countfiles, i=i+1), on='Date Local')
#     return dfm

# print(mergefiles(dfs, len(dfs)))
# print(dfm.columns)
#pd.DataFrame.to_csv('data/output_figures/sandiego_2014_fires/air_quality_csv/weather_chem_sd_stations.csv')
file_path2 = os.path.join(base_path,"output_figures","sandiego_2014_fires", "air_quality_csv",                     
                         "sd_chemical_composition_2014_mean_values_only.csv")

# To check if path is created
os.path.exists(file_path2)
sd_atm_df = pd.read_csv(file_path2, parse_dates=['Date Local'],
                                index_col=['Date Local'])
sd_atm_df.head(2)
# sd_weather_2014_df.reset_index(inplace = True)
# sd_weather_2014_df.head()
# # Create plot space
# fig, (ax1, ax2, ax3, ax4, ax5) = plt.subplots(5, 
#                             sharex = True, figsize = (10, 12))
# # Add subtitle 
# fig.subplots_adjust(bottom=0.15, left=0.2)
# fig.suptitle(""" Figure 1(a-e). Atmospheric conditions during wildfires\n
#                 Mar - June 2014, San Diego, CA.""", fontsize = 20)

sd_atm_df.shape
# fig, (ax1, ax2) = plt.subplots(2, 1
#                             sharex = True, figsize = (10, 12))
sd_atm_df.plot(x='NO2_mean', y='CO_mean', style='o', c='b')  
plt.title('NO2 vs CO')  
plt.xlabel('NO2 (ppb)')  
plt.ylabel('CO (ppm)')  
plt.show()
sd_atm_df.plot(x='O3_ppb_mean', y='NO2_mean', style='o', c='r')  
plt.title('O3 vs NO2')  
plt.xlabel('O3 (ppb)')  
plt.ylabel('NO2 (pp)b')  
plt.show()
plt.figure(figsize=(4,4))
plt.tight_layout()
sns.distplot(sd_atm_df['NO2_mean'])<jupyter_output><empty_output><jupyter_text>Our next step is to divide the data into “attributes” and “labels”.Attributes are the independent variables while labels are dependent variables whose values are to be predicted. In our dataset, we only have two columns. We want to predict the MaxTemp depending upon the MinTemp recorded. Therefore our attribute set will consist of the “MinTemp” column which is stored in the X variable, and the label will be the “MaxTemp” column which is stored in y variable.<jupyter_code>sd_atm_df
X = sd_atm_df['NO2_mean'].values.reshape(-1,1)
y = sd_atm_df['CO_mean'].values.reshape(-1,1)<jupyter_output><empty_output><jupyter_text>Next, we split 80% of the data to the training set while 20% of the data to test set using below code.The test_size variable is where we actually specify the proportion of the test set.<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)<jupyter_output><empty_output><jupyter_text>After splitting the data into training and testing sets, finally, the time is to train our algorithm. For that, we need to import LinearRegression class, instantiate it, and call the fit() method along with our training data.As we have discussed that the linear regression model basically finds the best value for the intercept and slope, which results in a line that best fits the data. To see the value of the intercept and slope calculated by the linear regression algorithm for our dataset, execute the following code.<jupyter_code>regressor = LinearRegression()  
regressor.fit(X_train, y_train) #training the algorithm
#To retrieve the intercept:
print(regressor.intercept_)
#For retrieving the slope:
print(regressor.coef_)<jupyter_output>[0.09713389]
[[0.0292762]]<jupyter_text>This means that for every one unit of change in NO2_mean, the change in the CO is about 0.029%.Now that we have trained our algorithm, it’s time to make some predictions. To do so, we will use our test data and see how accurately our algorithm predicts the percentage score. To make predictions on the test data, execute the following script:<jupyter_code># print(X_test)
X_test.shape
y_pred = regressor.predict(X_test)<jupyter_output><empty_output><jupyter_text>Now compare the actual output values for X_test with the predicted values, execute the following script:<jupyter_code>df = pd.DataFrame({'Actual': y_test.flatten(), 'Predicted': y_pred.flatten()})
# df.head()
print(df)<jupyter_output>Actual  Predicted
0   39.496778  33.963768
1   29.229364  31.405086
2   24.700636  25.743098
3   23.405000  29.826982
4   37.794300  30.561945
..        ...        ...
68  38.900727  32.440135
69  38.552900  33.083343
70  36.794000  28.852245
71  29.982500  33.297434
72  36.659667  32.139480

[73 rows x 2 columns]<jupyter_text>We can also visualize comparison result as a bar graph using the below script :Note: As the number of records is huge, for representation purpose I’m taking just 15 records.<jupyter_code>df1 = df.head(19)
df1.plot(kind='bar',figsize=(8,5))
plt.grid(which='major', linestyle='-', linewidth='0.5', color='green')
plt.grid(which='minor', linestyle=':', linewidth='0.5', color='black')
plt.show()<jupyter_output><empty_output><jupyter_text>Though our model is not very precise, the predicted percentages are close to the actual ones.Let's plot our straight line with the test data :<jupyter_code>plt.scatter(X_test, y_test,  color='gray')
plt.plot(X_test, y_pred, color='red', linewidth=2)
plt.show()<jupyter_output><empty_output><jupyter_text>The straight line in the above graph shows our algorithm is correct.The final step is to evaluate the performance of the algorithm. This step is particularly important to compare how well different algorithms perform on a particular dataset. For regression algorithms, three evaluation metrics are commonly used:1. Mean Absolute Error (MAE) is the mean of the absolute value of the errors. It is calculated as:![image.png](attachment:image.png)2. Mean Squared Error (MSE) is the mean of the squared errors and is calculated as:![image.png](attachment:image.png)3. Root Mean Squared Error (RMSE) is the square root of the mean of the squared errors:![image.png](attachment:image.png)Luckily, we don’t have to perform these calculations manually. The Scikit-Learn library comes with pre-built functions that can be used to find out these values for us.Let’s find the values for these metrics using our test data.Mean Absolute Error (MAE) is the mean of the absolute value of the errors. It is calculated as:<jupyter_code>print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))  
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))<jupyter_output>Mean Absolute Error: 0.03790606258368175
Mean Squared Error: 0.0023326923930414834
Root Mean Squared Error: 0.04829795433599112<jupyter_text>Multiple Linear Regression![image.png](attachment:image.png)We just performed linear regression in the above section involving two variables. Almost all the real-world problems that you are going to encounter will have more than two variables. Linear regression involving multiple variables is called “multiple linear regression” or multivariate linear regression. The steps to perform multiple linear regression are almost similar to that of simple linear regression. The difference lies in the evaluation. You can use it to find out which factor has the highest impact on the predicted output and how different variables relate to each other.<jupyter_code># To check empty columns
sd_atm_df.isnull().any()<jupyter_output><empty_output><jupyter_text>Once the above code is executed, all the columns should give False, In case for any column you find True result, then remove all the null values from that column using below code.<jupyter_code>dataset = sd_atm_df.fillna(method='ffill')
dataset.describe()<jupyter_output><empty_output><jupyter_text>Our next step is to divide the data into “attributes” and “labels”. X variable contains all the attributes/features and y variable contains labels.<jupyter_code>X = dataset[[ 'NO2_mean','CO_mean', 'PM2.5_mean']].values
y = dataset['O3_ppb_mean'].values
plt.figure(figsize=(4,4))
plt.tight_layout()
sns.distplot(dataset['O3_ppb_mean'])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
regressor = LinearRegression()  
regressor.fit(X_train, y_train)

# regressor = LinearRegression()  
# regressor.fit(X_train, y_train) #training the algorithm
#To retrieve the intercept:
print(regressor.intercept_)
#For retrieving the slope:
print(regressor.coef_)
import pandas as pd  
import numpy as np  
import matplotlib.pyplot as plt  
import seaborn as seabornInstance 
from sklearn.model_selection import train_test_split 
from sklearn.linear_model import LinearRegression
from sklearn import metrics
%matplotlib inline
# coeff_df = pd.DataFrame(regressor.coef_, X.columns)  
# coeff_df
y_pred = regressor.predict(X_test)
df = pd.DataFrame({'Actual': y_test, 'Predicted': y_pred})
df.head()
df1.plot(kind='bar',figsize=(10,6))
plt.grid(which='major', linestyle='-', linewidth='0.5', color='green')
plt.grid(which='minor', linestyle=':', linewidth='0.5', color='black')
plt.show()
print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))  
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))<jupyter_output>Mean Absolute Error: 5.175979035211536
Mean Squared Error: 42.42135163302012
Root Mean Squared Error: 6.513167557572899
{'fraction_non_alphanumeric': 0.11125987414647208, 'fraction_numerical': 0.042642924086223055, 'mean_word_length': 3.869295958279009, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '50083', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 5393, 'n_tokens_neox': 4809, 'n_words': 1311, 'path': 'test_notebooks/test_notebooks_aq/SciKit_MLR.ipynb', 'repo_name': 'shaheen19/wildfires_causes_consequences', 'size': '14938'}