<jupyter_start><jupyter_text>Carbon analysis.   DATA DESCRIPTIVE ANALYSIS First import libraries<jupyter_code>import pandas as pd
import numpy as np
import seaborn as sns                       #visualisation
import matplotlib.pyplot as plt             #visualisation
%matplotlib inline     
sns.set(color_codes=True)
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_curve
from IPython.display import display, HTML
from sklearn.metrics import r2_score<jupyter_output><empty_output><jupyter_text>**Import** **data**<jupyter_code>df = pd.read_csv("Data.csv")
# To display the top 5 rows 
df.head(5)<jupyter_output><empty_output><jupyter_text>Checking the types of data<jupyter_code>df.dtypes<jupyter_output><empty_output><jupyter_text>**Dropping irrelevant columns**<jupyter_code>df=df.drop(['BLDG_PUBID','BLDG_STOR_A','LCA_YEAR','LCA_REFPERIOD','LCA_SOUR_CODE','LCA_STAGES','LCA_BLDG_SCOPE','LCA_MAT_Q','$BLDG_AREA_FT2' ],axis=1)
df.head(5)

df.shape<jupyter_output><empty_output><jupyter_text>**Dublicate Check**<jupyter_code>duplicate_rows_df = df[df.duplicated()]
print("number of duplicate rows: ", duplicate_rows_df.shape)<jupyter_output>number of duplicate rows:  (58, 10)<jupyter_text>Drop Dublicates<jupyter_code>df = df.drop_duplicates()
df.head(5)
df.count()<jupyter_output><empty_output><jupyter_text>Dropping the missing or null values.<jupyter_code>print(df.isnull().sum())
df = df.dropna()    # Dropping the missing values and zeros
df.count()
print(df.isnull().sum())   # After dropping the values<jupyter_output>BLDG_TYP           0
BLDG_US            0
BLDG_LOC_REGION    0
BLDG_NEW_REN       0
$BLDG_AREA_M2      0
$BLDG_STOR_A       0
EC_WB_EX_OPER      0
EC_LCAA_PERM2      0
Average_Area_m2    0
Average_Storey     0
dtype: int64<jupyter_text>**Detecting Outliers**<jupyter_code>sns.boxplot(x=df['EC_WB_EX_OPER'])
Q1 = df.quantile(0.25)
Q3 = df.quantile(0.75)
IQR = Q3 - Q1
print(IQR)<jupyter_output>EC_WB_EX_OPER      21576.9575
EC_LCAA_PERM2        286.3400
Average_Area_m2    66500.0000
Average_Storey        16.0000
dtype: float64<jupyter_text>Outliers elimination<jupyter_code>df = df[~((df < (Q1 - 1.5 * IQR)) |(df > (Q3 + 1.5 * IQR))).any(axis=1)]
df.shape<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: FutureWarning: Automatic reindexing on DataFrame vs Series comparisons is deprecated and will raise ValueError in a future version.  Do `left, right = left.align(right, axis=1, copy=False)` before e.g. `left == right`
  """Entry point for launching an IPython kernel.<jupyter_text>**Change Column Names **<jupyter_code>df = df.rename(columns={'$BLDG_AREA_M2':'BLDG_AREA_M2','$BLDG_AREA_FT2':'BLDG_AREA_FT2','$BLDG_STOR_A':'BLDG_STOR_A'})
df.head(5)<jupyter_output><empty_output><jupyter_text>Histogram---------<jupyter_code>df.BLDG_AREA_M2.value_counts().plot(kind='bar', figsize=(10,5))
plt.title("Building Area Distribution")
plt.ylabel('Numbers')
plt.xlabel('Area (m2)');
df.BLDG_STOR_A.value_counts().plot(kind='bar', figsize=(10,5))
plt.title("Building Storey")
plt.ylabel('Numbers')
plt.xlabel('Storey Nos');
df.BLDG_US.value_counts().plot(kind='bar', figsize=(10,5))
plt.title("Building Types")
plt.ylabel('Numbers')
plt.xlabel('Type');
df.BLDG_TYP.value_counts().plot(kind='bar', figsize=(10,5))
plt.title("Building Types")
plt.ylabel('Numbers')
plt.xlabel('Area (m2)');<jupyter_output><empty_output><jupyter_text>Let's check corr betweeen total carbon and unit carbon.<jupyter_code>plt.figure(figsize=(10,5))
c= df.corr()
sns.heatmap(c,cmap="BrBG",annot=True)
fig, ax = plt.subplots(figsize=(18,6))
ax.scatter(df['Average_Area_m2'], df['EC_LCAA_PERM2'])
ax.set_xlabel('Average_Area_m2')
ax.set_ylabel('EC_LCAA_PERM2')
plt.show()
fig, ax = plt.subplots(figsize=(18,6))
ax.scatter(df['BLDG_TYP'], df['EC_LCAA_PERM2'])
ax.set_xlabel('BLDG_TYP')
ax.set_ylabel('EC_LCAA_PERM2')
plt.show()
fig, ax = plt.subplots(figsize=(18,6))
ax.scatter(df['BLDG_STOR_A'], df['EC_LCAA_PERM2'])
ax.set_xlabel('BLDG_STOR_A')
ax.set_ylabel('EC_LCAA_PERM2')
plt.show()<jupyter_output><empty_output><jupyter_text>Statistical data of the data frame<jupyter_code>df['EC_LCAA_PERM2'].describe()
fig, ax = plt.subplots(figsize=(18,6))
ax = sns.boxplot(x='BLDG_STOR_A', y='EC_LCAA_PERM2', data=df, orient="v")
fig, ax = plt.subplots(figsize=(18,6))
ax = sns.boxplot(x='BLDG_AREA_M2', y='EC_LCAA_PERM2', data=df, orient="v")
df['BLDG_TYP_CAT']=df['BLDG_TYP'].astype('category').cat.codes
df['BLDG_NEW_REN_CAT']=df['BLDG_NEW_REN'].astype('category').cat.codes

df.corr()
plt.figure(figsize=(10,5))
c= df.corr(method='pearson')
sns.heatmap(c,cmap="BrBG",annot=True)<jupyter_output><empty_output><jupyter_text>We can say that there is no significant relation between different variables. However the most effective items on LCAA per m2 is;1.   Building Area2.   Building Storey3.   Building Type<jupyter_code>df=df.drop(['EC_WB_EX_OPER'],axis=1)
df.head(5)
df.corr()
plt.figure(figsize=(10,5))
c= df.corr(method='pearson')
sns.heatmap(c,cmap="BrBG",annot=True)<jupyter_output><empty_output><jupyter_text>ML Models sklearn.ensemble.RandomForestRegressor Test Data Split<jupyter_code>features = df.drop(["EC_LCAA_PERM2",'BLDG_TYP','BLDG_US','BLDG_LOC_REGION','BLDG_NEW_REN','BLDG_AREA_M2','BLDG_STOR_A'], axis=1).columns
features
df_train, df_test = train_test_split(df, test_size=0.15)
clf = RandomForestRegressor(n_estimators=20)
clf.fit(df_train[features], df_train["EC_LCAA_PERM2"])
predictions = clf.predict(df_test[features])

display(predictions)
score = clf.score(df_test[features], df_test["EC_LCAA_PERM2"])
print("Accuracy: ", score)
ig = plt.figure(figsize=(10, 9))
ax = fig.add_subplot(111)

df_f = pd.DataFrame(clf.feature_importances_, columns=["importance"])
df_f["labels"] = features
df_f.sort_values("importance", inplace=True, ascending=False)
display(df_f.head(5))

index = np.arange(len(clf.feature_importances_))
bar_width = 0.5
rects = plt.barh(index , df_f["importance"], bar_width, alpha=0.4, color='b', label='Main')
plt.yticks(index, df_f["labels"])
plt.show()
r2_score(df_test["EC_LCAA_PERM2"], predictions)<jupyter_output><empty_output><jupyter_text>Although the model is not so powerful, we can say that the most important thing in carbon calculation is building area, story number and type of building.  Let's change features<jupyter_code>features2 = df.drop(["EC_LCAA_PERM2",'BLDG_TYP','BLDG_US','BLDG_LOC_REGION','BLDG_NEW_REN','BLDG_AREA_M2','BLDG_STOR_A',"BLDG_NEW_REN_CAT"], axis=1).columns
features2
df_train2, df_test2 = train_test_split(df, test_size=0.15)
clf2 = RandomForestRegressor(n_estimators=20)
clf2.fit(df_train2[features2], df_train2["EC_LCAA_PERM2"])
predictions2 = clf2.predict(df_test2[features2])

display(predictions2)
score2 = clf2.score(df_test2[features2], df_test2["EC_LCAA_PERM2"])
print("Accuracy: ", score2)
ig = plt.figure(figsize=(10, 9))
ax = fig.add_subplot(111)

df_f = pd.DataFrame(clf2.feature_importances_, columns=["importance"])
df_f["labels"] = features2
df_f.sort_values("importance", inplace=True, ascending=False)
display(df_f.head(5))

index = np.arange(len(clf2.feature_importances_))
bar_width = 0.5
rects = plt.barh(index , df_f["importance"], bar_width, alpha=0.4, color='b', label='Main')
plt.yticks(index, df_f["labels"])
plt.show()<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: MatplotlibDeprecationWarning: Adding an axes using the same arguments as a previous axes currently reuses the earlier instance.  In a future version, a new instance will always be created and returned.  Meanwhile, this warning can be suppressed, and the future behavior ensured, by passing a unique label to each axes instance.
{'fraction_non_alphanumeric': 0.14883780660074483, 'fraction_numerical': 0.027481700269680238, 'mean_word_length': 7.7309417040358746, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '534225', 'license': 'Apache-2.0', 'n_tokens_mistral': 3283, 'n_tokens_neox': 3081, 'n_words': 498, 'path': 'Carbon.ipynb', 'repo_name': 'Aydino/carbon', 'size': '7787'}