<jupyter_start><jupyter_text>"""Perform the following operations using R/Python on the Air Quality Index data sets i) Data Cleaning ii)  Data Transformation iii) Data Integration iv) Error Correcting v) Model Building """<jupyter_code>import pandas as pd 
import numpy as np
df = pd.read_csv("Datasets\AirQualityodisha.csv",delimiter=',') 
print(df.head())
x=df['PM 2.5'].isnull() # detects null value N/A print(x.head(n=1000))
print(df.isnull().sum())# finds per feature number of null values
median = df['PM 2.5'].median()
updated = df['PM 2.5'].fillna(median, inplace=False) 
print (updated.head(n=1300))
print(df.head(n=50)) 
so2_total=df.groupby('City')["SO2"].sum().rename('SO2_Total').reset_index() 
df1 = df.merge(so2_total)
print(so2_total.iloc[0:20,0]) 
print(df1.head(n=5))
df["so2_total1"]=df.groupby('City')["SO2"].transform('sum') #"""df2 = df.merge(so2_total1)
print(df.head(n=5))
df = pd.read_csv("datasets\weather_data_katraj.csv",delimiter=',')
print(list(df))
df3 = pd.concat([df1,df],axis=1) # adding columns horizontally print(df3.columns.values)
print(list(df3))
erows = df3.iloc[191:200,:]
print(erows.head())
df4 = pd.concat([df3,erows],axis=0) # adding rows horizontally print(df4.tail(n=20))
df3=df3.append(erows)# adding rows horizontally alternative way print(df3.tail(n=1))

print(list(df))
df_wna = df.dropna(subset=[' pm25']) 
x=df[' pm25'].isnull().sum()
print("Total number of null values"+ str(x)) 
dfnew = pd.read_csv("Datasets\AirQualityodisha.csv",delimiter=',') 
print(dfnew.head())
x=dfnew['PM 2.5'].isnull() # detects null value N/A print(x.head(n=1000))
print(dfnew.isnull().sum())# finds per feature number of null values
median = dfnew['PM 2.5'].median()
updated = dfnew['PM 2.5'].fillna(median, inplace=False) 
print (updated.head(n=1300))
print(dfnew.head(n=50)) 
so2_total=dfnew.groupby('City')["SO2"].sum().rename('SO2_Total').reset_index() 
df1 = dfnew.merge(so2_total)
print(so2_total.iloc[0:20,0]) 
print(df1.head(n=5))
dfnew["so2_total1"]=dfnew.groupby('City')["SO2"].transform('sum') #"""df2 = df.merge(so2_total1)
print(dfnew.head(n=5))
import numpy as np 
import matplotlib.pyplot as plt 
from operator import mul
  
def compute_regcoef(x, y): 
    # number of observations/points 
    # n = np.size(x) 
    l = len(x)
    print("Length of lis t "+str(l))
    
    # mean of x and y vector 
    #m_x, m_y = np.mean(x), np.mean(y) 
    
    # calculating cross-deviation and deviation about x 
    l=int(l)
    sumxy =sum(list(map(mul,x,y)))
    num= l*sumxy - sum(x)*sum(y) 
    sumxx = sum(list(map(mul,x,x)))
    denom = l*sumxx - sum(x)*sum(x) 
  
    # calculating regression coefficients 
    b1 = num / denom
    
    b0 = (sum(y) - b1*sum(x)) / l 
  
    return(b0, b1) 
  
def plot_regline(x, y, b): 
    # plotting the actual points as scatter plot 
    plt.scatter(x, y, color = "b",marker = "o", s = 30) 
    # predicted response vector 
    b=list(b)
    y_pred = b[0] + b[1]*np.float_(x )
     
    # plotting the regression line 
    plt.plot(x, y_pred, color = "g") 
    prey =b[0] + b[1] * x[3]
    print("Predicted y is " +str(prey))
    # putting labels 
    plt.xlabel('x') 
    plt.ylabel('y') 
  
    # function to show plot 
    plt.show() 
  
def fun1(): 
    # observations 
   # x = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) 
    #y = np.array([1, 3, 2, 5, 7, 8, 8, 9, 10, 12]) 
    #x=[0,1,2,3,4,5,6,7,8,9]
    x=[1,2,3,4,5,6,7,8,9,10]
    y=[1,2,3,3,5,6,6,7,9,11]
  
    # estimating coefficients 
    b = compute_regcoef(x, y) 
    print ("Regression Coefficients are b0,b1 ", b[0],b[1])
 
    # plotting regression line 
    plot_regline(x, y, b) 
  
fun1()<jupyter_output>Length of lis t 10
Regression Coefficients are b0,b1  -0.33333333333333287 1.0242424242424242
Predicted y is 3.7636363636363637
{'fraction_non_alphanumeric': 0.18032786885245902, 'fraction_numerical': 0.059227921734531994, 'mean_word_length': 4.153950953678474, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '272053', 'license': 'MIT', 'n_tokens_mistral': 1674, 'n_tokens_neox': 1532, 'n_words': 324, 'path': 'Part B/PartB 2.ipynb', 'repo_name': 'Yashdew/DSBDA-SL6', 'size': '3782'}