<jupyter_start><jupyter_text>Data Cleaning of Housing.com ___<jupyter_code>import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
df = pd.read_csv(r'data_housing_com.csv')
df.head()<jupyter_output><empty_output><jupyter_text>Cleaning Location Data<jupyter_code>df['location'] = [x[:-8] for x in df['location']]

for i,x in enumerate(df['location']):
    
    if len(x.split(','))> 1:
        df['location'][i] = x.split(',')[-1]
        
for i,x in enumerate(df['location']):
    
    if x == 'Dattagalli Third Stage':
        df['location'][i] = 'Dattagalli'
    
df.head()<jupyter_output><empty_output><jupyter_text>Beds Column Cleaning* A new column **Type** is added to the dataframe which consits of the type of the property<jupyter_code>df['beds1'] = [x.split(' B')[0] for x in df['beds1']]
df['Type'] = ''
for i,x in enumerate(df['beds2']):
    if 'villa' in x.lower():
        df['Type'][i] = 'Villa'
    elif 'apartment' in x.lower():
        df['Type'][i] = 'Apartment'
    else:
        df['Type'][i] = 'House'
        
    if 'bhk' in x.lower():
        df['beds1'][i] = x.split(' B')[0].split(',')[-1]
    
alphabets = df[df['beds1'].str.contains('[A-Za-z]')].index
df.drop(alphabets,0,inplace=True)
df.drop(['beds2'],1, inplace=True)
df.rename(columns={'location':'Location','beds1':'Beds','price':'Price','area':'Area'},inplace=True)
df['Price'] = [x[1:-1] for x in df['Price']]
df['Price'] = [x.split(',')[-1] for x in df['Price']]
df['Area']= [x.split(',')[0][-4: ] for x in df['Area']]
df['Area'] = [int(x) for x in df['Area']]
df.head(20)<jupyter_output><empty_output><jupyter_text>Creating a Temporary DataFrame to fill out the missing values The missing values can be filled with the median house value in that locality<jupyter_code>temp_df = df[df['Price'] != '']
temp_df['Beds'] = temp_df['Beds'].apply(lambda x: round(float(x)))
temp_df['Price'] = temp_df['Price'].apply(lambda x: int(x.split(',')[-1]))
temp_df.describe()
temp_df['Location'] = [x.strip(' ') for x in temp_df['Location']]<jupyter_output><empty_output><jupyter_text>The missing datas are replaced by the median of the price for the corresponding locality<jupyter_code>missing_data = temp_df.groupby(['Location']).median()
missing_data.head()
missing_prices = missing_data['Price']
missing_prices.head()
missing_df = df[df['Price'] == '']
missing_df.drop(23,0,inplace=True)<jupyter_output><empty_output><jupyter_text>Passing the missing Values to the Main Data Frame<jupyter_code>for i,x in enumerate(missing_df['Location']):
    #print(x)
    indexs = missing_df.index
    try:
        df['Price'][indexs[i]] = missing_prices[x.strip(' ')]
        #print('1')
    except KeyError:
       # print('0')
        pass
      
missing_df.index<jupyter_output><empty_output><jupyter_text>DataFrame with filled values<jupyter_code>df.head(20)
df.isnull().sum()
df['Price'] = [int(x) for x in df['Price']]
df['Area'] = [int(x) for x in df['Area']]
df['Beds'] = [round(float(x)) for x in df['Beds']]<jupyter_output><empty_output><jupyter_text>Lets replace the missing values with a rather large number, its treated as an outlier<jupyter_code>df.replace(0,-999999,inplace= True)
df.to_csv('data_cleaned.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19000919399325775, 'fraction_numerical': 0.011339258351210543, 'mean_word_length': 5.476190476190476, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '372179', 'license': 'MIT', 'n_tokens_mistral': 1250, 'n_tokens_neox': 1224, 'n_words': 251, 'path': 'Dataset/data_housing_com/Data_cleaning.ipynb', 'repo_name': 'karthikmprakash/Mysore-Real-Estate-Analysis', 'size': '3263'}