<jupyter_start><jupyter_text>Normalizing the data<jupyter_code>colsList = ["actual_area",
            "poolcnt",
            "latitude",
            "longitude",
            "unitcnt",
            "lotsizesquarefeet",
            "bedroomcnt",
            "calculatedbathnbr",
            "hashottuborspa",
            "fireplacecnt",
            "taxvaluedollarcnt",
            "buildingqualitytypeid",
            "garagecarcnt",
            "age",
            "taxamount"]
prop_data_ahp = prop_data[colsList]
# prop_data_ahp
for col in prop_data_ahp.columns:
    prop_data_ahp[col] = (prop_data_ahp[col] - prop_data_ahp[col].mean())/prop_data_ahp[col].std(ddof=0)
# prop_data_ahp.isnull().sum()
for cols in prop_data_ahp.columns.values:
    print prop_data_ahp[cols].value_counts(dropna=False)<jupyter_output>-0.130731     47739
-0.199816      8968
-0.249310      7211
-0.323551      7111
-0.447286      6024
-0.364796      5956
-0.385418      5837
-0.406041      5436
-0.298804      5319
-0.220439      4743
-0.348298      4685
-0.292617      4658
-0.261684      4585
-0.302929      4520
-0.271995      4478
-0.274057      4281
-0.335925      4211
-0.459659      4166
-0.132793      4162
-0.397792      4148
-0.422539      4147
-0.150322      4029
-0.117326      3987
-0.496780      3926
-0.137949      3919
-0.352422      3888
-0.224563      3831
-0.076082      3799
-0.307053      3784
-0.168883      3690
              ...  
 6.255530         1
 6.173555         1
 5.930210         1
 37.453189        1
 10.936313        1
 3.555018         1
 4.396415         1
-0.859219         1
 5.301225         1
 6.710255         1
 5.229562         1
 46.473969        1
 10.368164        1
 7.095379         1
 3.632353         1
 5.497654         1
 7.580007         1
 5.365670         1
 3.451906         1
[...]<jupyter_text>Analytical Hierarchical Processing<jupyter_code>rel_imp_matrix = pd.read_csv("rel_imp_matrix.csv", index_col=0)
# rel_imp_matrix
import fractions


for col in rel_imp_matrix.columns.values:
    temp_list = rel_imp_matrix[col].tolist()
    rel_imp_matrix[col] = [float(fractions.Fraction(x))  for x in temp_list]
    
#     data = [float(fractions.Fraction(x)) for x in data]
# rel_imp_matrix

for col in rel_imp_matrix.columns.values:
    rel_imp_matrix[col] /= rel_imp_matrix[col].sum()
    
# rel_imp_matrix
rel_imp_matrix["row_sum"] = rel_imp_matrix.sum(axis=1)

rel_imp_matrix["score"] = rel_imp_matrix["row_sum"]/rel_imp_matrix.shape[0]

rel_imp_matrix.to_csv("final_score_matrix.csv", index=False)
# rel_imp_matrix
ahp_column_score = rel_imp_matrix["score"]
ahp_column_score
prop_data_ahp.info()
prop_data_ahp.drop('sum', axis=1,inplace=True)
prop_data_ahp.keys()<jupyter_output><empty_output><jupyter_text>SAW<jupyter_code>sum_series = pd.Series(0, index=prop_data_ahp.index,dtype='float32')

for col in prop_data_ahp.columns:
    sum_series = sum_series+ prop_data_ahp[col] * ahp_column_score[col]
prop_data_ahp["sum"] = sum_series.astype('float32')
prop_data_ahp["sum"]
# prop_data_ahp["sum"] = prop_data_ahp.sum(axis=1)
    
prop_data_ahp["sum"].describe()
prop_data_ahp.sort_values(by='sum', inplace=True)
prop_data_ahp.head(n=10)
prop_data_ahp.tail(n=10)
print prop_data[colsList].iloc[1252741],"\n\n"
print prop_data[colsList].iloc[342941]
# #imputing airconditioningtypeid, making some NaN to 1.0 where heatingorsystemtypeid == 2

# prop_data.loc[(prop_data["heatingorsystemtypeid"]==2.0) & (pd.isnull(prop_data["airconditioningtypeid"])), "airconditioningtypeid"] = 1.0

# prop_data["airconditioningtypeid"].fillna(-1, inplace=True)
# print prop_data["airconditioningtypeid"].value_counts()

# prop_data[["airconditioningtypeid", "heatingorsystemtypeid"]].head()

# duplicate_or_not_useful_cols = pd.Series(['calculatedbathnbr', 'assessmentyear', 'fullbathcnt', 
#                     'regionidneighborhood', 'propertyzoningdesc', 'censustractandblock'])#,'finishedsquarefeet12'])
# prop_data.drop(duplicate_or_not_useful_cols, axis=1, inplace=True)

# prop_data["buildingqualitytypeid"].fillna(prop_data["buildingqualitytypeid"].mean(), inplace=True)
# prop_data["calculatedfinishedsquarefeet"].interpolate(inplace=True)
# prop_data["heatingorsystemtypeid"].fillna(-1, inplace=True)
# prop_data["lotsizesquarefeet"].fillna(prop_data["lotsizesquarefeet"].median(), inplace=True)
# prop_data.drop(["numberofstories"], axis=1, inplace=True)
# #removing propertycountylandusecode because it is not in interpretable format
# prop_data.drop(["propertycountylandusecode"], axis=1, inplace=True)
# prop_data["regionidcity"].interpolate(inplace=True)
# prop_data["regionidzip"].interpolate(inplace=True)
# prop_data["yearbuilt"].fillna(prop_data["yearbuilt"].mean(), inplace=True)

# #impute structuretaxvaluedollarcnt, taxvaluedollarcnt, landtaxvaluedollarcnt, taxamount by interpolation
# cols_to_interpolate = ["structuretaxvaluedollarcnt", "taxvaluedollarcnt", "landtaxvaluedollarcnt", "taxamount"]
# for c in cols_to_interpolate:
#     prop_data[c].interpolate(inplace=True)

    
# #imputing garagecarcnt on basis of propertylandusetypeid
# #All the residential places have 1 or 2 garagecarcnt, hence using random filling for those values.

# prop_data.loc[(prop_data["propertylandusetypeid"]==261) & (pd.isnull(prop_data["garagecarcnt"])), "garagecarcnt"] = np.random.randint(1,3)
# prop_data.loc[(prop_data["propertylandusetypeid"]==266) & (pd.isnull(prop_data["garagecarcnt"])), "garagecarcnt"] = np.random.randint(1,3)

# prop_data["garagecarcnt"].fillna(-1, inplace=True)

# prop_data["garagecarcnt"].value_counts(dropna=False)

# #imputing garagetotalsqft using the garagecarcnt

# prop_data.loc[(prop_data["garagecarcnt"]==-1) & (pd.isnull(prop_data["garagetotalsqft"]) | (prop_data["garagetotalsqft"] == 0)), "garagetotalsqft"] = -1
# prop_data.loc[(prop_data["garagecarcnt"]==1) & (pd.isnull(prop_data["garagetotalsqft"]) | (prop_data["garagetotalsqft"] == 0)), "garagetotalsqft"] = np.random.randint(180, 400)
# prop_data.loc[(prop_data["garagecarcnt"]==2) & (pd.isnull(prop_data["garagetotalsqft"]) | (prop_data["garagetotalsqft"] == 0)), "garagetotalsqft"] = np.random.randint(400, 720)
# prop_data.loc[(prop_data["garagecarcnt"]==3) & (pd.isnull(prop_data["garagetotalsqft"]) | (prop_data["garagetotalsqft"] == 0)), "garagetotalsqft"] = np.random.randint(720, 880)
# prop_data.loc[(prop_data["garagecarcnt"]==4) & (pd.isnull(prop_data["garagetotalsqft"]) | (prop_data["garagetotalsqft"] == 0)), "garagetotalsqft"] = np.random.randint(880, 1200)

# #interpolate the remaining missing values
# prop_data["garagetotalsqft"].interpolate(inplace=True)

# prop_data["garagetotalsqft"].value_counts(dropna=False)


# #imputing unitcnt using propertylandusetypeid
            
# prop_data.loc[(prop_data["propertylandusetypeid"]==261) & pd.isnull(prop_data["unitcnt"]), "unitcnt"] = 1
# prop_data.loc[(prop_data["propertylandusetypeid"]==266) & pd.isnull(prop_data["unitcnt"]), "unitcnt"] = 1
# prop_data.loc[(prop_data["propertylandusetypeid"]==269) & pd.isnull(prop_data["unitcnt"]), "unitcnt"] = 1

# prop_data.loc[(prop_data["propertylandusetypeid"]==246) & pd.isnull(prop_data["unitcnt"]), "unitcnt"] = 2
# prop_data.loc[(prop_data["propertylandusetypeid"]==247) & pd.isnull(prop_data["unitcnt"]), "unitcnt"] = 3
# prop_data.loc[(prop_data["propertylandusetypeid"]==248) & pd.isnull(prop_data["unitcnt"]), "unitcnt"] = 4

# prop_data["unitcnt"].value_counts(dropna=False)<jupyter_output><empty_output><jupyter_text>Distance MetricWe will be using weighted Manhattan distance as a distance metric<jupyter_code>dist_imp_matrix = pd.read_csv("./dist_metric.csv", index_col=0)
dist_imp_matrix
import fractions


for col in dist_imp_matrix.columns.values:
    temp_list = dist_imp_matrix[col].tolist()
    dist_imp_matrix[col] = [float(fractions.Fraction(x))  for x in temp_list]
    
# dist_imp_matrix
for col in dist_imp_matrix.columns.values:
    dist_imp_matrix[col] /= dist_imp_matrix[col].sum()
dist_imp_matrix["row_sum"] = dist_imp_matrix.sum(axis=1)

dist_imp_matrix["score"] = dist_imp_matrix["row_sum"]/dist_imp_matrix.shape[0]

dist_imp_matrix.to_csv("final_score_matrix_Q2.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14394124847001225, 'fraction_numerical': 0.07417380660954713, 'mean_word_length': 4.865757358219669, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '639', 'license': 'Apache-2.0', 'n_tokens_mistral': 3482, 'n_tokens_neox': 3124, 'n_words': 425, 'path': 'HW3/HW3.ipynb', 'repo_name': 'jay-z007/Data-Science-Fundamentals', 'size': '8170'}