<jupyter_start><jupyter_text>Exploring Data<jupyter_code>#Check uniqueness of column before setting it as a index
hennepin['PIN'].nunique(dropna=True)/len(hennepin['PIN'])
#set index and sort to make processing fast
hennepin=hennepin.set_index(['PIN']).sort_index()
def convert_to_categorical(df,cols):
    for col in cols:
        unique_vals=len(df[col].unique())
        
        unique_ratio=unique_vals/len(df)
        if unique_ratio<=0.2:
            df[col]=df[col].astype('category')
    return df
column_list=list(hennepin.select_dtypes(include=['object']).columns.values)
len(column_list)
#remove value in column list which we dont want to convert to categorical
column_list.remove('geometry')
hennepin=convert_to_categorical(hennepin,column_list)
hennepin.info(verbose=False)
type(hennepin)
hennepin.plot()
from pandas_summary import DataFrameSummary
hennepin_stats=DataFrameSummary(hennepin.drop('geometry',axis=1))
#to create random index
#m['id'] = [random.randint(0,1000) for x in range(m.shape[0])]
#to list columns with missing records drop them from the analysis

ans=[]
for i in range(len(m.index)):
    if m.values[i]==float(1):
        ans.append(m.index[i])
hennepin=hennepin.drop(ans,axis=1)
#Finds missingness in the data 
msno.matrix(hennepin,labels=True, sort=True)
def delete_near_null(df,cols,populated_threshold=0.01):
    for col in cols:
        non_null_rows =df[col].count()
        total_rows=len(df[col])
        populated_ratio=non_null_rows/total_rows
        if populated_ratio<=populated_threshold:
            del df[col]
    return df
# Get the full list of columns to check
column_list=list(hennepin.columns.values)

#Dont want to edit geometry colum
column_list.remove('geometry')

hennepin=delete_near_null(hennepin,column_list)
hennepin.shape
#Filterning on Geodesic Features
hennepin=hennepin[hennepin['geometry'].geom_type!='Point']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11780104712041885, 'fraction_numerical': 0.007329842931937173, 'mean_word_length': 5.612456747404845, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '340834', 'license': 'MIT', 'n_tokens_mistral': 664, 'n_tokens_neox': 674, 'n_words': 135, 'path': 'Packages_geospatial.ipynb', 'repo_name': 'Inamdarpushkar/GIS-and-Remotesensing', 'size': '1910'}