<jupyter_start><jupyter_text>DBSCAN (Density-Based Spatial Clustering of Applications with Noise)Find core samples of high density and expand clusters from them.The minimum number of samples in a neighborhood for a point to be considered as a core point was set at 10 and the maximum distance between two samples within the same neighborhood was set at 0.1.<jupyter_code>import pandas as pd


df1=pd.read_csv('team_out_1.csv')
df2=pd.read_csv('team_out_a2.csv')

df=df1.append(df2)
df.dropna(inplace=True)
df.reset_index(inplace=True,drop=True)
df=df[df.Total_Expenses>0]
df=df[df.Program_Exp<=1]
df<jupyter_output><empty_output><jupyter_text>small_df=df[df.Total_Expenses<1000000]med_df=df[df.Total_Expenses>1000000]med_df=med_df[df.Total_Expenses<10000000]large_df=df[df.Total_Expenses<50000000]large_df=large_df[df.Total_Expenses>10000000]national_df=df[df.Total_Expenses>50000000]<jupyter_code>#Getting a list of positive businesses

temp=df[df.Program_Exp>.9] 
temp=temp[temp.Liabilities_To_Asset<.2]
lst_temp=list(temp['EIN'])
df.reset_index(drop=True,inplace=True)
norm_df=df.copy()

norm_df=norm_df[['Program_Exp','Liabilities_To_Asset','Working_Capital','Surplus_Margin','Total_Expenses']]

from sklearn import preprocessing

x = norm_df.values #returns a numpy array
min_max_scaler = preprocessing.MinMaxScaler()
x_scaled = min_max_scaler.fit_transform(x)
norm_df = pd.DataFrame(x_scaled)
norm_df["Filename"]=df['Filename']

norm_df["EIN"]=df['EIN']
norm_df.columns=['Program_Exp','Liabilities_To_Asset','Working_Capital','Surplus_Margin','Total_Expenses','Filename','EIN']
norm_df.set_index('EIN',inplace=True)
norm_df

df2 = df.copy()
df2.reset_index(inplace=True, drop = True)
print(norm_df2)
Y_class_df = pd.DataFrame()
X_class_df=norm_df.loc[lst_temp]
X_class_df['Efficiency'] = 1


Y_class_df['Efficiency'] = X_class_df['Efficiency'] 
X_class_df.drop('Efficiency', axis=1, inplace=True)

new_df=norm_df[['Program_Exp','Liabilities_To_Asset','Working_Capital','Surplus_Margin']]
new_df.reset_index(inplace=True,drop=True)
X_class_df=X_class_df[['Program_Exp','Liabilities_To_Asset','Working_Capital','Surplus_Margin']]
# X_class_df=X_class_df.drop(X_class_df.index[2])   #OUTLIER REMOVER
X_class_df.reset_index(inplace=True,drop=True)
import numpy as np

from sklearn.cluster import DBSCAN
from sklearn import metrics
from sklearn.datasets.samples_generator import make_blobs
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import TruncatedSVD

svd = TruncatedSVD(n_components=2, n_iter=7)
X = svd.fit_transform(new_df)
db = DBSCAN(eps=0.1, min_samples=10).fit(X)
core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
core_samples_mask[db.core_sample_indices_] = True
labels = db.labels_
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
print(n_clusters_)

outliers_index = []
for i in range(len(labels)):
    if(labels[i]==-1):
        outliers_index.append(i)
        #print(df2.loc[i])
outliers = df2.loc[outliers_index]
import matplotlib.pyplot as plt

# Black removed and is used for noise instead.
unique_labels = set(labels)
colors = 'cyan'
for k, col in zip(unique_labels, colors):
    if k == -1:
        # Black used for noise.
        col = 'k'

    class_member_mask = (labels == k)

    xy = X[class_member_mask & core_samples_mask]
    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=col,
             markeredgecolor='k', markersize=14)

    xy = X[class_member_mask & ~core_samples_mask]
    plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=col,
             markeredgecolor='k', markersize=6)

plt.title('Estimated number of clusters: %d' % n_clusters_)
plt.show()

outliers<jupyter_output>1
{'fraction_non_alphanumeric': 0.12864540746797493, 'fraction_numerical': 0.023712183156173343, 'mean_word_length': 7.083700440528634, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '436065', 'license': 'MIT', 'n_tokens_mistral': 1442, 'n_tokens_neox': 1376, 'n_words': 243, 'path': 'DBSCAN.ipynb', 'repo_name': 'rbazaz/IRS-990-Analysis', 'size': '3669'}