<jupyter_start><jupyter_text>\Keeping only numeric columns<jupyter_code>X = cfe.drop(columns = ['fips', 'county', 'province_state', 'covid_last_update'])<jupyter_output><empty_output><jupyter_text>\Scaling<jupyter_code>ss = StandardScaler()
X_sc = ss.fit_transform(X)<jupyter_output><empty_output><jupyter_text>\KMeans<jupyter_code>def optimize_kmeans(krange, scaled_X):
    
    max_score= -1
    
    for i in range(2,krange):
        kmeans = KMeans(n_clusters=i)
        kmeans.fit(scaled_X)
        score = silhouette_score(scaled_X, kmeans.labels_)
        if score > max_score:
            max_score = score
            k = i    
    print(f'StandardScaler silhouette score: {max_score} for optimal k of {k}')
optimize_kmeans(10, X_sc)
def kmeans_range(krange, scaled_X):
    
    for i in range(2,krange):
        kmeans = KMeans(n_clusters=i)
        kmeans.fit(scaled_X)
        score = silhouette_score(scaled_X, kmeans.labels_)
        k=i
        print(f'StandardScaler silhouette score: {score} for optimal k of {k}')
kmeans_range(10, X_sc)<jupyter_output>StandardScaler silhouette score: 0.42945415594366265 for optimal k of 2
StandardScaler silhouette score: 0.34145168294328876 for optimal k of 3
StandardScaler silhouette score: 0.2916211486466191 for optimal k of 4
StandardScaler silhouette score: 0.27796020085640616 for optimal k of 5
StandardScaler silhouette score: 0.17114562258838356 for optimal k of 6
StandardScaler silhouette score: 0.19488073716278653 for optimal k of 7
StandardScaler silhouette score: 0.21173223514611286 for optimal k of 8
StandardScaler silhouette score: 0.19127984137207238 for optimal k of 9<jupyter_text>\DBSCAN<jupyter_code>def DBSCAN_optimize(X_scaled, min_sample_range): 
   
    max_score=-1     

    for eps in np.linspace(.2, 5, 50):
        for minsamples in range(2, min_sample_range):
            
            dbscan = DBSCAN(eps=eps, min_samples=minsamples)
            dbscan.fit(X_scaled)
            
            if len(set(dbscan.labels_)) > 1:
               
                score = silhouette_score(X_scaled, dbscan.labels_)
                nclusters = len(set(dbscan.labels_))
                
                if score > max_score:
                    max_score = score
                    best_eps = eps
                    best_minsamples = minsamples
                    best_clusters = nclusters
    
    print(f'Opitimal silhoute score: {round(max_score, 2)} with eps of: {round(best_eps, 2)}')
    print(f'min_samples of: {best_minsamples} and {best_clusters} clusters.')
    return 
DBSCAN_optimize(X_sc, 10)<jupyter_output>Opitimal silhoute score: 0.61 with eps of: 4.71
min_samples of: 2 and 2 clusters.
{'fraction_non_alphanumeric': 0.08603351955307263, 'fraction_numerical': 0.0659217877094972, 'mean_word_length': 3.082066869300912, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '272554', 'license': 'CC0-1.0', 'n_tokens_mistral': 1070, 'n_tokens_neox': 920, 'n_words': 209, 'path': 'code/clustering-so-far.ipynb', 'repo_name': 'minoot7/Indexing-Natural-Disasters-and-Outbreaks', 'size': '2685'}