<jupyter_start><jupyter_text>a day of data (no accident)<jupyter_code>model_02_before = Birch(n_clusters=None, threshold=0.2)

list_of_nmpys_for_graphs = []

removed_outliers_numpy = []
list_of_nmpys_for_graphs = []

labels_dict = {}
label_1_dict = {}
label_0_dict = {}
add_data_to_this = pd.DataFrame(columns = columns)

fit_number = 1

list_of_dfs

for i in range(0, len(list_of_dfs)):
    numpy_array = outliers_removal(list_of_dfs[i])
    
    model_02_before.partial_fit(numpy_array[:, :-2])
    
    model_copy = copy.deepcopy(model_02_before)
    
    removed_outliers_numpy.append(numpy_array)
    new_nmpy = np.concatenate(removed_outliers_numpy, axis=0)
    
    n = plot_silhouette_scores(model_copy, new_nmpy[:, :-2], 2, 10, fit_number)
#     print(n)
    
    model_copy.set_params(n_clusters = n)
    model_copy.partial_fit()
    
    labels_dict[f'fit_{fit_number}'] = model_copy.predict(new_nmpy[:, :-2])
    unique, counts = np.unique(labels_dict[f'fit_{fit_number}'], return_counts=True)
    label_1_dict[f'fit_{fit_number}'] = counts[1]
    label_0_dict[f'fit_{fit_number}'] = counts[0]
#     print(collections.Counter(labels_dict[f'fit_{fit_number}']))
#     print(get_cal_score(new_nmpy[:, :-2], labels_dict[f'fit_{fit_number}']))
    list_of_nmpys_for_graphs.append(new_nmpy)
    fit_number += 1

col_counter = 0
row_counter = 0

fit_number = 1

nrows = 4
ncols = 4

figure, axs = plt.subplots(nrows=nrows, ncols=ncols,figsize=(14,14))

for i in list_of_nmpys_for_graphs:
    
#     numpy_array = i.to_numpy()
    
    figure.tight_layout(rect=[0, 0.03, 1, 0.95])
    
    axs[row_counter,col_counter].scatter(i[:, 5], i[:, 4], c =labels_dict[f'fit_{fit_number}'], cmap='rainbow', alpha=0.7 )
    axs[row_counter,col_counter].set_xlabel('Time of the day')
    axs[row_counter,col_counter].set_ylim(0, 750)
    axs[row_counter,col_counter].set_ylabel('Car number')
    axs[row_counter,col_counter].set_title(f'Fit {fit_number}')
    
    if fit_number >= 9:
        axs[row_counter,col_counter].axvline(x=143, color='g')
    if col_counter == (ncols-1):
        col_counter = 0
        row_counter += 1
    else:
        col_counter += 1
    
    fit_number += 1
for i in label_1_dict.values():
    print(i)
    
print('\n')

for i in label_0_dict.values():
    print(i)
model_001_before = Birch(n_clusters=None, threshold=0.01)

list_of_nmpys_for_graphs = []

removed_outliers_numpy = []
list_of_nmpys_for_graphs = []

labels_dict = {}
label_1_dict = {}
label_0_dict = {}
add_data_to_this = pd.DataFrame(columns = columns)

fit_number = 1

list_of_dfs

for i in range(0, len(list_of_dfs)):
    numpy_array = outliers_removal(list_of_dfs[i])
    
    model_001_before.partial_fit(numpy_array[:, :-2])
    
    model_copy = copy.deepcopy(model_001_before)
    
    removed_outliers_numpy.append(numpy_array)
    new_nmpy = np.concatenate(removed_outliers_numpy, axis=0)
    
    n = plot_silhouette_scores(model_copy, new_nmpy[:, :-2], 2, 10, fit_number)
#     print(n)
    
    model_copy.set_params(n_clusters = n)
    model_copy.partial_fit()
    
    labels_dict[f'fit_{fit_number}'] = model_copy.predict(new_nmpy[:, :-2])
    unique, counts = np.unique(labels_dict[f'fit_{fit_number}'], return_counts=True)
    label_1_dict[f'fit_{fit_number}'] = counts[1]
    label_0_dict[f'fit_{fit_number}'] = counts[0]
#     print(collections.Counter(labels_dict[f'fit_{fit_number}']))
#     print(get_cal_score(new_nmpy[:, :-2], labels_dict[f'fit_{fit_number}']))
    list_of_nmpys_for_graphs.append(new_nmpy)
    fit_number += 1
    
for i in label_1_dict.values():
    print(i)
    
print('\n')

for i in label_0_dict.values():
    print(i)
col_counter = 0
row_counter = 0

fit_number = 1

nrows = 4
ncols = 4

figure, axs = plt.subplots(nrows=nrows, ncols=ncols,figsize=(14,14))

for i in list_of_nmpys_for_graphs:
    
#     numpy_array = i.to_numpy()
    
    figure.tight_layout(rect=[0, 0.03, 1, 0.95])
    
    axs[row_counter,col_counter].scatter(i[:, 5], i[:, 4], c =labels_dict[f'fit_{fit_number}'], cmap='rainbow', alpha=0.7 )
    axs[row_counter,col_counter].set_xlabel('Time of the day')
    axs[row_counter,col_counter].set_ylim(0, 750)
    axs[row_counter,col_counter].set_ylabel('Car number')
    axs[row_counter,col_counter].set_title(f'Fit {fit_number}')
    
    if fit_number >= 9:
        axs[row_counter,col_counter].axvline(x=143, color='g')
    if col_counter == (ncols-1):
        col_counter = 0
        row_counter += 1
    else:
        col_counter += 1
    
    fit_number += 1
model_005_before = Birch(n_clusters=None, threshold=0.05)

list_of_nmpys_for_graphs = []

removed_outliers_numpy = []
list_of_nmpys_for_graphs = []

labels_dict = {}
label_1_dict = {}
label_0_dict = {}
add_data_to_this = pd.DataFrame(columns = columns)

fit_number = 1

list_of_dfs

for i in range(0, len(list_of_dfs)):
    numpy_array = outliers_removal(list_of_dfs[i])
    
    model_005_before.partial_fit(numpy_array[:, :-2])
    
    model_copy = copy.deepcopy(model_005_before)
    
    removed_outliers_numpy.append(numpy_array)
    new_nmpy = np.concatenate(removed_outliers_numpy, axis=0)
    
    n = plot_silhouette_scores(model_copy, new_nmpy[:, :-2], 2, 10, fit_number)
#     print(n)
    
    model_copy.set_params(n_clusters = n)
    model_copy.partial_fit()
    
    labels_dict[f'fit_{fit_number}'] = model_copy.predict(new_nmpy[:, :-2])
    unique, counts = np.unique(labels_dict[f'fit_{fit_number}'], return_counts=True)
    label_1_dict[f'fit_{fit_number}'] = counts[1]
    label_0_dict[f'fit_{fit_number}'] = counts[0]
#     print(collections.Counter(labels_dict[f'fit_{fit_number}']))
#     print(get_cal_score(new_nmpy[:, :-2], labels_dict[f'fit_{fit_number}']))
    list_of_nmpys_for_graphs.append(new_nmpy)
    fit_number += 1
    
for i in label_0_dict.values():
    print(i)
    
print('\n')

for i in label_1_dict.values():
    print(i)
model_1_before = Birch(n_clusters=None, threshold=0.1)

list_of_nmpys_for_graphs = []

removed_outliers_numpy = []
list_of_nmpys_for_graphs = []

labels_dict = {}
label_1_dict = {}
label_0_dict = {}
add_data_to_this = pd.DataFrame(columns = columns)

fit_number = 1

list_of_dfs

for i in range(0, len(list_of_dfs)):
    numpy_array = outliers_removal(list_of_dfs[i])
    
    model_1_before.partial_fit(numpy_array[:, :-2])
    
    model_copy = copy.deepcopy(model_1_before)
    
    removed_outliers_numpy.append(numpy_array)
    new_nmpy = np.concatenate(removed_outliers_numpy, axis=0)
    
    n = plot_silhouette_scores(model_copy, new_nmpy[:, :-2], 2, 10, fit_number)
#     print(n)
    
    model_copy.set_params(n_clusters = n)
    model_copy.partial_fit()
    
    labels_dict[f'fit_{fit_number}'] = model_copy.predict(new_nmpy[:, :-2])
    unique, counts = np.unique(labels_dict[f'fit_{fit_number}'], return_counts=True)
    label_1_dict[f'fit_{fit_number}'] = counts[1]
    label_0_dict[f'fit_{fit_number}'] = counts[0]
#     print(collections.Counter(labels_dict[f'fit_{fit_number}']))
#     print(get_cal_score(new_nmpy[:, :-2], labels_dict[f'fit_{fit_number}']))
    list_of_nmpys_for_graphs.append(new_nmpy)
    fit_number += 1
    
for i in label_0_dict.values():
    print(i)
    
print('\n')

for i in label_1_dict.values():
    print(i)
model_3_before = Birch(n_clusters=None, threshold=0.3)

list_of_nmpys_for_graphs = []

removed_outliers_numpy = []
list_of_nmpys_for_graphs = []

labels_dict = {}
label_1_dict = {}
label_0_dict = {}
add_data_to_this = pd.DataFrame(columns = columns)

fit_number = 1

list_of_dfs

for i in range(0, len(list_of_dfs)):
    numpy_array = outliers_removal(list_of_dfs[i])
    
    model_3_before.partial_fit(numpy_array[:, :-2])
    
    model_copy = copy.deepcopy(model_3_before)
    
    removed_outliers_numpy.append(numpy_array)
    new_nmpy = np.concatenate(removed_outliers_numpy, axis=0)
    
    n = plot_silhouette_scores(model_copy, new_nmpy[:, :-2], 2, 10, fit_number)
#     print(n)
    
    model_copy.set_params(n_clusters = n)
    model_copy.partial_fit()
    
    labels_dict[f'fit_{fit_number}'] = model_copy.predict(new_nmpy[:, :-2])
    unique, counts = np.unique(labels_dict[f'fit_{fit_number}'], return_counts=True)
    try:
        label_1_dict[f'fit_{fit_number}'] = counts[1]
    except:
        label_1_dict[f'fit_{fit_number}'] = 0
    label_0_dict[f'fit_{fit_number}'] = counts[0]
#     print(collections.Counter(labels_dict[f'fit_{fit_number}']))
#     print(get_cal_score(new_nmpy[:, :-2], labels_dict[f'fit_{fit_number}']))
    list_of_nmpys_for_graphs.append(new_nmpy)
    fit_number += 1
    
for i in label_0_dict.values():
    print(i)
    
print('\n')

for i in label_1_dict.values():
    print(i)
model_5_before = Birch(n_clusters=None, threshold=0.5)

list_of_nmpys_for_graphs = []

removed_outliers_numpy = []
list_of_nmpys_for_graphs = []

labels_dict = {}
label_1_dict = {}
label_0_dict = {}
add_data_to_this = pd.DataFrame(columns = columns)

fit_number = 1

list_of_dfs

for i in range(0, len(list_of_dfs)):
    numpy_array = outliers_removal(list_of_dfs[i])
    
    model_5_before.partial_fit(numpy_array[:, :-2])
    
    model_copy = copy.deepcopy(model_5_before)
    
    removed_outliers_numpy.append(numpy_array)
    new_nmpy = np.concatenate(removed_outliers_numpy, axis=0)
    
    n = plot_silhouette_scores(model_copy, new_nmpy[:, :-2], 2, 10, fit_number)
#     print(n)
    
    model_copy.set_params(n_clusters = n)
    model_copy.partial_fit()
    
    labels_dict[f'fit_{fit_number}'] = model_copy.predict(new_nmpy[:, :-2])
    unique, counts = np.unique(labels_dict[f'fit_{fit_number}'], return_counts=True)
    try:
        label_1_dict[f'fit_{fit_number}'] = counts[1]
    except:
        label_1_dict[f'fit_{fit_number}'] = 0
    label_0_dict[f'fit_{fit_number}'] = counts[0]
#     print(collections.Counter(labels_dict[f'fit_{fit_number}']))
#     print(get_cal_score(new_nmpy[:, :-2], labels_dict[f'fit_{fit_number}']))
    list_of_nmpys_for_graphs.append(new_nmpy)
    fit_number += 1
    
for i in label_0_dict.values():
    print(i)
    
print('\n')

for i in label_1_dict.values():
    print(i)<jupyter_output>0
0
0
0
0
0
0.44263196232739926
0.44667732378280317
0.9512186336393842
0.9356075505109377
{'fraction_non_alphanumeric': 0.13716077201920251, 'fraction_numerical': 0.030273341824238267, 'mean_word_length': 4.389651531151003, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '152854', 'license': 'MIT', 'n_tokens_mistral': 4370, 'n_tokens_neox': 4206, 'n_words': 560, 'path': 'experiments/repetition2/Birch SIL SOCRES.ipynb', 'repo_name': 'martaradzi/RTX', 'size': '10207'}