<jupyter_start><jupyter_text>Test OPTICS: Individual Tests<jupyter_code>import os, glob, getpass, sys
import warnings

import numpy as np
import matplotlib.pyplot as plt
from astropy.table import Table, join, vstack, hstack, Column, MaskedColumn, unique
from astropy.utils.exceptions import AstropyWarning

user = getpass.getuser()
sys.path.append('/Users/' + user + '/Dropbox/my_python_packages')
path = '../../'
from gaia.cluster_comparison_n import Comparison_n
from pyclustering.cluster.optics import optics, ordering_analyser
def set_plot(ftsize, title, xlim = [-10,2500], ylim = [0.15,1.5], xstick_step = 500, showlegend = True, showylabel = True):
    plt.ylim(ylim)
    plt.xlim(xlim)
    plt.xlabel('Point', fontsize = ftsize)
    if showylabel:
        plt.ylabel(r'Reachability Distance', fontsize = ftsize)
    plt.xticks(np.arange(np.min(xlim)+10, np.max(xlim)+1, xstick_step), fontsize = ftsize)
    plt.yticks(fontsize = ftsize)
    if showlegend:
        plt.legend(loc = 'upper right', fontsize = ftsize*0.8)
    plt.grid()
    plt.title(title, fontsize = ftsize)
# Read data ====================================
warnings.simplefilter('ignore', AstropyWarning)
data     = Table.read(path + 'sample_gaia/gaia_sample_cleaned.vot',       format = 'votable')
control  = Table.read(path + 'sample_control/OPH___control_sample.vot',   format = 'votable') ; control.label = 'Control'
data_scl = Table.read(path + 'sample_clusters/cl_scale/scl_standard.vot', format = 'votable')

# Optics data preparison ======================= 
data_scl = data_scl.to_pandas().values
# Compute dataset ordering =============================
mpts   = np.int(input('Introduce mpts: '))
eps_0  = 2.0
model  = optics(data_scl, eps_0, mpts, ccore=False) #ccore=False to make it work in MacOs !!
model.process()
orders = ordering_analyser(model.get_ordering())
# Make Plot ============================================
fig       = plt.figure(figsize=[25,9])
ftsize    = 36
linewidth = 3

#===============
plt.subplot(121)
plt.plot(orders.cluster_ordering, 'b-', label = 'mPts = ' + str(mpts), linewidth = linewidth)
pp = set_plot(ftsize, title='Gaia Sample', ylim = [0.05, 1.8], xlim=[-10,2250], xstick_step=750)
#===============
plt.subplot(122)
plt.plot(orders.cluster_ordering, 'b-', label = 'mPts = ' + str(mpts), linewidth = linewidth)
pp = set_plot(ftsize, title='Zoom-in', ylim = [0.1, 0.45], xlim=[0,1000], xstick_step=200, showlegend=False, showylabel=False)

plt.show()
fig.savefig('optics_rdist_mpts_' + np.str(mpts) + '.pdf',bbox_inches = 'tight', overwrite = True)
# Run OPTICS for the given eps =========================
eps      = np.float(input('Introduce eps: '))
model    = optics(data_scl, eps, mpts, ccore=False) #ccore=False to make it work in MacOs !!
model.process()
clusters = model.get_clusters()

print('==============================')
for cluster in clusters:
    print('Members detected: ', len(cluster))
    
# Make Plot ============================================
fig       = plt.figure(figsize=[15,8])
ftsize    = 22
linewidth = 2

plt.plot(orders.cluster_ordering, '-', label = 'mPts = ' + str(mpts), linewidth = linewidth, mew = 0)
pp = set_plot(ftsize, title='Gaia Sample', xlim=[-10,1000], ylim = [0.1, 0.35])
plt.hlines(xmin=-10,xmax=2000, y=eps, linestyles='--')
plt.show()
# Compare Clusters =============================
control.label = 'Control'
cl_list = []
index   = -1
for cluster in clusters:
    index = index + 1
    cl_inp = data[cluster]
    cl_inp.label = 'optics_' + np.str(index)
    cl_list.append(cl_inp)
#     cl_inp.write('optics_mpts_' + np.str(mpts) + '_' + np.str(index) + '.vot', format = 'votable', overwrite = True)

# Show Plots ============
final   = Comparison_n()
final.load_clusters(control=control, cluster_list=cl_list)
hists = final.plot_3_hist(x2_bins=5, x3_bins=5, linewidth=3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21722265321955003, 'fraction_numerical': 0.02430824928885441, 'mean_word_length': 5.110584518167457, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '348244', 'license': 'MIT', 'n_tokens_mistral': 1458, 'n_tokens_neox': 1370, 'n_words': 286, 'path': 'sample_clusters/cl_optics/01_run_optics.ipynb', 'repo_name': 'hectorcanovas/data_mining_Gaia_DR2', 'size': '3867'}