<jupyter_start><jupyter_text>The Hopkins statistic is a way of measuring the cluster tendency of a data set. It belongs to the family of sparse sampling tests.<jupyter_code># dataset = pd.read_csv('./Absenteeism_at_work.csv', sep=';')
dataset = pd.read_pickle('./state_clubbed_df.pickle')
sampling_size = 350
rows, columns = dataset.shape
print ("Total rows : {}\t Total columns: {}".format(rows, columns))
dataset.head()
# Scaling and Preprocessing the dataset

dataset.drop(['state', 'year', 'total'], axis=1, inplace=True)
dataset = pd.DataFrame(scale(dataset))
# Sample n observations from D : P

if sampling_size > dataset.shape[0]:
    raise Exception(
        'The number of sample of sample is bigger than the shape of D')

sample_dataset = dataset.sample(n=sampling_size)
sample_dataset.shape
sample_dataset.head()
# Get the distance to their neirest neighbors in D : X

tree = BallTree(dataset, leaf_size=2)
dist, _ = tree.query(sample_dataset, k=2)
sample_knn_dist = dist[:, 1]
sample_knn_dist
# Randomly simulate n points with the same variation as in D : Q.

max_data = dataset.max()
min_data = dataset.min()

uniform_sel_x = np.random.uniform(min_data[0], max_data[0], sampling_size)
uniform_sel_y = np.random.uniform(min_data[1], max_data[1], sampling_size)

uniform_obsv = np.column_stack((uniform_sel_x, uniform_sel_y))
if len(max_data) >= 2:
    for i in range(2, len(max_data)):
        temp = np.random.uniform(min_data[i], max_data[i], sampling_size)
        to_stack = (uniform_obsv, temp)
        uniform_obsv = np.column_stack(to_stack)

uniform_obsv_df = pd.DataFrame(uniform_obsv)
uniform_obsv_df.columns = dataset.columns
uniform_obsv_df.shape
uniform_obsv_df.head()
# Get the distance to their neirest neighbors in D : Y

tree = BallTree(dataset, leaf_size=2)
dist, _ = tree.query(uniform_obsv_df, k=1)
uniform_knn_dist = dist
# Calculate the Hopkins Score

x = sum(sample_knn_dist)
y = sum(uniform_knn_dist)

if x + y == 0:
    raise Exception('The denominator of the hopkins statistics is null')

h_stat = (y[0] / (x + y)[0])
print ("The hopkins statistics measure is {}".format(h_stat))<jupyter_output>The hopkins statistics measure is 0.9571890578451321
{'fraction_non_alphanumeric': 0.1148741418764302, 'fraction_numerical': 0.016475972540045767, 'mean_word_length': 5.263610315186247, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '584321', 'license': 'MIT', 'n_tokens_mistral': 767, 'n_tokens_neox': 742, 'n_words': 208, 'path': 'Hopkins_statistics.ipynb', 'repo_name': 'rynaa/crime-analysis-report-1', 'size': '2185'}