# Path: hub_toolbox/io.py
# def load_dexter():
# def check_is_nD_array(arr:np.ndarray, n:int, arr_type=''):
# def check_distance_matrix_shape(D:np.ndarray):
# def check_distance_matrix_shape_fits_vectors(D:np.ndarray, vectors:np.ndarray):
# def check_distance_matrix_shape_fits_labels(D:np.ndarray, classes:np.ndarray):
# def check_vector_matrix_shape_fits_labels(X:np.ndarray, classes:np.ndarray):
# def check_sample_shape_fits(D:np.ndarray, idx:np.ndarray):
# def check_valid_metric_parameter(metric:str):
# def matrix_split(rows, cols, elem_size=8, nr_matrices=4): # pragma: no cover
# def random_sparse_matrix(size, density=0.05):
# def save_csr_matrix(file, matrix):
# def load_csr_matrix(file):
#     def __init__(self, unit='kB'):
#     def _factor(self):
#     def total(self):
#     def used(self):
#     def used_real(self):
#     def shared(self):
#     def buffers(self):
#     def cached(self):
#     def user_free(self):
#     def swap(self):
#     def swap_free(self):
#     def swap_used(self):
#     D = cosine_distance(vectors)
#     S = sparse.rand(size, size, density, 'csr')
# class FreeMemLinux(object): # pragma: no cover
# 
# Path: hub_toolbox/htlogging.py
# class ConsoleLogging(Logging):
#     """Convenience functions for time-stamped logging to the console"""
# 
#     def message(self, *objs, flush=True):
#         """Log normal program function"""
#         print(self._current_time, 'INFO:', *objs)
#         if flush:
#             sys.stdout.flush()
# 
#     def warning(self, *objs, flush=True):
#         """Log warning (program can still continue)"""
#         print(self._current_time, 'WARNING:', *objs, file=sys.stderr)
#         if flush:
#             sys.stderr.flush()
# 
#     def error(self, *objs, flush=True):
#         """Log error (program fails)"""
#         print(self._current_time, 'ERROR:', *objs, file=sys.stderr)
#         if flush:
#             sys.stderr.flush()
# 
# Path: hub_toolbox/utils.py
# class SynchronizedCounter(object):
#     """ A multiprocessing-safe counter for progress information. """
#     def __init__(self, init:int=-1):
#         self.val = Value('i', init)
# 
#     def increment_and_get_value(self, n=1) -> int:
#         """ Obtain a lock before incrementing, since += isn't atomic. """
#         with self.val.get_lock():
#             self.val.value += n
#             return self.val.value
# 
#     @property
#     def value(self) -> int:
#         return self.val.value


import ctypes
import multiprocessing as mp
import numpy as np
from functools import partial
from multiprocessing import RawArray, Pool, cpu_count
from scipy import stats
from scipy.sparse.base import issparse
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.utils.extmath import row_norms
from sklearn.utils.validation import check_random_state
from hub_toolbox import io
from hub_toolbox.htlogging import ConsoleLogging
from hub_toolbox.utils import SynchronizedCounter
    from hub_toolbox.io import load_dexter
                             f" CPU cores. Was {n_jobs} instead.")
        if verbose < 0:
            raise ValueError(f"Verbosity level 'verbose' must be >= 0, "
                             f"but was {verbose}.")

    def _k_neighbors(self, X, Y, kth, n_test, start, end):
        assert end - start == self.k, f'Implementation error'
        X_norm = row_norms(X, squared=True)
        Y_norm = row_norms(Y, squared=True)
        try:
            batch_size = self.kwargs['batch_size']
        except KeyError:
            batch_size = 64
        n_batches = np.ceil(n_test / batch_size).astype(int)
        if self.n_jobs == 1:
            Dk = np.zeros((n_test, self.k), dtype=np.int32)
            for i in range(n_batches):
                if self.verbose > 1 \
                    or self.verbose > 0 and (i % 10 == 0 or i+1 == n_batches):
                    log.message(f'k neighbors (from vectors): batch '
                                f'{i+1}/{n_batches} with batch size '
                                f'{batch_size}.', flush=True)
                d = pairwise_distances(
                    X[i*batch_size:(i+1)*batch_size, :], Y, self.metric,
                    self.n_jobs, squared=True, X_norm_squared=X_norm[
                        i*batch_size:(i+1)*batch_size].reshape(1, -1),
                    Y_norm_squared=Y_norm)
                nn = np.argpartition(d, kth=kth)[:, start:end]
                Dk[i*batch_size:(i+1)*batch_size, :] = nn
        else:

# Path: hub_toolbox/io.py
# def load_dexter():
# def check_is_nD_array(arr:np.ndarray, n:int, arr_type=''):
# def check_distance_matrix_shape(D:np.ndarray):
# def check_distance_matrix_shape_fits_vectors(D:np.ndarray, vectors:np.ndarray):
# def check_distance_matrix_shape_fits_labels(D:np.ndarray, classes:np.ndarray):
# def check_vector_matrix_shape_fits_labels(X:np.ndarray, classes:np.ndarray):
# def check_sample_shape_fits(D:np.ndarray, idx:np.ndarray):
# def check_valid_metric_parameter(metric:str):
# def matrix_split(rows, cols, elem_size=8, nr_matrices=4): # pragma: no cover
# def random_sparse_matrix(size, density=0.05):
# def save_csr_matrix(file, matrix):
# def load_csr_matrix(file):
#     def __init__(self, unit='kB'):
#     def _factor(self):
#     def total(self):
#     def used(self):
#     def used_real(self):
#     def shared(self):
#     def buffers(self):
#     def cached(self):
#     def user_free(self):
#     def swap(self):
#     def swap_free(self):
#     def swap_used(self):
#     D = cosine_distance(vectors)
#     S = sparse.rand(size, size, density, 'csr')
# class FreeMemLinux(object): # pragma: no cover
# 
# Path: hub_toolbox/htlogging.py
# class ConsoleLogging(Logging):
#     """Convenience functions for time-stamped logging to the console"""
# 
#     def message(self, *objs, flush=True):
#         """Log normal program function"""
#         print(self._current_time, 'INFO:', *objs)
#         if flush:
#             sys.stdout.flush()
# 
#     def warning(self, *objs, flush=True):
#         """Log warning (program can still continue)"""
#         print(self._current_time, 'WARNING:', *objs, file=sys.stderr)
#         if flush:
#             sys.stderr.flush()
# 
#     def error(self, *objs, flush=True):
#         """Log error (program fails)"""
#         print(self._current_time, 'ERROR:', *objs, file=sys.stderr)
#         if flush:
#             sys.stderr.flush()
# 
# Path: hub_toolbox/utils.py
# class SynchronizedCounter(object):
#     """ A multiprocessing-safe counter for progress information. """
#     def __init__(self, init:int=-1):
#         self.val = Value('i', init)
# 
#     def increment_and_get_value(self, n=1) -> int:
#         """ Obtain a lock before incrementing, since += isn't atomic. """
#         with self.val.get_lock():
#             self.val.value += n
#             return self.val.value
# 
#     @property
#     def value(self) -> int:
#         return self.val.value


# Path: hub_toolbox/hubness.py
import ctypes
import multiprocessing as mp
import numpy as np
from functools import partial
from multiprocessing import RawArray, Pool, cpu_count
from scipy import stats
from scipy.sparse.base import issparse
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.utils.extmath import row_norms
from sklearn.utils.validation import check_random_state
from hub_toolbox import io
from hub_toolbox.htlogging import ConsoleLogging
from hub_toolbox.utils import SynchronizedCounter
    from hub_toolbox.io import load_dexter

                             f" CPU cores. Was {n_jobs} instead.")
        if verbose < 0:
            raise ValueError(f"Verbosity level 'verbose' must be >= 0, "
                             f"but was {verbose}.")

    def _k_neighbors(self, X, Y, kth, n_test, start, end):
        assert end - start == self.k, f'Implementation error'
        X_norm = row_norms(X, squared=True)
        Y_norm = row_norms(Y, squared=True)
        try:
            batch_size = self.kwargs['batch_size']
        except KeyError:
            batch_size = 64
        n_batches = np.ceil(n_test / batch_size).astype(int)
        if self.n_jobs == 1:
            Dk = np.zeros((n_test, self.k), dtype=np.int32)
            for i in range(n_batches):
                if self.verbose > 1 \
                    or self.verbose > 0 and (i % 10 == 0 or i+1 == n_batches):
                    log.message(f'k neighbors (from vectors): batch '
                                f'{i+1}/{n_batches} with batch size '
                                f'{batch_size}.', flush=True)
                d = pairwise_distances(
                    X[i*batch_size:(i+1)*batch_size, :], Y, self.metric,
                    self.n_jobs, squared=True, X_norm_squared=X_norm[
                        i*batch_size:(i+1)*batch_size].reshape(1, -1),
                    Y_norm_squared=Y_norm)
                nn = np.argpartition(d, kth=kth)[:, start:end]
                Dk[i*batch_size:(i+1)*batch_size, :] = nn
        else:

            counter = SynchronizedCounter()