<commit_before>import scipy
import numpy as np


def hellopipilworld():
    return u'<NAME>! (Hello "World"! in Nahuatl.)'


def list2cls(in_list, name_of_out='output.cls'):
    """This function creates a CLS file from a list-like object"""
    # print("~~~~~"+str(metadata_subset.shape)+'~~~~~')
    cls = open(name_of_out, 'w')
    cls.write("{}\t{}\t1\n".format(len(in_list), len(np.unique(in_list))))
    cls.write("#\t{}\n".format("\t".join(np.unique(in_list).astype(str))))
    cls.write('\t'.join(in_list.astype(str))+'\n')
    cls.close()


def custom_pearson(x, y):
    return scipy.stats.pearsonr(x, y)[0]


def custom_spearman(x, y):
    return scipy.stats.spearmanr(x, y)[0]

<commit_msg>turning correlations into distances, only custom_pearson for now<commit_after>import scipy
import numpy as np


def hellopipilworld():
    return u'<NAME>! (Hello "World"! in Nahuatl.)'


def list2cls(in_list, name_of_out='output.cls'):
    """This function creates a CLS file from a list-like object"""
    # print("~~~~~"+str(metadata_subset.shape)+'~~~~~')
    cls = open(name_of_out, 'w')
    cls.write("{}\t{}\t1\n".format(len(in_list), len(np.unique(in_list))))
    cls.write("#\t{}\n".format("\t".join(np.unique(in_list).astype(str))))
    cls.write('\t'.join(in_list.astype(str))+'\n')
    cls.close()


def custom_pearson_corr(x, y):
    return scipy.stats.pearsonr(x, y)[0]


def custom_pearson_dist(x, y):
    return 1 - scipy.stats.pearsonr(x, y)[0]


def custom_spearman(x, y):
    return scipy.stats.spearmanr(x, y)[0]


{'fraction_non_alphanumeric': 0.20273794002607562, 'fraction_numerical': 0.00651890482398957, 'mean_word_length': 4.50179211469534, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '732668', 'n_tokens_mistral': 631, 'n_tokens_neox': 600, 'n_words': 110}