<commit_before>import random
import math
import matplotlib.pyplot as plt
import numpy as np

# TODO: test!!! especially that width is what you think it is!
def random_ring(n, d, width):
    
    scale = 1.0/(1 - (1 - width)**d)
    
    samples = [[0]*d for i in range(n)]
    
    for i in range(n):
        
        for j in range(d):
            samples[i][j] = random.gauss(0,1)
            
        r = math.sqrt(sum(map(lambda x: x**2, samples[i])))
        u = (1.0 - random.random()/scale)**(1.0/d)
        samples[i] = map(lambda x: u * 1/r * x, samples[i])
    
    return samples

def correlated_data(n, corr, dim = 2):
    mean = [0]*dim
    cov = [[corr]*dim for i in range(dim)]
    for i in range(dim):
        cov[i][i] = 1
    samples = np.random.multivariate_normal(mean, cov, n)
    return samples

def correlated_pairs(n, pairs, corr):
    
    d = pairs*2
    samples = [[0]*(d + 1) for i in range(n)]

    for i in range(pairs):
        next = correlated_data(n, corr)
        for j in range(n):
            samples[j][(2*i):(2*i + 1)] = next[j]
    
    return samples
<commit_msg>Fixed bug in synthetic data<commit_after>import random
import math
import matplotlib.pyplot as plt
import numpy as np

# TODO: test!!! especially that width is what you think it is!
def random_ring(n, d, width):
    
    scale = 1.0/(1 - (1 - width)**d)
    
    samples = [[0]*d for i in range(n)]
    
    for i in range(n):
        
        for j in range(d):
            samples[i][j] = random.gauss(0,1)
            
        r = math.sqrt(sum(map(lambda x: x**2, samples[i])))
        u = (1.0 - random.random()/scale)**(1.0/d)
        samples[i] = map(lambda x: u * 1/r * x, samples[i])
    
    return samples

def correlated_data(n, corr, dim = 2):
    mean = [0]*dim
    cov = [[corr]*dim for i in range(dim)]
    for i in range(dim):
        cov[i][i] = 1
    samples = np.random.multivariate_normal(mean, cov, n)
    return samples

def correlated_pairs(n, pairs, corr):
    
    d = pairs*2
    samples = [[0]*d for i in range(n)]

    for i in range(pairs):
        next_cols = correlated_data(n, corr)
        for j in range(n):
            samples[j][(2*i):(3*i)] = next_cols[j]
    
    return samples

{'fraction_non_alphanumeric': 0.1476147614761476, 'fraction_numerical': 0.018901890189018902, 'mean_word_length': 2.2170767004341534, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1349543', 'n_tokens_mistral': 823, 'n_tokens_neox': 798, 'n_words': 238}