<commit_before>###################################
# utility functions
###################################


# using DataStructures

# centers each column
function colCenter!(A::Matrix{Float64})

    (n,m) = size(A)

    # get mean of each column; convert to vector
    colMeans = mean(A,dims=1) |> vec

    for i=1:n
        for j=1:m
            A[i,j] = A[i,j] - colMeans[j]
        end
    end
end

# centers each row
function rowCenter!(A::Matrix{Float64})

    (n,m) = size(A)

    # get mean of each column; convert to vector
    rowMeans = mean(A,dims=2) |> vec

    for i=1:m
        for j=1:n
            A[j,i] = A[j,i] - rowMeans[j]
        end
    end
end

function colDivide!(A::Matrix{Float64},x::Vector{Float64})

    (n,m) = size(A)
    if(length(x)!=m)
        error("Matrix and vector size do not match.")
    end

    for i=1:n
        for j=1:m
            A[i,j] = A[i,j]/x[j]
        end
    end
end

function colStandardize!(A::Matrix{Float64})

    colCenter!(A)
    s = std(A,dims=1) |> vec
    colDivide!(A,s)

end

function rowDivide!(A::Matrix{Float64},x::Vector{Float64})

    (n,m) = size(A)
    if(length(x)!=n)
        error("Matrix and vector size do not match.")
    end

    for i=1:m
        for j=1:n
            A[j,i] = A[j,i]/x[j]
        end
    end
end

"""
perform random shuffles of vector
the first column is the original vector if original=true
"""
function shuffleVector(rng::AbstractRNG,x::Vector{Float64},
                 nshuffle::Int64,original::Bool=true)
    xx = zeros(length(x),nshuffle+1)
    xx[:,1] = x
    for i=1:nshuffle
        xx[:,i+1] = shuffle(rng,x)
    end

    return xx
end

function compareValues(x_true::Array{Float64,1}, x::Array{Float64,1}, tolerance::Float64, threshold::Float64)
    if size(x_true) != size(x)
        error("Dimention Mismatch! Must compare two arrays of same length!")
    end

    passes = falses(size(x_true))
    t_passes = falses(0)
    for i in 1:size(x_true)[1]
        e = abs(x[i]-x_true[i])
        if e <= tolerance
            passes[i] = true
        end
<commit_msg>Implement option for shuffleVector to include original data.<commit_after>###################################
# utility functions
###################################


# using DataStructures

# centers each column
function colCenter!(A::Matrix{Float64})

    (n,m) = size(A)

    # get mean of each column; convert to vector
    colMeans = mean(A,dims=1) |> vec

    for i=1:n
        for j=1:m
            A[i,j] = A[i,j] - colMeans[j]
        end
    end
end

# centers each row
function rowCenter!(A::Matrix{Float64})

    (n,m) = size(A)

    # get mean of each column; convert to vector
    rowMeans = mean(A,dims=2) |> vec

    for i=1:m
        for j=1:n
            A[j,i] = A[j,i] - rowMeans[j]
        end
    end
end

function colDivide!(A::Matrix{Float64},x::Vector{Float64})

    (n,m) = size(A)
    if(length(x)!=m)
        error("Matrix and vector size do not match.")
    end

    for i=1:n
        for j=1:m
            A[i,j] = A[i,j]/x[j]
        end
    end
end

function colStandardize!(A::Matrix{Float64})

    colCenter!(A)
    s = std(A,dims=1) |> vec
    colDivide!(A,s)

end

function rowDivide!(A::Matrix{Float64},x::Vector{Float64})

    (n,m) = size(A)
    if(length(x)!=n)
        error("Matrix and vector size do not match.")
    end

    for i=1:m
        for j=1:n
            A[j,i] = A[j,i]/x[j]
        end
    end
end

"""
perform random shuffles of vector
the first column is the original vector if original=true
"""
function shuffleVector(rng::AbstractRNG,x::Vector{Float64},
                 nshuffle::Int64,original::Bool=true)
    if(original)
        xx = zeros(length(x),nshuffle+1)
        xx[:,1] = x
        istart = 1
    else
        xx = zeros(length(x),nshuffle+1)
        istart = 0
    end

    for i=1:nshuffle
        xx[:,i+istart] = shuffle(rng,x)
    end

    return xx
end

function compareValues(x_true::Array{Float64,1}, x::Array{Float64,1}, tolerance::Float64, threshold::Float64)
    if size(x_true) != size(x)
        error("Dimention Mismatch! Must compare two arrays of same length!")
    end

    passes = falses(size(x_true))
    t_passes = falses(0)
    for i in 1:size(x_true)[1]
        e = abs(x[i]-x_true[i])
        if e <= tolerance
            passes[i] = true
        end

{'fraction_non_alphanumeric': 0.16882516188714153, 'fraction_numerical': 0.021739130434782608, 'mean_word_length': 2.5921926910299002, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5642478', 'n_tokens_mistral': 1634, 'n_tokens_neox': 1560, 'n_words': 368}