<commit_before>import numpy as np
cimport numpy as np
import sys
DTYPE = np.float64
ctypedef np.float64_t DTYPE_t

#function to do pca on an array of vectors then reduce dimensionality. Always converts data into columns of data
def pca(np.ndarray[DTYPE_t,ndim=2] vectors,int dim,str rowColumn):
    """Does principal component analysis on vectors and reduces dimensionality to dim.

    Args:
        vectors: Data to do pca on.
        dim: Dimensionality to reduce to.
        rowColumn: Flag that specifies how data is formatted.

    Returns:
        reduced: Data with rediced dimensions.
        eValues: Eigenvalues.
        eVectors: Eigenvectors.
        meanVec: Average of original vectors.

     Raises:
     """

    cdef np.ndarray[DTYPE_t,ndim=2] vectors2,centerVecs,covMat,eVectors,reducedDim
    cdef np.ndarray[DTYPE_t,ndim=1] meanVec,eValues
    cdef np.ndarray[np.int_t,ndim=1] idx
    cdef int ii
    if rowColumn == 'c':
        vectors2 = vectors
    elif rowColumn == 'r':
        vectors2 = vectors.T
    else:
        print 'Malformed rowColumn flag.'
        sys.exit()

    centerVecs = np.zeros_like(vectors2,dtype=DTYPE)
#Mean of each row
    meanVec = vectors2.mean(axis=1)
#Subtract mean
    for ii in xrange(vectors2.shape[1]):
        centerVecs[:,ii] = vectors2[:,ii]-meanVec
#Compute covariance matrix
    covMat = np.dot(centerVecs,centerVecs.T)
#Compute eigen info
    eValues,eVectors = np.linalg.eigh(covMat)
    idx = np.argsort(eValues)
    eValues = eValues[idx]
    eVectors = eVectors[:,idx]
#Project back onto a reduced dimensionality basis
    reducedDim = np.array([np.dot(np.array([eVectors[:,-1-ii]]),centerVecs)[0] for ii in xrange(dim)])

#Transpose back to original
    if rowColumn == 'r':
        reducedDim = reducedDim.T
    return (reducedDim,eValues,eVectors,meanVec)

def reconst(np.ndarray[DTYPE_t,ndim=2] reducedDim,np.ndarray[DTYPE_t,ndim=2] eVectors,np.ndarray[DTYPE_t,ndim=2] meanVec,str rowColumn):
    """Takes vectors from reduced dimensionality basis and returns them to full dimensionality basis.

    Args:
        reducedDim: Vectors of reduced dimensionality.
        eVectors: Original basis vectors.
        meanVec: Mean of original vectors.
        rowColumn: Flag that determines data format.<commit_msg>fixed dim of reconstitute<commit_after>import numpy as np
cimport numpy as np
import sys
DTYPE = np.float64
ctypedef np.float64_t DTYPE_t

#function to do pca on an array of vectors then reduce dimensionality. Always converts data into columns of data
def pca(np.ndarray[DTYPE_t,ndim=2] vectors,int dim,str rowColumn):
    """Does principal component analysis on vectors and reduces dimensionality to dim.

    Args:
        vectors: Data to do pca on.
        dim: Dimensionality to reduce to.
        rowColumn: Flag that specifies how data is formatted.

    Returns:
        reduced: Data with rediced dimensions.
        eValues: Eigenvalues.
        eVectors: Eigenvectors.
        meanVec: Average of original vectors.

     Raises:
     """

    cdef np.ndarray[DTYPE_t,ndim=2] vectors2,centerVecs,covMat,eVectors,reducedDim
    cdef np.ndarray[DTYPE_t,ndim=1] meanVec,eValues
    cdef np.ndarray[np.int_t,ndim=1] idx
    cdef int ii
    if rowColumn == 'c':
        vectors2 = vectors
    elif rowColumn == 'r':
        vectors2 = vectors.T
    else:
        print 'Malformed rowColumn flag.'
        sys.exit()

    centerVecs = np.zeros_like(vectors2,dtype=DTYPE)
#Mean of each row
    meanVec = vectors2.mean(axis=1)
#Subtract mean
    for ii in xrange(vectors2.shape[1]):
        centerVecs[:,ii] = vectors2[:,ii]-meanVec
#Compute covariance matrix
    covMat = np.dot(centerVecs,centerVecs.T)
#Compute eigen info
    eValues,eVectors = np.linalg.eigh(covMat)
    idx = np.argsort(eValues)
    eValues = eValues[idx]
    eVectors = eVectors[:,idx]
#Project back onto a reduced dimensionality basis
    reducedDim = np.array([np.dot(np.array([eVectors[:,-1-ii]]),centerVecs)[0] for ii in xrange(dim)])

#Transpose back to original
    if rowColumn == 'r':
        reducedDim = reducedDim.T
    return (reducedDim,eValues,eVectors,meanVec)

def reconst(np.ndarray[DTYPE_t,ndim=2] reducedDim,np.ndarray[DTYPE_t,ndim=2] eVectors,np.ndarray[DTYPE_t,ndim=1] meanVec,str rowColumn):
    """Takes vectors from reduced dimensionality basis and returns them to full dimensionality basis.

    Args:
        reducedDim: Vectors of reduced dimensionality.
        eVectors: Original basis vectors.
        meanVec: Mean of original vectors.
        rowColumn: Flag that determines data format.
{'fraction_non_alphanumeric': 0.09222027972027973, 'fraction_numerical': 0.009615384615384616, 'mean_word_length': 3.4264990328820115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7596883', 'n_tokens_mistral': 1527, 'n_tokens_neox': 1469, 'n_words': 422}