<commit_before>import pandas as pd
import numpy as np
from sklearn.preprocessing import scale
from sklearn.decomposition import PCA as PCA1
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from scipy import stats
import matplotlib.pyplot as plt
from pySPM.SPM import SPM_image
import matplotlib as mpl
from matplotlib import cm
from pySPM import collection


class PCA:

    def __init__(self, data):
        self.data = data
        self.pca = None
        self.standX = None

    def scatter(self, **kargs):
        pd.tools.plotting.scatter_matrix(self.data, diagonal="kde", **kargs)
        plt.tight_layout()

    def corr(self):
        corrmat = self.data.corr()
        return corrmat

    def corrShow(self):
        import seaborn as sns
        corrmat = self.corr()
        sns.heatmap(corrmat).xaxis.tick_top()

    def hinton(self, max_weight=None, ax=None, matrix = None, xlabel=None, ylabel=None):
        """Draw Hinton diagram for visualizing a weight matrix."""
        if matrix is None:
            matrix = self.corr()
        ax = ax if ax is not None else plt.gca()

        if not max_weight:
            max_weight = 2**np.ceil(np.log(np.abs(matrix).max())/np.log(2))

        ax.patch.set_facecolor('lightgray')
        ax.set_aspect('equal', 'box')
        ax.xaxis.set_major_locator(plt.NullLocator())
        ax.yaxis.set_major_locator(plt.NullLocator())

        for (x, y), w in np.ndenumerate(matrix):
            color = 'red' if w > 0 else 'blue'
            size = np.sqrt(np.abs(w))
            rect = plt.Rectangle([y - size / 2, x - size / 2], size, size,
                                 facecolor=color, edgecolor=color)
            ax.add_patch(rect)

        nxticks = matrix.shape[1]
        nyticks = matrix.shape[0]
        if xlabel is None:
            xlabel = list(matrix.columns)
        if ylabel is None:
            ylabel = list(matrix.index)
        ax.xaxis.tick_top()
        ax.set_xticks(range(nxticks))
        ax.set_xticklabels(xlabel, rotation=90)
        ax.set_yticks(range(nyticks))
        ax.set_yticklabels(ylabel)
        ax.grid(False)

        ax.autoscale_view()
        ax.invert_yaxis()

    def standardized(self):
        self.standX = pd.DataFrame(
            scale(self.data), index=self.data.index, columns=self.data.columns)
        return self.standX

    def runPCA(self):
        if self.standX is None:<commit_msg>Improve label of hinton plot<commit_after>import pandas as pd
import numpy as np
from sklearn.preprocessing import scale
from sklearn.decomposition import PCA as PCA1
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from scipy import stats
import matplotlib.pyplot as plt
from pySPM.SPM import SPM_image
import matplotlib as mpl
from matplotlib import cm
from pySPM import collection
import re

class PCA:

    def __init__(self, data):
        self.data = data
        self.pca = None
        self.standX = None

    def scatter(self, **kargs):
        pd.tools.plotting.scatter_matrix(self.data, diagonal="kde", **kargs)
        plt.tight_layout()

    def corr(self):
        corrmat = self.data.corr()
        return corrmat

    def corrShow(self):
        import seaborn as sns
        corrmat = self.corr()
        sns.heatmap(corrmat).xaxis.tick_top()

    def hinton(self, max_weight=None, ax=None, matrix = None, xlabel=None, ylabel=None):
        """Draw Hinton diagram for visualizing a weight matrix."""
        if matrix is None:
            matrix = self.corr()
        ax = ax if ax is not None else plt.gca()

        if not max_weight:
            max_weight = 2**np.ceil(np.log(np.abs(matrix).max())/np.log(2))

        ax.patch.set_facecolor('lightgray')
        ax.set_aspect('equal', 'box')
        ax.xaxis.set_major_locator(plt.NullLocator())
        ax.yaxis.set_major_locator(plt.NullLocator())

        for (x, y), w in np.ndenumerate(matrix):
            color = 'red' if w > 0 else 'blue'
            size = np.sqrt(np.abs(w))
            rect = plt.Rectangle([y - size / 2, x - size / 2], size, size,
                                 facecolor=color, edgecolor=color)
            ax.add_patch(rect)

        nxticks = matrix.shape[1]
        nyticks = matrix.shape[0]
        if xlabel is None:
            xlabel = []
            for x in list(matrix.columns):
                x = re.sub(r'\^([0-9]+)',r'^{\1}', x)
                x = re.sub(r'_([0-9]+)',r'_{\1}', x)
                if x[-1] in ['+','-']:
                    x = x[:-1]+'^'+x[-1]
                xlabel.append('$'+x+'$')
        if ylabel is None:
            ylabel = list(matrix.index)
        ax.xaxis.tick_top()
        ax.set_xticks(range(nxticks))
        ax.set_xticklabels(xlabel, rotation=90)
        ax.set_yticks(range(nyticks))
        ax.set_yticklabels(ylabel)
        ax.grid(False)

        ax.autoscale_view()
        ax.invert_yaxis()

    def standardized(self):
        self.standX = pd.DataFrame(
            scale(self.data), index=self.data.index, columns=self.data.columns)
        return self.standX

    def runPCA(self):
        if self.standX is None:
{'fraction_non_alphanumeric': 0.11545223676499317, 'fraction_numerical': 0.005665168978316077, 'mean_word_length': 2.208020050125313, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '990026', 'n_tokens_mistral': 1721, 'n_tokens_neox': 1657, 'n_words': 394}