<commit_before># yellowbrick.text.tsne
# Implements TSNE visualizations of documents in 2D space.
#
# Author:   <NAME> <<EMAIL>>
# Created:  Mon Feb 20 06:33:29 2017 -0500
#
# Copyright (C) 2016 B<EMAIL>
# For license information, see LICENSE.txt
#
# ID: tsne.py [] <EMAIL> $

"""
Implements TSNE visualizations of documents in 2D space.
"""

##########################################################################
## Imports
##########################################################################

import numpy as np
import matplotlib.pyplot as plt

from collections import defaultdict

from yellowbrick.text.base import TextVisualizer
from yellowbrick.exceptions import YellowbrickValueError
from yellowbrick.style.colors import resolve_colors, get_color_cycle

from sklearn.manifold import TSNE
from sklearn.pipeline import Pipeline
from sklearn.decomposition import TruncatedSVD, PCA

##########################################################################
## Quick Methods
##########################################################################

def tsne(X, y=None, ax=None, decompose='svd', decompose_by=50, classes=None,
           colors=None, colormap=None, **kwargs):
    """
    Display a projection of a vectorized corpus in two dimensions using TSNE,
    a nonlinear dimensionality reduction method that is particularly well
    suited to embedding in two or three dimensions for visualization as a
    scatter plot. TSNE is widely used in text analysis to show clusters or
    groups of documents or utterances and their relative proximities.

    Parameters
    ----------

    X : ndarray or DataFrame of shape n x m
        A matrix of n instances with m features representing the corpus of
        vectorized documents to visualize with tsne.

    y : ndarray or Series of length n
        An optional array or series of target or class values for instances.
        If this is specified, then the points will be colored according to
        their class. Often cluster labels are passed in to color the documents
        in cluster space, so this method is used both for classification and
        clustering methods.

    ax : matplotlib axes
        The axes to plot the figure on.

    decompose : string or None
        A preliminary decomposition is often used prior to TSNE to make the
        projection faster. Specify `"svd"` for sparse data or `"pca"` for
        dense data. If decompose is None, the original data set will be used.

    decompose_by : int
        Specify the number of components for preliminary decomposition, by
        default this is 50; the more components, the slower TSNE will be.

    classes : list of strings
        The names of the classes in the target, used to create a legend.

    colors : list or tuple of colors
        Specify the colors for each individual class

    colormap : string or matplotlib cmap
        Sequential colormap for continuous target

    kwargs : dict
        Pass any additional keyword arguments to the TSNE transformer.

    Returns
    -------
    ax : matplotlib axes
        Returns the axes that the parallel coordinates were drawn on.
    """
    # Instantiate the visualizer
    visualizer = TSNEVisualizer(
        ax, decompose, decompose_by, classes, colors, colormap, **kwargs
    )

    # Fit and transform the visualizer (calls draw)
    visualizer.fit(X, y, **kwargs)
    visualizer.transform(X)

    # Return the axes object on the visualizer
    return visualizer.ax


##########################################################################
## TSNEVisualizer
##########################################################################

class TSNEVisualizer(TextVisualizer):
    """
    Display a projection of a vectorized corpus in two dimensions using TSNE,
    a nonlinear dimensionality reduction method that is particularly well
    suited to embedding in two or three dimensions for visualization as a
    scatter plot. TSNE is widely used in text analysis to show clusters or
    groups of documents or utterances and their relative proximities.

    TSNE will return a scatter plot of the vectorized corpus, such that each
    point represents a document or utterance. The distance between two points
    in the visual space is embedded using the probability distribution of
    pairwise similarities in the higher dimensionality; thus TSNE shows
    clusters of similar documents and the relationships between groups of
    documents as a scatter plot.

    TSNE can be used with either clustering or classification; by specifying
    the ``classes`` argument, points will be colored based on their similar
    traits. For example, by passing cluster.labels_ as ``y`` in ``fit()``, all
    points in the same cluster will be grouped together. This extends the
    neighbor embedding with more information about similarity, and can allow
    better interpretation of both clusters and classes.

    For more, see https://lvdmaaten.github.io/tsne/

    Parameters
    ----------

    ax : matplotlib axes
        The axes to plot the figure on.

    decompose : string or None
        A preliminary decomposition is often used prior to TSNE to make the
        projection faster. Specify `"svd"` for sparse data or `"pca"` for
        dense data. If decompose is None, the original data set will be used.

    decompose_by : int
        Specify the number of components for preliminary decomposition, by
        default this is 50; the more components, the slower TSNE will be.

    classes : list of strings
        The names of the classes in the target, used to create a legend.

    colors : list or tuple of colors
        Specify the colors for each individual class
<commit_msg>repairs underscore issue in tsne init method that was causing sphinxdocs build warnings<commit_after># yellowbrick.text.tsne
# Implements TSNE visualizations of documents in 2D space.
#
# Author:   <NAME> <<EMAIL>>
# Created:  Mon Feb 20 06:33:29 2017 -0500
#
# Copyright (C) 2016 Bengfort.com
# For license information, see LICENSE.txt
#
# ID: tsne.py [] <EMAIL> $

"""
Implements TSNE visualizations of documents in 2D space.
"""

##########################################################################
## Imports
##########################################################################

import numpy as np
import matplotlib.pyplot as plt

from collections import defaultdict

from yellowbrick.text.base import TextVisualizer
from yellowbrick.exceptions import YellowbrickValueError
from yellowbrick.style.colors import resolve_colors, get_color_cycle

from sklearn.manifold import TSNE
from sklearn.pipeline import Pipeline
from sklearn.decomposition import TruncatedSVD, PCA

##########################################################################
## Quick Methods
##########################################################################

def tsne(X, y=None, ax=None, decompose='svd', decompose_by=50, classes=None,
           colors=None, colormap=None, **kwargs):
    """
    Display a projection of a vectorized corpus in two dimensions using TSNE,
    a nonlinear dimensionality reduction method that is particularly well
    suited to embedding in two or three dimensions for visualization as a
    scatter plot. TSNE is widely used in text analysis to show clusters or
    groups of documents or utterances and their relative proximities.

    Parameters
    ----------

    X : ndarray or DataFrame of shape n x m
        A matrix of n instances with m features representing the corpus of
        vectorized documents to visualize with tsne.

    y : ndarray or Series of length n
        An optional array or series of target or class values for instances.
        If this is specified, then the points will be colored according to
        their class. Often cluster labels are passed in to color the documents
        in cluster space, so this method is used both for classification and
        clustering methods.

    ax : matplotlib axes
        The axes to plot the figure on.

    decompose : string or None
        A preliminary decomposition is often used prior to TSNE to make the
        projection faster. Specify `"svd"` for sparse data or `"pca"` for
        dense data. If decompose is None, the original data set will be used.

    decompose_by : int
        Specify the number of components for preliminary decomposition, by
        default this is 50; the more components, the slower TSNE will be.

    classes : list of strings
        The names of the classes in the target, used to create a legend.

    colors : list or tuple of colors
        Specify the colors for each individual class

    colormap : string or matplotlib cmap
        Sequential colormap for continuous target

    kwargs : dict
        Pass any additional keyword arguments to the TSNE transformer.

    Returns
    -------
    ax : matplotlib axes
        Returns the axes that the parallel coordinates were drawn on.
    """
    # Instantiate the visualizer
    visualizer = TSNEVisualizer(
        ax, decompose, decompose_by, classes, colors, colormap, **kwargs
    )

    # Fit and transform the visualizer (calls draw)
    visualizer.fit(X, y, **kwargs)
    visualizer.transform(X)

    # Return the axes object on the visualizer
    return visualizer.ax


##########################################################################
## TSNEVisualizer
##########################################################################

class TSNEVisualizer(TextVisualizer):
    """
    Display a projection of a vectorized corpus in two dimensions using TSNE,
    a nonlinear dimensionality reduction method that is particularly well
    suited to embedding in two or three dimensions for visualization as a
    scatter plot. TSNE is widely used in text analysis to show clusters or
    groups of documents or utterances and their relative proximities.

    TSNE will return a scatter plot of the vectorized corpus, such that each
    point represents a document or utterance. The distance between two points
    in the visual space is embedded using the probability distribution of
    pairwise similarities in the higher dimensionality; thus TSNE shows
    clusters of similar documents and the relationships between groups of
    documents as a scatter plot.

    TSNE can be used with either clustering or classification; by specifying
    the ``classes`` argument, points will be colored based on their similar
    traits. For example, by passing ``cluster.labels_`` as ``y`` in ``fit()``, all
    points in the same cluster will be grouped together. This extends the
    neighbor embedding with more information about similarity, and can allow
    better interpretation of both clusters and classes.

    For more, see https://lvdmaaten.github.io/tsne/

    Parameters
    ----------

    ax : matplotlib axes
        The axes to plot the figure on.

    decompose : string or None
        A preliminary decomposition is often used prior to TSNE to make the
        projection faster. Specify `"svd"` for sparse data or `"pca"` for
        dense data. If decompose is None, the original data set will be used.

    decompose_by : int
        Specify the number of components for preliminary decomposition, by
        default this is 50; the more components, the slower TSNE will be.

    classes : list of strings
        The names of the classes in the target, used to create a legend.

    colors : list or tuple of colors
        Specify the colors for each individual class

{'fraction_non_alphanumeric': 0.12156760514424748, 'fraction_numerical': 0.004866180048661801, 'mean_word_length': 3.4129601226993866, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1326862', 'n_tokens_mistral': 2876, 'n_tokens_neox': 2660, 'n_words': 1384}