<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2016, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
import numpy as np
from matplotlib import pyplot as plt

from utils import (find_cluster_assignments,
                   cluster_distance_matrix)



def plot_inter_sequence_distances(output_dir,
                                  plot_id,
                                  distance_func,
                                  sdrs,
                                  cluster_ids,
                                  ignore_noise):
  cluster_assignments, sdr_slices = find_cluster_assignments(sdrs, cluster_ids,
                                                             ignore_noise)

  distance_mat = cluster_distance_matrix(sdr_slices, distance_func)

  title = 'distance_matrix_%s' % plot_id
  outputFile = '%s/%s' % (output_dir, '%s.png' % title)
  plot_distance_mat(distance_mat, title, outputFile)



def plot_distance_mat(distance_mat, title, output_file):
  plt.figure()
  plt.imshow(distance_mat, interpolation="nearest")
  plt.colorbar()<commit_msg>plot 2D projections (tSNE / MDS) for each snapshot of the clustering run<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2016, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
import numpy as np
from collections import OrderedDict
from matplotlib import pyplot as plt
from matplotlib import colors

from utils import (find_cluster_assignments,
                   cluster_distance_matrix,
                   project_clusters_2D)



def plot_inter_sequence_distances(output_dir,
                                  plot_id,
                                  distance_func,
                                  sdrs,
                                  cluster_ids,
                                  ignore_noise):
  cluster_assignments, sdr_slices = find_cluster_assignments(sdrs, cluster_ids,
                                                             ignore_noise)

  distance_mat = cluster_distance_matrix(sdr_slices, distance_func)

  title = 'distance_matrix_%s' % plot_id
  output_file = '%s/%s' % (output_dir, '%s.png' % title)
  plot_distance_mat(distance_mat, title, output_file)

  projections = project_clusters_2D(distance_mat, method='mds')
  title = '2d_projections_%s' % plot_id
  output_file = '%s/%s' % (output_dir, '%s.png' % title)
  plot_2D_projections(title, output_file, cluster_assignments, projections)


def plot_2D_projections(title, output_file, cluster_assignments, projections):
  """
  Visualize SDR cluster projections
  """

  color_list = colors.cnames.keys()
  plt.figure()
  color_list = color_list
  color_names = []
  for i in range(len(cluster_assignments)):
    cluster_id = int(cluster_assignments[i])
    if cluster_id not in color_names:
      color_names.append(cluster_id)
    projection = projections[i]
    label = 'Category %s' % cluster_id
    if len(color_list) > cluster_id:
      color = color_list[cluster_id]
    else:
      color = 'black'
    plt.scatter(projection[0], projection[1], label=label, alpha=0.5,
                color=color, marker='o', edgecolor='black')

  # Add nicely formatted legend
  handles, labels = plt.gca().get_legend_handles_labels()
  by_label = OrderedDict(zip(labels, handles))
  plt.legend(by_label.values(), by_label.keys(), scatterpoints=1, loc=2)

  plt.title(title)
  plt.draw()
  plt.savefig(output_file)
  print('==> saved: %s' % output_file)


def plot_distance_mat(distance_mat, title, output_file):
  plt.figure()
  plt.imshow(distance_mat, interpolation="nearest")
  plt.colorbar()
{'fraction_non_alphanumeric': 0.1298895751450496, 'fraction_numerical': 0.004117536964252292, 'mean_word_length': 2.4256410256410255, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '393034', 'n_tokens_mistral': 1498, 'n_tokens_neox': 1435, 'n_words': 449}