<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import numpy

from classification.classification_network import createNetwork
from fluent.encoders.cio_encoder import CioEncoder
from fluent.models.classification_model import ClassificationModel
from nupic.data.file_record_stream import FileRecordStream



class ClassificationModelHTM(ClassificationModel):
  """
  Class to run the survey response classification task with nupic network
  """

  def __init__(self, inputFilePath, verbosity=1, numLabels=3, spTrainingSize=0,
               tmTrainingSize=0, clsTrainingSize=0, classifierType="KNN"):
    """
    @param inputFilePath      (str)       Path to data formatted for network
                                          API
    @param spTrainingSize     (int)       Number of samples the network has to
                                          be trained on before training the
                                          spatial pooler
    @param tmTrainingSize     (int)       Number of samples the network has to
                                          be trained on before training the
                                          temporal memory
    @param clsTrainingSize    (int)       Number of samples the network has to
                                          be trained on before training the
                                          classifier
    @param classifierType     (str)       Either "KNN" or "CLA"
    See ClassificationModel for remaining parameters
    """
    self.spTrainingSize = spTrainingSize
    self.tmTrainingSize = tmTrainingSize
    self.clsTrainingSize = clsTrainingSize

    super(ClassificationModelHTM, self).__init__(verbosity=verbosity,
      numLabels=numLabels)

    # Initialize Network
    self.classifierType = classifierType
    self.recordStream = FileRecordStream(streamID=inputFilePath)
    self.encoder = CioEncoder(cacheDir="./experiments/cache")
    self._initModel()


  def _initModel(self):
    """Initialize the network and related variables"""
    if self.classifierType == "CLA":
      classifier_params = {
        "steps": "1",
        "implementation": "py",
        "clVerbosity": self.verbosity
      }
    elif self.classifierType == "KNN":
      classifier_params = {
        "k": self.numLabels,
        "distThreshold": 0,
        "maxCategoryCount": self.numLabels
      }
    else:
      raise ValueError("Classifier type {} is not supported.".format(
        self.classifierType))

    self.network = createNetwork(
      (self.recordStream, "py.LanguageSensor", self.encoder, self.numLabels,
      "py.{}ClassifierRegion".format(self.classifierType), classifier_params))

    self.network.initialize()

    spatialPoolerRegion = self.network.regions["SP"]
    temporalMemoryRegion = self.network.regions["TM"]
    classifierRegion = self.network.regions["classifier"]

    spatialPoolerRegion.setParameter("learningMode", False)
    temporalMemoryRegion.setParameter("learningMode", False)
    classifierRegion.setParameter("learningMode", False)

    self.numTrained = 0
    self.oldClassifications = None
    self.lengthOfCurrentSequence = 0


  def encodePattern(self, sample):
    """
    Put each token in its own dictionary with its bitmap
    @param sample     (list)            Tokenized sample, where each item is a
                                        string token.
    @return           (list)            The sample text, sparsity, and bitmap
                                        for each token. Since the network will
                                        do the actual encoding, the bitmap and
                                        sparsity will be None
    Example return list:
      [{
        "text": "Example text",
        "sparsity": 0.0,
        "bitmap": None
      }]
    """
    return [{"text": t,
             "sparsity": None,
             "bitmap": None} for t in sample]


  def resetModel(self):
    """
    Reset the model by creating a new network since the network API does not
    support resets.
    """
    self.recordStream.clear()
    self._initModel()


  def trainModel(self):
    """
    Train the network on the input to FileRecordStream.  Train the spatial
    pooler if the network has been trained on enough (self.cpTrainingSize)
    samples. Train the temporal memory if the network has been trained on
    enough (self.tpTrainingSize) samples. Train the classifier if the network
    has been trained on enough (self.clsTrainingSize) samples.
    """
    sensorRegion = self.network.regions["sensor"]
    spatialPoolerRegion = self.network.regions["SP"]
    temporalMemoryRegion = self.network.regions["TM"]<commit_msg>fix typo<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import numpy

from classification.classification_network import createNetwork
from fluent.encoders.cio_encoder import CioEncoder
from fluent.models.classification_model import ClassificationModel
from nupic.data.file_record_stream import FileRecordStream



class ClassificationModelHTM(ClassificationModel):
  """
  Class to run the survey response classification task with nupic network
  """

  def __init__(self, inputFilePath, verbosity=1, numLabels=3, spTrainingSize=0,
               tmTrainingSize=0, clsTrainingSize=0, classifierType="KNN"):
    """
    @param inputFilePath      (str)       Path to data formatted for network
                                          API
    @param spTrainingSize     (int)       Number of samples the network has to
                                          be trained on before training the
                                          spatial pooler
    @param tmTrainingSize     (int)       Number of samples the network has to
                                          be trained on before training the
                                          temporal memory
    @param clsTrainingSize    (int)       Number of samples the network has to
                                          be trained on before training the
                                          classifier
    @param classifierType     (str)       Either "KNN" or "CLA"
    See ClassificationModel for remaining parameters
    """
    self.spTrainingSize = spTrainingSize
    self.tmTrainingSize = tmTrainingSize
    self.clsTrainingSize = clsTrainingSize

    super(ClassificationModelHTM, self).__init__(verbosity=verbosity,
      numLabels=numLabels)

    # Initialize Network
    self.classifierType = classifierType
    self.recordStream = FileRecordStream(streamID=inputFilePath)
    self.encoder = CioEncoder(cacheDir="./experiments/cache")
    self._initModel()


  def _initModel(self):
    """Initialize the network and related variables"""
    if self.classifierType == "CLA":
      classifier_params = {
        "steps": "1",
        "implementation": "py",
        "clVerbosity": self.verbosity
      }
    elif self.classifierType == "KNN":
      classifier_params = {
        "k": self.numLabels,
        "distThreshold": 0,
        "maxCategoryCount": self.numLabels
      }
    else:
      raise ValueError("Classifier type {} is not supported.".format(
        self.classifierType))

    self.network = createNetwork(
      (self.recordStream, "py.LanguageSensor", self.encoder, self.numLabels,
      "py.{}ClassifierRegion".format(self.classifierType), classifier_params))

    self.network.initialize()

    spatialPoolerRegion = self.network.regions["SP"]
    temporalMemoryRegion = self.network.regions["TM"]
    classifierRegion = self.network.regions["classifier"]

    spatialPoolerRegion.setParameter("learningMode", False)
    temporalMemoryRegion.setParameter("learningMode", False)
    classifierRegion.setParameter("learningMode", False)

    self.numTrained = 0
    self.oldClassifications = None
    self.lengthOfCurrentSequence = 0


  def encodePattern(self, sample):
    """
    Put each token in its own dictionary with its bitmap
    @param sample     (list)            Tokenized sample, where each item is a
                                        string token.
    @return           (list)            The sample text, sparsity, and bitmap
                                        for each token. Since the network will
                                        do the actual encoding, the bitmap and
                                        sparsity will be None
    Example return list:
      [{
        "text": "Example text",
        "sparsity": 0.0,
        "bitmap": None
      }]
    """
    return [{"text": t,
             "sparsity": None,
             "bitmap": None} for t in sample]


  def resetModel(self):
    """
    Reset the model by creating a new network since the network API does not
    support resets.
    """
    self.recordStream.clear()
    self._initModel()


  def trainModel(self):
    """
    Train the network on the input to FileRecordStream.  Train the spatial
    pooler if the network has been trained on enough (self.spTrainingSize)
    samples. Train the temporal memory if the network has been trained on
    enough (self.tmTrainingSize) samples. Train the classifier if the network
    has been trained on enough (self.clsTrainingSize) samples.
    """
    sensorRegion = self.network.regions["sensor"]
    spatialPoolerRegion = self.network.regions["SP"]
    temporalMemoryRegion = self.network.regions["TM"]
{'fraction_non_alphanumeric': 0.09820551736452103, 'fraction_numerical': 0.0028568877778769753, 'mean_word_length': 2.2319676860934794, 'pattern_counts': {'":': 28, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5874032', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2638, 'n_words': 934}