<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

""" Useful scripts for classification network configs. """

import copy
from collections import namedtuple

from htmresearch.frameworks.classification.classification_network import (
  TEST_PARTITION_NAME)

# Region names
SENSOR_CONFIG = "sensorRegionConfig"
SP_CONFIG = "spRegionConfig"
TM_CONFIG = "tmRegionConfig"
TP_CONFIG = "tpRegionConfig"
CLASSIFIER_CONFIG = "classifierRegionConfig"

# Region types
SENSOR_TYPE = "py.RecordSensor"
SP_REGION_TYPE = "py.SPRegion"
TM_REGION_TYPE = "py.TPRegion"
TP_REGION_TYPE = "py.TPRegion"
CLA_CLASSIFIER_TYPE = "py.CLAClassifierRegion"
KNN_CLASSIFIER_TYPE = "py.KNNClassifierRegion"



def generateNetworkPartitions(networkConfig, numRecords):
  """
  Find the number of partitions for the input data based on a specific
  networkConfig. Partition: Region names and index at which the
  region is to begin learning. The final partition is reserved as a test set.

  @param networkConfig: (dict) the configuration of this network.
  @param numRecords: (int) Number of records of the input dataset.
  @return partitions: (list of namedtuples) Paritions. I.e. the region names 
  and index at which the region is to begin learning. The final partition is 
  reserved as a test set.
  """
  Partition = namedtuple("Partition", "partName index")

  # Add regions to partition list in order of learning.
  regionConfigs = ("spRegionConfig", "tmRegionConfig", "tpRegionConfig",
                   "classifierRegionConfig")
  partitions = []
  maxNumPartitions = 5
  i = 0
  for region in regionConfigs:
    if networkConfig[region].get("regionEnabled"):
      learnIndex = i * numRecords / maxNumPartitions
      partitions.append(Partition(
        partName=networkConfig[region].get("regionName"), index=learnIndex))
      i += 1

  testIndex = numRecords * (maxNumPartitions - 1) / maxNumPartitions
  partitions.append(Partition(partName=TEST_PARTITION_NAME, index=testIndex))

  return partitions



def generateSampleNetworkConfig(templateNetworkConfig, maxCategoryCount):
  """
  Generate a sample network configuration for sensor data experiments, using a 
  template network params dict.

  @param templateNetworkConfig: (dict) template network config based on which
  other network configs are generated.
  @param maxCategoryCount: (int) the maximum number of categories to classify. 
  @return networkConfigurations: (list) network configs.
  """

  networkConfigurations = []
  
  # Basic classifier params
  CLA_CLASSIFIER_PARAMS = {
    "steps": "0",
    "implementation": "cpp",
    "maxCategoryCount": maxCategoryCount,
    "clVerbosity": 0
  }
  KNN_CLASSIFIER_PARAMS = {
    "k": 1,
    'distThreshold': 0,
    'maxCategoryCount': maxCategoryCount,
  }

  # First config: SP and TM enabled. UP disabled. KNN Classifier.
  networkConfig = copy.deepcopy(templateNetworkConfig)
  networkConfig[SP_CONFIG]["regionEnabled"] = True
  networkConfig[TM_CONFIG]["regionEnabled"] = True
  networkConfig[TP_CONFIG]["regionEnabled"] = False
  networkConfig[CLASSIFIER_CONFIG]["regionType"] = KNN_CLASSIFIER_TYPE
  networkConfig[CLASSIFIER_CONFIG]["regionParams"] = KNN_CLASSIFIER_PARAMS
  networkConfigurations.append(networkConfig)

  # First config: SP and TM enabled. UP disabled. CLA Classifier.
  networkConfig = copy.deepcopy(templateNetworkConfig)
  networkConfig[SP_CONFIG]["regionEnabled"] = True
  networkConfig[TM_CONFIG]["regionEnabled"] = True
  networkConfig[TP_CONFIG]["regionEnabled"] = False
  networkConfig[CLASSIFIER_CONFIG]["regionType"] = CLA_CLASSIFIER_TYPE
  networkConfig[CLASSIFIER_CONFIG]["regionParams"] = CLA_CLASSIFIER_PARAMS
  networkConfigurations.append(networkConfig)

  # Second config: SP enabled. TM and UP disabled. CLA Classifier.
  networkConfig = copy.deepcopy(templateNetworkConfig)
  networkConfig[SP_CONFIG]["regionEnabled"] = True
  networkConfig[TM_CONFIG]["regionEnabled"] = False
  networkConfig[TP_CONFIG]["regionEnabled"] = False
  networkConfig[CLASSIFIER_CONFIG]["regionType"] = CLA_CLASSIFIER_TYPE
  networkConfig[CLASSIFIER_CONFIG]["regionParams"] = CLA_CLASSIFIER_PARAMS
  networkConfigurations.append(networkConfig)

  return networkConfigurations<commit_msg>added SDR classifier and JSON network config<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

""" Useful scripts for classification network configs. """

import copy
from collections import namedtuple

from htmresearch.frameworks.classification.classification_network import (
  TEST_PARTITION_NAME)

# Region names
SENSOR_CONFIG = "sensorRegionConfig"
SP_CONFIG = "spRegionConfig"
TM_CONFIG = "tmRegionConfig"
TP_CONFIG = "tpRegionConfig"
CLASSIFIER_CONFIG = "classifierRegionConfig"

# Region types
SENSOR_TYPE = "py.RecordSensor"
SP_REGION_TYPE = "py.SPRegion"
TM_REGION_TYPE = "py.TPRegion"
TP_REGION_TYPE = "py.TPRegion"
CLA_CLASSIFIER_TYPE = "py.CLAClassifierRegion"
SDR_CLASSIFIER_TYPE = "py.SDRClassifierRegion"
KNN_CLASSIFIER_TYPE = "py.KNNClassifierRegion"



def generateNetworkPartitions(networkConfig, numRecords):
  """
  Find the number of partitions for the input data based on a specific
  networkConfig. Partition: Region names and index at which the
  region is to begin learning. The final partition is reserved as a test set.

  @param networkConfig: (dict) the configuration of this network.
  @param numRecords: (int) Number of records of the input dataset.
  @return partitions: (list of namedtuples) Paritions. I.e. the region names 
  and index at which the region is to begin learning. The final partition is 
  reserved as a test set.
  """
  Partition = namedtuple("Partition", "partName index")

  # Add regions to partition list in order of learning.
  regionConfigs = ("spRegionConfig", "tmRegionConfig", "tpRegionConfig",
                   "classifierRegionConfig")
  partitions = []
  maxNumPartitions = 5
  i = 0
  for region in regionConfigs:
    if networkConfig[region].get("regionEnabled"):
      learnIndex = i * numRecords / maxNumPartitions
      partitions.append(Partition(
        partName=networkConfig[region].get("regionName"), index=learnIndex))
      i += 1

  testIndex = numRecords * (maxNumPartitions - 1) / maxNumPartitions
  partitions.append(Partition(partName=TEST_PARTITION_NAME, index=testIndex))

  return partitions



def generateSampleNetworkConfig(templateNetworkConfig, maxCategoryCount):
  """
  Generate a sample network configuration for sensor data experiments, using a 
  template network params dict.

  @param templateNetworkConfig: (dict) template network config based on which
  other network configs are generated.
  @param maxCategoryCount: (int) the maximum number of categories to classify. 
  @return networkConfigurations: (list) network configs.
  """

  networkConfigurations = []
  
  # Basic classifier params
  CLA_CLASSIFIER_PARAMS = {
    "steps": 0,
    "alpha": 0.01,
    "implementation": "cpp",
    "maxCategoryCount": maxCategoryCount,
    "verbosity": 0
  }
  
  SDR_CLASSIFIER_PARAMS = {
    "steps": 0,
    "alpha": 0.01,
    "implementation": "py",
    "maxCategoryCount": maxCategoryCount,
    "verbosity": 0
  }
  
  KNN_CLASSIFIER_PARAMS = {
    "k": 1,
    "maxCategoryCount": maxCategoryCount
  }

  # SP and TM enabled. UP disabled. KNN Classifier.
  # networkConfig = copy.deepcopy(templateNetworkConfig)
  # networkConfig[SP_CONFIG]["regionEnabled"] = True
  # networkConfig[TM_CONFIG]["regionEnabled"] = True
  # networkConfig[TP_CONFIG]["regionEnabled"] = False
  # networkConfig[CLASSIFIER_CONFIG]["regionType"] = KNN_CLASSIFIER_TYPE
  # networkConfig[CLASSIFIER_CONFIG]["regionParams"] = KNN_CLASSIFIER_PARAMS
  # networkConfigurations.append(networkConfig)
  # 
  # SP enabled. TM and UP disabled. KNN Classifier.
  # networkConfig = copy.deepcopy(templateNetworkConfig)
  # networkConfig[SP_CONFIG]["regionEnabled"] = True
  # networkConfig[TM_CONFIG]["regionEnabled"] = False
  # networkConfig[TP_CONFIG]["regionEnabled"] = False
  # networkConfig[CLASSIFIER_CONFIG]["regionType"] = KNN_CLASSIFIER_TYPE
  # networkConfig[CLASSIFIER_CONFIG]["regionParams"] = KNN_CLASSIFIER_PARAMS
  # networkConfigurations.append(networkConfig)
  
  ## SDR Classifier ##
  
  # SP and TM enabled. UP disabled. SDR Classifier.
  networkConfig = copy.deepcopy(templateNetworkConfig)
  networkConfig[SP_CONFIG]["regionEnabled"] = True
  networkConfig[TM_CONFIG]["regionEnabled"] = True
  networkConfig[TP_CONFIG]["regionEnabled"] = False
  networkConfig[CLASSIFIER_CONFIG]["regionType"] = SDR_CLASSIFIER_TYPE
  networkConfig[CLASSIFIER_CONFIG]["regionParams"] = SDR_CLASSIFIER_PARAMS
  networkConfigurations.append(networkConfig)

  # SP enabled. TM and UP disabled. SDR Classifier.
  networkConfig = copy.deepcopy(templateNetworkConfig)
  networkConfig[SP_CONFIG]["regionEnabled"] = True
  networkConfig[TM_CONFIG]["regionEnabled"] = False
  networkConfig[TP_CONFIG]["regionEnabled"] = False
  networkConfig[CLASSIFIER_CONFIG]["regionType"] = SDR_CLASSIFIER_TYPE
  networkConfig[CLASSIFIER_CONFIG]["regionParams"] = SDR_CLASSIFIER_PARAMS
  networkConfigurations.append(networkConfig)  
  
  # SP and TM and UP enabled. SDR Classifier.
  networkConfig = copy.deepcopy(templateNetworkConfig)
  networkConfig[SP_CONFIG]["regionEnabled"] = True
  networkConfig[TM_CONFIG]["regionEnabled"] = True
  networkConfig[TP_CONFIG]["regionEnabled"] = True
  networkConfig[CLASSIFIER_CONFIG]["regionType"] = SDR_CLASSIFIER_TYPE
  networkConfig[CLASSIFIER_CONFIG]["regionParams"] = SDR_CLASSIFIER_PARAMS
  networkConfigurations.append(networkConfig)
  
  
  ## CLA Classifier ##
  
  # First config: SP and TM enabled. UP disabled. CLA Classifier.
  networkConfig = copy.deepcopy(templateNetworkConfig)
  networkConfig[SP_CONFIG]["regionEnabled"] = True
  networkConfig[TM_CONFIG]["regionEnabled"] = True
  networkConfig[TP_CONFIG]["regionEnabled"] = False
  networkConfig[CLASSIFIER_CONFIG]["regionType"] = CLA_CLASSIFIER_TYPE
  networkConfig[CLASSIFIER_CONFIG]["regionParams"] = CLA_CLASSIFIER_PARAMS
  networkConfigurations.append(networkConfig)

  # Second config: SP enabled. TM and UP disabled. CLA Classifier.
  networkConfig = copy.deepcopy(templateNetworkConfig)
  networkConfig[SP_CONFIG]["regionEnabled"] = True
  networkConfig[TM_CONFIG]["regionEnabled"] = False
  networkConfig[TP_CONFIG]["regionEnabled"] = False
  networkConfig[CLASSIFIER_CONFIG]["regionType"] = CLA_CLASSIFIER_TYPE
  networkConfig[CLASSIFIER_CONFIG]["regionParams"] = CLA_CLASSIFIER_PARAMS
  networkConfigurations.append(networkConfig)

  return networkConfigurations
{'fraction_non_alphanumeric': 0.10995184590690209, 'fraction_numerical': 0.0026484751203852326, 'mean_word_length': 4.623194945848375, 'pattern_counts': {'":': 17, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6645904', 'n_tokens_mistral': 3603, 'n_tokens_neox': 3498, 'n_words': 978}