<commit_before>#!/usr/bin/env python

# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
import os
import numpy
from PyRegion import PyRegion
from datetime import datetime, timedelta
from nupic.data.fieldmeta import FieldMetaType

#-------------------------------------------------------------------------------
class RecordSensor(PyRegion):
  """
  A Record Sensor (RS) retrieves an information "record" and encodes
  it to be suitable as input to an HTM.

  An information record is analogous database record -- it is just a
  collection of typed values: date, amount, category, location, etc.

  The RS may obtain information from one of three sources:
    . a file (e.g. csv or tsv)
    . a sql database (not yet implemented)
    . a data generator (for artificial data)

  The RS encodes a record using an encoding scheme that can be specified
  programmatically.

  An RS is essentially a shell containing two objects:

  1. A DataSource object gets one record at a time. This record is returned
  either as a dictionary or a user-defined object. The fields within a record
  correspond to entries in the dictionary or attributes of the object. For
  example, a DataSource might return:

    dict(date="02-01-2010 23:12:23", amount=4.95, country="US", _reset=0, _sequenceId=0)

  or an object with attributes "date", "amount" and "country".

  The _reset and _sequenceId attributes must always exist, and are provided by
  the DataSource if not directly present in the data.

  DataSource methods are:
  -- getNext() -- return the next record, which is a dict
  -- TBD: something like getIterationCount()?

  2. A MultiEncoder object encodes one record into a fixed-sparsity
  distributed representation. MultiEncoder is defined in
  nupic.encoders

  The DataSource and MultiEncoder are supplied after the node is created,
  not in the node itself.

  Example usage in NuPIC:

  from nupic.net import Network
  from nupic.encoders import MultiEncoder
  from nupic.data import FunctionSource

  n = Network()
  s = n.addRegion("sensor", "py.RecordSensor", "")
  myparser = SmartParser()  # custom parser for Smart data;
  mysource = FunctionSource("mydata.txt", myparser)
  myencoder = MultiEncoder()
  ... set up myencoder ...<commit_msg>Fixed comment explaining use of RecordSensor.py<commit_after>#!/usr/bin/env python

# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
import os
import numpy
from PyRegion import PyRegion
from datetime import datetime, timedelta
from nupic.data.fieldmeta import FieldMetaType

#-------------------------------------------------------------------------------
class RecordSensor(PyRegion):
  """
  A Record Sensor (RS) retrieves an information "record" and encodes
  it to be suitable as input to an HTM.

  An information record is analogous database record -- it is just a
  collection of typed values: date, amount, category, location, etc.

  The RS may obtain information from one of three sources:
    . a file (e.g. csv or tsv)
    . a sql database (not yet implemented)
    . a data generator (for artificial data)

  The RS encodes a record using an encoding scheme that can be specified
  programmatically.

  An RS is essentially a shell containing two objects:

  1. A DataSource object gets one record at a time. This record is returned
  either as a dictionary or a user-defined object. The fields within a record
  correspond to entries in the dictionary or attributes of the object. For
  example, a DataSource might return:

    dict(date="02-01-2010 23:12:23", amount=4.95, country="US", _reset=0, _sequenceId=0)

  or an object with attributes "date", "amount" and "country".

  The _reset and _sequenceId attributes must always exist, and are provided by
  the DataSource if not directly present in the data.

  DataSource methods are:
  -- getNext() -- return the next record, which is a dict
  -- TBD: something like getIterationCount()?

  2. A MultiEncoder object encodes one record into a fixed-sparsity
  distributed representation. MultiEncoder is defined in
  nupic.encoders

  The DataSource and MultiEncoder are supplied after the node is created,
  not in the node itself.

  Example usage in NuPIC:

  from nupic.net import Network
  from nupic.encoders import MultiEncoder
  from nupic.data.file.file_record_stream import FileRecordStream

  n = Network()
  s = n.addRegion("sensor", "py.RecordSensor", "")
  mysource = FileRecordStream("mydata.txt")
  myencoder = MultiEncoder()
  ... set up myencoder ...
{'fraction_non_alphanumeric': 0.13079835910381823, 'fraction_numerical': 0.008204480908804039, 'mean_word_length': 2.8511543134872417, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7628941', 'n_tokens_mistral': 1658, 'n_tokens_neox': 1576, 'n_words': 793}