<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""
Initial experiment runner for classification survey question responses.

EXAMPLE: from the fluent directory, run...
  python experiments/baseline_experiment.py
  data/sample_reviews/sample_reviews_data_training.csv

  - The runner sets up the data path such that the experiment runs on a single
  data file located in the nupic.fluent/data directory. The data path MUST BE
  SPECIFIED at the cmd line.
  - This example runs the ClassificationModelRandomSDR subclass of Model. To use
  a different model, use cmd line args modelName and modelModuleName.
  - The call to readCSV() below is specific for the format of this data file,
  and should be changed for CSVs with different columns.

Please note the following definitions:
- k-fold cross validation: the training dataset is split
  differently for each of the k trials. The majority of the dataset is used for
  training, and a small portion (1/k) is held out for evaluation; this
  evaluation data is different from the test data.
- classification and label are used interchangeably
"""


import argparse
import collections
import cPickle as pkl
import itertools
import numpy
import os
import time

from fluent.utils.csv_helper import readCSV
from fluent.utils.data_split import KFolds
from fluent.utils.text_preprocess import TextPreprocess


def runExperiment(model, patterns, idxSplits, batch):
  """
  Trains the model on patterns specified by the first entry of idxSplits, then
  tests on the patterns of the second entry on idxSplits.

  @param model          (Model)       Classification model instance.
  @param patterns       (list)        Each item is a dict with the sample
                                      encoding a numpy array bitmap in field
                                      "bitmap".
  @param idxSplits      (tuple)       Tuple of train/eval split data indices.
  @param batch          (bool)        Whether or not to train on all the data
                                      in a batch
  @return                             Return same as testing().
  """
  model.resetModel()
  training(model, [patterns[i] for i in idxSplits[0]], batch)
  return testing(model, [patterns[i] for i in idxSplits[1]])


# training() and testing() methods send one data sample at a time to the model,
# i.e. streaming input.
def training(model, trainSet, batch):
  """
  Trains model on the bitmap patterns and corresponding labels lists one at a
  time (i.e. streaming).
  @param batch          (bool)        Whether or not to train on all the data
                                      in a batch
  """
  if batch:
    samples = [s["pattern"] for s in trainSet]
    labels = [s["labels"] for s in trainSet]
    model.trainModel(samples, labels)
  else:
    for sample in trainSet:
      model.trainModel([sample["pattern"]], [sample["labels"]])


def testing(model, evalSet):
  """
  Tests model on the bitmap patterns and corresponding labels lists, one at a
  time (i.e. streaming).

  @return trialResults    (list)      List of two lists, where the first list
      is the model's predicted classifications, and the second list is the
      actual classifications.
  """
  trialResults = ([], [])
  for sample in evalSet:
    predicted = model.testModel(sample["pattern"])
    trialResults[0].append(predicted)
    trialResults[1].append(sample["labels"])
  return trialResults


def calculateResults(model, results, refs, indices, fileName):
  """
  Evaluate the results, returning accuracy and confusion matrix, and writing
  the confusion matrix to a CSV.

  TODO: csv writing broken until ClassificationModel confusion matrix is fixed
  """
  result = model.evaluateResults(results, refs, indices)
  # result[1].to_csv(fileName)
  return result


def computeExpectedAccuracy(predictedLabels, dataPath):
  """
  Compute the accuracy of the models predictions against what we expect it to
  predict; considers only single classification.
  """
  _, expectedLabels = readCSV(dataPath, 2, [3])
  if len(expectedLabels) != len(predictedLabels):
    raise ValueError("Lists of labels must have the same length.")

  accuracy = len([i for i in xrange(len(expectedLabels))
    if expectedLabels[i]==predictedLabels[i]]) / float(len(expectedLabels))

  print "Accuracy against expected classifications = ", accuracy


def setupData(args):
  """ Performs data preprocessing and setup given the user-specified args.

  @param args       (Namespace)     User-provided arguments via the cmd line.
  @return           (tuple)         Tuple where first entry is a list of the
      samples, the second is the list of gold labels per example, the third is
      the list of all possible labels, and the fourth is the labels per example
      in the data.
  """
  dataDict = readCSV(args.dataPath, 2, args.numLabels)

  # Collect each possible label string into a list, where the indices will be
  # their references throughout the experiment.
  labelReference = list(set(
    itertools.chain.from_iterable(map(lambda x: x[1], dataDict.values()))))

  for id, data in dataDict.iteritems():
    comment, labels = data
    dataDict[id] = (comment, numpy.array([labelReference.index(label)
                                    for label in labels],<commit_msg>updating because of readCSV<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""
Initial experiment runner for classification survey question responses.

EXAMPLE: from the fluent directory, run...
  python experiments/baseline_experiment.py
  data/sample_reviews/sample_reviews_data_training.csv

  - The runner sets up the data path such that the experiment runs on a single
  data file located in the nupic.fluent/data directory. The data path MUST BE
  SPECIFIED at the cmd line.
  - This example runs the ClassificationModelRandomSDR subclass of Model. To use
  a different model, use cmd line args modelName and modelModuleName.
  - The call to readCSV() below is specific for the format of this data file,
  and should be changed for CSVs with different columns.

Please note the following definitions:
- k-fold cross validation: the training dataset is split
  differently for each of the k trials. The majority of the dataset is used for
  training, and a small portion (1/k) is held out for evaluation; this
  evaluation data is different from the test data.
- classification and label are used interchangeably
"""


import argparse
import collections
import cPickle as pkl
import itertools
import numpy
import os
import time

from fluent.utils.csv_helper import readCSV
from fluent.utils.data_split import KFolds
from fluent.utils.text_preprocess import TextPreprocess


def runExperiment(model, patterns, idxSplits, batch):
  """
  Trains the model on patterns specified by the first entry of idxSplits, then
  tests on the patterns of the second entry on idxSplits.

  @param model          (Model)       Classification model instance.
  @param patterns       (list)        Each item is a dict with the sample
                                      encoding a numpy array bitmap in field
                                      "bitmap".
  @param idxSplits      (tuple)       Tuple of train/eval split data indices.
  @param batch          (bool)        Whether or not to train on all the data
                                      in a batch
  @return                             Return same as testing().
  """
  model.resetModel()
  training(model, [patterns[i] for i in idxSplits[0]], batch)
  return testing(model, [patterns[i] for i in idxSplits[1]])


# training() and testing() methods send one data sample at a time to the model,
# i.e. streaming input.
def training(model, trainSet, batch):
  """
  Trains model on the bitmap patterns and corresponding labels lists one at a
  time (i.e. streaming).
  @param batch          (bool)        Whether or not to train on all the data
                                      in a batch
  """
  if batch:
    samples = [s["pattern"] for s in trainSet]
    labels = [s["labels"] for s in trainSet]
    model.trainModel(samples, labels)
  else:
    for sample in trainSet:
      model.trainModel([sample["pattern"]], [sample["labels"]])


def testing(model, evalSet):
  """
  Tests model on the bitmap patterns and corresponding labels lists, one at a
  time (i.e. streaming).

  @return trialResults    (list)      List of two lists, where the first list
      is the model's predicted classifications, and the second list is the
      actual classifications.
  """
  trialResults = ([], [])
  for sample in evalSet:
    predicted = model.testModel(sample["pattern"])
    trialResults[0].append(predicted)
    trialResults[1].append(sample["labels"])
  return trialResults


def calculateResults(model, results, refs, indices, fileName):
  """
  Evaluate the results, returning accuracy and confusion matrix, and writing
  the confusion matrix to a CSV.

  TODO: csv writing broken until ClassificationModel confusion matrix is fixed
  """
  result = model.evaluateResults(results, refs, indices)
  # result[1].to_csv(fileName)
  return result


def computeExpectedAccuracy(predictedLabels, dataPath):
  """
  Compute the accuracy of the models predictions against what we expect it to
  predict; considers only single classification.
  """
  dataDict = readCSV(dataPath, 2, [3])
  expectedLabels = [data[1] for id, data in dataDict.iteritems()]

  if len(expectedLabels) != len(predictedLabels):
    raise ValueError("Lists of labels must have the same length.")

  accuracy = len([i for i in xrange(len(expectedLabels))
    if expectedLabels[i]==predictedLabels[i]]) / float(len(expectedLabels))

  print "Accuracy against expected classifications = ", accuracy


def setupData(args):
  """ Performs data preprocessing and setup given the user-specified args.

  @param args       (Namespace)     User-provided arguments via the cmd line.
  @return           (tuple)         Tuple where first entry is a list of the
      samples, the second is the list of gold labels per example, the third is
      the list of all possible labels, and the fourth is the labels per example
      in the data.
  """
  dataDict = readCSV(args.dataPath, 2, args.numLabels)

  # Collect each possible label string into a list, where the indices will be
  # their references throughout the experiment.
  labelReference = list(set(
    itertools.chain.from_iterable(map(lambda x: x[1], dataDict.values()))))

  for id, data in dataDict.iteritems():
    comment, labels = data
    dataDict[id] = (comment, numpy.array([labelReference.index(label)
                                    for label in labels],
{'fraction_non_alphanumeric': 0.09341010876519514, 'fraction_numerical': 0.002479206653870761, 'mean_word_length': 2.953525134366108, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5551403', 'n_tokens_mistral': 3264, 'n_tokens_neox': 3165, 'n_words': 1445}