<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import copy
import imp
import json
import logging
from optparse import OptionParser
import os
import pprint
import shutil
import string
import subprocess
import sys
import unittest2 as unittest

from nupic.database.ClientJobsDAO import ClientJobsDAO
from nupic.swarming.utils import generatePersistentJobGUID
from nupic.swarming import HypersearchWorker
from nupic.swarming.permutationhelpers import PermuteChoices
from nupic.frameworks.opf.exp_generator import ExpGenerator
from nupic.data import dictutils
from nupic.frameworks.opf.expdescriptionapi import OpfEnvironment
from nupic.frameworks.opf.opfutils import (InferenceType,
                                           InferenceElement)
from nupic.support import aggregationDivide
from nupic.support.unittesthelpers.testcasebase import (
  TestCaseBase as HelperTestCaseBase)



g_debug = False

# Our __main__ entry block sets this to an instance of MyTestEnvironment()
g_myEnv = None



class MyTestEnvironment(object):

  def __init__(self, options):

    # Save all command line options
    self.options = options

    # Build installation root (e.g., ~/nta/current)
    installRootDir = os.path.abspath(options.installDir)
    if not os.path.exists(installRootDir):
      raise RuntimeError("install directory %s doesn't exist" % \
                         (options.installDir))
    _debugOut("installRootDir=<%s>" % (installRootDir,))


    # Where this script is running from (autotest expgenerator_test.py may have
    # copied it from its original location)
    self.testRunDir = os.path.dirname(os.path.abspath(__file__))
    _debugOut("self.testRunDir=<%s>" % (self.testRunDir,))

    # Where to place generated files
    self.testOutDir = os.path.join(self.testRunDir, 'expGeneratorOut')
    shutil.rmtree(self.testOutDir, ignore_errors=True)
    os.makedirs(self.testOutDir)

    print "Generating experiment description files in:", \
                  os.path.abspath(self.testOutDir)

    # Where to find out datasets
    self.datasetSrcDir = os.path.join(installRootDir, "share", "prediction",
                                 "data", "extra")


    return



class ExperimentTestBaseClass(HelperTestCaseBase):

  # We will load the description.py and permutations.py files as modules
  # multiple times in order to verify that they are valid python scripts. To
  # facilitate this, we reload with a unique module name
  # ("expGenerator_generated_script%d") each time.
  __pythonScriptImportCount = 0

  @classmethod
  def newScriptImportName(cls):
    cls.__pythonScriptImportCount += 1
    name = "expGenerator_generated_script%d" % cls.__pythonScriptImportCount
    return name


  def setUp(self):
    """ Method called to prepare the test fixture. This is called by the
    unittest framework immediately before calling the test method; any exception
    raised by this method will be considered an error rather than a test
    failure. The default implementation does nothing.
    """

    pass


  def tearDown(self):
    """ Method called immediately after the test method has been called and the
    result recorded. This is called even if the test method raised an exception,
    so the implementation in subclasses may need to be particularly careful
    about checking internal state. Any exception raised by this method will be
    considered an error rather than a test failure. This method will only be
    called if the setUp() succeeds, regardless of the outcome of the test
    method. The default implementation does nothing.
    """
    # Reset our log items
    self.resetExtraLogItems()


  def shortDescription(self):
    """ Override to force unittest framework to use test method names instead
    of docstrings in the report.
    """
    return None


  def checkPythonScript(self, scriptAbsPath):
    self.assertTrue(os.path.isabs(scriptAbsPath))
<commit_msg>Cleaning up after test<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import copy
import imp
import json
import logging
from optparse import OptionParser
import os
import pprint
import shutil
import string
import subprocess
import sys
import unittest2 as unittest

from nupic.database.ClientJobsDAO import ClientJobsDAO
from nupic.swarming.utils import generatePersistentJobGUID
from nupic.swarming import HypersearchWorker
from nupic.swarming.permutationhelpers import PermuteChoices
from nupic.frameworks.opf.exp_generator import ExpGenerator
from nupic.data import dictutils
from nupic.frameworks.opf.expdescriptionapi import OpfEnvironment
from nupic.frameworks.opf.opfutils import (InferenceType,
                                           InferenceElement)
from nupic.support import aggregationDivide
from nupic.support.unittesthelpers.testcasebase import (
  TestCaseBase as HelperTestCaseBase)



g_debug = False

# Our __main__ entry block sets this to an instance of MyTestEnvironment()
g_myEnv = None



class MyTestEnvironment(object):

  def __init__(self, options):

    # Save all command line options
    self.options = options

    # Build installation root (e.g., ~/nta/current)
    installRootDir = os.path.abspath(options.installDir)
    if not os.path.exists(installRootDir):
      raise RuntimeError("install directory %s doesn't exist" % \
                         (options.installDir))
    _debugOut("installRootDir=<%s>" % (installRootDir,))


    # Where this script is running from (autotest expgenerator_test.py may have
    # copied it from its original location)
    self.testRunDir = os.path.dirname(os.path.abspath(__file__))
    _debugOut("self.testRunDir=<%s>" % (self.testRunDir,))

    # Where to place generated files
    self.testOutDir = os.path.join(self.testRunDir, 'expGeneratorOut')
    shutil.rmtree(self.testOutDir, ignore_errors=True)
    os.makedirs(self.testOutDir)

    print "Generating experiment description files in:", \
                  os.path.abspath(self.testOutDir)

    # Where to find out datasets
    self.datasetSrcDir = os.path.join(installRootDir, "share", "prediction",
                                 "data", "extra")


    return


  def cleanUp(self):
    shutil.rmtree(self.testOutDir, ignore_errors=True)
    return



class ExperimentTestBaseClass(HelperTestCaseBase):

  # We will load the description.py and permutations.py files as modules
  # multiple times in order to verify that they are valid python scripts. To
  # facilitate this, we reload with a unique module name
  # ("expGenerator_generated_script%d") each time.
  __pythonScriptImportCount = 0

  @classmethod
  def newScriptImportName(cls):
    cls.__pythonScriptImportCount += 1
    name = "expGenerator_generated_script%d" % cls.__pythonScriptImportCount
    return name


  def setUp(self):
    """ Method called to prepare the test fixture. This is called by the
    unittest framework immediately before calling the test method; any exception
    raised by this method will be considered an error rather than a test
    failure. The default implementation does nothing.
    """

    pass


  def tearDown(self):
    """ Method called immediately after the test method has been called and the
    result recorded. This is called even if the test method raised an exception,
    so the implementation in subclasses may need to be particularly careful
    about checking internal state. Any exception raised by this method will be
    considered an error rather than a test failure. This method will only be
    called if the setUp() succeeds, regardless of the outcome of the test
    method. The default implementation does nothing.
    """
    self.resetExtraLogItems()
    g_myEnv.cleanUp()


  def shortDescription(self):
    """ Override to force unittest framework to use test method names instead
    of docstrings in the report.
    """
    return None


  def checkPythonScript(self, scriptAbsPath):
    self.assertTrue(os.path.isabs(scriptAbsPath))

{'fraction_non_alphanumeric': 0.09293113954905545, 'fraction_numerical': 0.0016250253910217347, 'mean_word_length': 3.27758470894874, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4901300', 'n_tokens_mistral': 2631, 'n_tokens_neox': 2524, 'n_words': 993}