<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

""" @file permutations_runner.py
"""

import collections
import imp
import csv
from datetime import datetime, timedelta
import os
import cPickle as pickle
import pprint
import shutil
import signal
import sys
import time
import subprocess
import tempfile
import uuid

from nupic.support import object_json as json
import nupic.database.ClientJobsDAO as cjdao
from nupic.swarming import HypersearchWorker, utils
from nupic.swarming.HypersearchV2 import HypersearchV2
from nupic.frameworks.opf.exp_generator.ExpGenerator import expGenerator

g_currentVerbosityLevel = 0
gCurrentSearch = None
gDefaultOptions = {'expDescJsonPath': None,
                  'expDescConfig': None,
                  'permutationsScriptPath': None,
                  'outputLabel': "swarm_out",
                  'permWorkDir': None,
                  'action': "run",
                  'searchMethod': "v2",
                  'timeout': None,
                  'exports': None,
                  'useTerminators': False,
                  'maxWorkers': 2,
                  'replaceReport': False,
                  'maxPermutations': None,
                  'genTopNDescriptions': 1}



class Verbosity(object):
  """ @private
  """
  WARNING = 0
  INFO = 1
  DEBUG = 2



def _termHandler(signal, frame):
  global gCurrentSearch
  try:
    jobrunner = gCurrentSearch
    jobID = jobrunner._HyperSearchRunner__searchJob.getJobID()
  except Exception as exc:
    print exc
  else:
    print "Canceling jobs due to receiving SIGTERM"
    cjdao.ClientJobsDAO.get().jobCancel(jobID)



def _setupInterruptHandling():
  signal.signal(signal.SIGTERM, _termHandler)
  signal.signal(signal.SIGINT, _termHandler)



def _verbosityEnabled(verbosityLevel):
  return verbosityLevel <= g_currentVerbosityLevel



def _emit(verbosityLevel, info):
  if _verbosityEnabled(verbosityLevel):
    print info



def _escape(s):
  """Escape commas, tabs, newlines and dashes in a string

  Commas are encoded as tabs
  """
  assert isinstance(s, str), \
        "expected %s but got %s; value=%s" % (type(str), type(s), s)
  s = s.replace('\\', '\\\\')
  s = s.replace('\n', '\\n')
  s = s.replace('\t', '\\t')
  s = s.replace(',', '\t')
  return s



def _engineServicesRunning():
  """ Return true if the engine services are running 
  """
  process = subprocess.Popen(["ps", "aux"], stdout=subprocess.PIPE)
  
  stdout = process.communicate()[0]
  result = process.returncode
  if result != 0:
    raise RuntimeError("Unable to check for running client job manager")
  
  # See if the CJM is running
  running = False
  for line in stdout.split("\n"):
    if "python" in line and "clientjobmanager.client_job_manager" in line:
      running = True
      break

  return running



def _runHyperSearch(runOptions):
  # Run HyperSearch
  startTime = time.time()
  search = _HyperSearchRunner(runOptions)
  # Save in global for the signal handler.
  global gCurrentSearch
  gCurrentSearch = search
  if runOptions['action'] in ('run', 'dryRun'):
    search.runNewSearch()
  else:
    search.pickupSearch()

  # Generate reports
  # Print results and generate report csv file
  model_params = _HyperSearchRunner.generateReport(
    options=runOptions,
    replaceReport=runOptions['replaceReport'],
    hyperSearchJob=search.peekSearchJob(),
    metricsKeys=search.getDiscoveredMetricsKeys())
  secs = time.time() - startTime
  hours = int(secs) / (60 * 60)
  secs -= hours * (60 * 60)
  minutes = int(secs) / 60
  secs -= minutes * 60
  print "Elapsed time (h:mm:ss): %d:%02d:%02d" % (hours, minutes, int(secs))
  jobID = search.peekSearchJob().getJobID()
  print "Hypersearch ClientJobs job ID: ", jobID

  #if _grokHyperSearchHasErrors(search.peekSearchJob()):
  #  # TODO: if job or some models failed, emit error messages
  #  sys.exit(1)
  #else:
  #  sys.exit(0)

  return model_params



def _injectDefaultOptions(options):
  global gDefaultOptions
  return dict(gDefaultOptions, **options)



def _validateOptions(options):
  if 'expDescJsonPath' not in options \
    and 'expDescConfig' not in options \
    and 'permutationsScriptPath' not in options:
    raise Exception("Options must contain one of the following: "
                    "expDescJsonPath, expDescConfig, or "
                    "permutationsScriptPath.")



def _generateExpFilesFromSwarmDescription(swarmDescriptionJson, outDir):
  # The expGenerator expects the JSON without newlines for an unknown reason.
  expDescConfig = json.dumps(swarmDescriptionJson)
  expDescConfig = expDescConfig.splitlines()
  expDescConfig = ''.join(expDescConfig)

  expGenerator([
    '--description=%s' % (expDescConfig),
    '--outDir=%s' % (outDir)])



def _runAction(runOptions):
  action = runOptions['action']
  # Print Grok HyperSearch results from the current or last run
  if action == 'report':
      return_value = _HyperSearchRunner.generateReport(
          options=runOptions,
          replaceReport=runOptions['replaceReport'],
          hyperSearchJob=None,
          metricsKeys=None)
  # Run HyperSearch via Grok
  elif action in ('run', 'dryRun', 'pickup'):
      return_value = _runHyperSearch(runOptions)
  else:
      raise Exception("Unhandled action: %s" % action)
  return return_value



def _checkOverwrite(options, outDir):
  overwrite = options['overwrite']
  if not overwrite:
    for name in ('description.py', 'permutations.py'):
      if os.path.exists(os.path.join(outDir, name)):
        raise RuntimeError("The %s file already exists and will be "
                           "overwritten by this tool. If it is OK to overwrite "
                           "this file, use the --overwrite option." % \
                           os.path.join(outDir, "description.py"))
  # The overwrite option has already been used, so should be removed from the
  # config at this point.
  del options['overwrite']



def runWithConfig(swarmConfig, outDir, options,
                    outputLabel='default', permWorkDir=None, verbosity=1):
  """
  Starts a swarm, given an dictionary configuration.
  @param swarmConfig {dict} A complete [swarm description](https://github.com/numenta/nupic/wiki/Running-Swarms#the-swarm-description) object.
  @param outDir {string} Path to write swarm details.
  @param outputLabel {string} Optional label for output (defaults to 'default').
  @param permWorkDir {string} Optional location of working directory (defaults
                              to None).
  @param verbosity {int} Optional (1,2,3) increasing verbosity of output.

  @returns {object} Model parameters
  """
  global g_currentVerbosityLevel
  g_currentVerbosityLevel = verbosity

  # Generate the description and permutations.py files in the same directory
  #  for reference.
  if permWorkDir is None:
    permWorkDir = os.getcwd()

  _checkOverwrite(options, outDir)

  _generateExpFilesFromSwarmDescription(swarmConfig, outDir)

  options['expDescConfig'] = swarmConfig
  options['outputLabel'] = outputLabel
  options['permWorkDir'] = permWorkDir

  runOptions = _injectDefaultOptions(options)
  _validateOptions(runOptions)

  return _runAction(runOptions)



def runWithJsonFile(expJsonFilePath, options, outputLabel, permWorkDir):
  """
  Starts a swarm, given a path to a JSON file containing configuration.

  This function is meant to be used with a CLI wrapper that passes command line
  arguments in through the options parameter.

  @param expJsonFilePath {string} Path to a JSON file containing the complete
                                 [swarm description](https://github.com/numenta/nupic/wiki/Running-Swarms#the-swarm-description).
  @param options {object} CLI options (optParse).
  @param outputLabel {string} Label for output.
  @param permWorkDir {string} Location of working directory.

  @returns {int} Swarm job id.
  """
  verbosity = options.verbosityCount
  optionsDict = vars(options)
  del optionsDict['verbosityCount']

  _setupInterruptHandling()

  with open(expJsonFilePath, 'rb') as jsonFile:
    expJsonConfig = json.loads(jsonFile.read())

  outDir = os.path.dirname(expJsonFilePath)
  return runWithConfig(expJsonConfig, outDir, optionsDict,
                       outputLabel=outputLabel, permWorkDir=permWorkDir,
                       verbosity=verbosity)



def runWithPermutationsScript(permutationsFilePath, options,
                                 outputLabel, permWorkDir):
  """
  Starts a swarm, given a path to a permutations.py script.

  This function is meant to be used with a CLI wrapper that passes command line
  arguments in through the options parameter.
<commit_msg>Turned outDir from arg into kwarg.<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

""" @file permutations_runner.py
"""

import collections
import imp
import csv
from datetime import datetime, timedelta
import os
import cPickle as pickle
import pprint
import shutil
import signal
import sys
import time
import subprocess
import tempfile
import uuid

from nupic.support import object_json as json
import nupic.database.ClientJobsDAO as cjdao
from nupic.swarming import HypersearchWorker, utils
from nupic.swarming.HypersearchV2 import HypersearchV2
from nupic.frameworks.opf.exp_generator.ExpGenerator import expGenerator

g_currentVerbosityLevel = 0
gCurrentSearch = None
gDefaultOptions = {'expDescJsonPath': None,
                  'expDescConfig': None,
                  'permutationsScriptPath': None,
                  'outputLabel': "swarm_out",
                  'permWorkDir': None,
                  'action': "run",
                  'searchMethod': "v2",
                  'timeout': None,
                  'exports': None,
                  'useTerminators': False,
                  'maxWorkers': 2,
                  'replaceReport': False,
                  'maxPermutations': None,
                  'genTopNDescriptions': 1}



class Verbosity(object):
  """ @private
  """
  WARNING = 0
  INFO = 1
  DEBUG = 2



def _termHandler(signal, frame):
  global gCurrentSearch
  try:
    jobrunner = gCurrentSearch
    jobID = jobrunner._HyperSearchRunner__searchJob.getJobID()
  except Exception as exc:
    print exc
  else:
    print "Canceling jobs due to receiving SIGTERM"
    cjdao.ClientJobsDAO.get().jobCancel(jobID)



def _setupInterruptHandling():
  signal.signal(signal.SIGTERM, _termHandler)
  signal.signal(signal.SIGINT, _termHandler)



def _verbosityEnabled(verbosityLevel):
  return verbosityLevel <= g_currentVerbosityLevel



def _emit(verbosityLevel, info):
  if _verbosityEnabled(verbosityLevel):
    print info



def _escape(s):
  """Escape commas, tabs, newlines and dashes in a string

  Commas are encoded as tabs
  """
  assert isinstance(s, str), \
        "expected %s but got %s; value=%s" % (type(str), type(s), s)
  s = s.replace('\\', '\\\\')
  s = s.replace('\n', '\\n')
  s = s.replace('\t', '\\t')
  s = s.replace(',', '\t')
  return s



def _engineServicesRunning():
  """ Return true if the engine services are running 
  """
  process = subprocess.Popen(["ps", "aux"], stdout=subprocess.PIPE)
  
  stdout = process.communicate()[0]
  result = process.returncode
  if result != 0:
    raise RuntimeError("Unable to check for running client job manager")
  
  # See if the CJM is running
  running = False
  for line in stdout.split("\n"):
    if "python" in line and "clientjobmanager.client_job_manager" in line:
      running = True
      break

  return running



def _runHyperSearch(runOptions):
  # Run HyperSearch
  startTime = time.time()
  search = _HyperSearchRunner(runOptions)
  # Save in global for the signal handler.
  global gCurrentSearch
  gCurrentSearch = search
  if runOptions['action'] in ('run', 'dryRun'):
    search.runNewSearch()
  else:
    search.pickupSearch()

  # Generate reports
  # Print results and generate report csv file
  model_params = _HyperSearchRunner.generateReport(
    options=runOptions,
    replaceReport=runOptions['replaceReport'],
    hyperSearchJob=search.peekSearchJob(),
    metricsKeys=search.getDiscoveredMetricsKeys())
  secs = time.time() - startTime
  hours = int(secs) / (60 * 60)
  secs -= hours * (60 * 60)
  minutes = int(secs) / 60
  secs -= minutes * 60
  print "Elapsed time (h:mm:ss): %d:%02d:%02d" % (hours, minutes, int(secs))
  jobID = search.peekSearchJob().getJobID()
  print "Hypersearch ClientJobs job ID: ", jobID

  #if _grokHyperSearchHasErrors(search.peekSearchJob()):
  #  # TODO: if job or some models failed, emit error messages
  #  sys.exit(1)
  #else:
  #  sys.exit(0)

  return model_params



def _injectDefaultOptions(options):
  global gDefaultOptions
  return dict(gDefaultOptions, **options)



def _validateOptions(options):
  if 'expDescJsonPath' not in options \
    and 'expDescConfig' not in options \
    and 'permutationsScriptPath' not in options:
    raise Exception("Options must contain one of the following: "
                    "expDescJsonPath, expDescConfig, or "
                    "permutationsScriptPath.")



def _generateExpFilesFromSwarmDescription(swarmDescriptionJson, outDir):
  # The expGenerator expects the JSON without newlines for an unknown reason.
  expDescConfig = json.dumps(swarmDescriptionJson)
  expDescConfig = expDescConfig.splitlines()
  expDescConfig = ''.join(expDescConfig)

  expGenerator([
    '--description=%s' % (expDescConfig),
    '--outDir=%s' % (outDir)])



def _runAction(runOptions):
  action = runOptions['action']
  # Print Grok HyperSearch results from the current or last run
  if action == 'report':
      return_value = _HyperSearchRunner.generateReport(
          options=runOptions,
          replaceReport=runOptions['replaceReport'],
          hyperSearchJob=None,
          metricsKeys=None)
  # Run HyperSearch via Grok
  elif action in ('run', 'dryRun', 'pickup'):
      return_value = _runHyperSearch(runOptions)
  else:
      raise Exception("Unhandled action: %s" % action)
  return return_value



def _checkOverwrite(options, outDir):
  overwrite = options['overwrite']
  if not overwrite:
    for name in ('description.py', 'permutations.py'):
      if os.path.exists(os.path.join(outDir, name)):
        raise RuntimeError("The %s file already exists and will be "
                           "overwritten by this tool. If it is OK to overwrite "
                           "this file, use the --overwrite option." % \
                           os.path.join(outDir, "description.py"))
  # The overwrite option has already been used, so should be removed from the
  # config at this point.
  del options['overwrite']



def runWithConfig(swarmConfig, options,
                  outDir=None, outputLabel='default',
                  permWorkDir=None, verbosity=1):
  """
  Starts a swarm, given an dictionary configuration.
  @param swarmConfig {dict} A complete [swarm description](https://github.com/numenta/nupic/wiki/Running-Swarms#the-swarm-description) object.
  @param outDir {string} Optional path to write swarm details.
  @param outputLabel {string} Optional label for output (defaults to 'default').
  @param permWorkDir {string} Optional location of working directory (defaults
                              to None).
  @param verbosity {int} Optional (1,2,3) increasing verbosity of output.

  @returns {object} Model parameters
  """
  global g_currentVerbosityLevel
  g_currentVerbosityLevel = verbosity

  # Generate the description and permutations.py files in the same directory
  #  for reference.
  if permWorkDir is None:
    permWorkDir = os.getcwd()

  _checkOverwrite(options, outDir)

  _generateExpFilesFromSwarmDescription(swarmConfig, outDir)

  options['expDescConfig'] = swarmConfig
  options['outputLabel'] = outputLabel
  options['permWorkDir'] = permWorkDir

  runOptions = _injectDefaultOptions(options)
  _validateOptions(runOptions)

  return _runAction(runOptions)



def runWithJsonFile(expJsonFilePath, options, outputLabel, permWorkDir):
  """
  Starts a swarm, given a path to a JSON file containing configuration.

  This function is meant to be used with a CLI wrapper that passes command line
  arguments in through the options parameter.

  @param expJsonFilePath {string} Path to a JSON file containing the complete
                                 [swarm description](https://github.com/numenta/nupic/wiki/Running-Swarms#the-swarm-description).
  @param options {object} CLI options (optParse).
  @param outputLabel {string} Label for output.
  @param permWorkDir {string} Location of working directory.

  @returns {int} Swarm job id.
  """
  verbosity = options.verbosityCount
  optionsDict = vars(options)
  del optionsDict['verbosityCount']

  _setupInterruptHandling()

  with open(expJsonFilePath, 'rb') as jsonFile:
    expJsonConfig = json.loads(jsonFile.read())

  outDir = os.path.dirname(expJsonFilePath)
  return runWithConfig(expJsonConfig, optionsDict, outDir=outDir,
                       outputLabel=outputLabel, permWorkDir=permWorkDir,
                       verbosity=verbosity)



def runWithPermutationsScript(permutationsFilePath, options,
                                 outputLabel, permWorkDir):
  """
  Starts a swarm, given a path to a permutations.py script.

  This function is meant to be used with a CLI wrapper that passes command line
  arguments in through the options parameter.

{'fraction_non_alphanumeric': 0.10571011774512869, 'fraction_numerical': 0.003959570699176826, 'mean_word_length': 3.1728260869565217, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '919369', 'n_tokens_mistral': 5642, 'n_tokens_neox': 5364, 'n_words': 1716}