<commit_before>import argparse
import os

import simplejson as json

from htmresearch.encoders import EncoderTypes
from htmresearch.frameworks.nlp.classification_model import ClassificationModel
from htmresearch.support.csv_helper import readCSV
from htmresearch.frameworks.nlp.model_factory import (
  ClassificationModelTypes,
  createModel)



class ModelSimilarityMetrics(object):
  pctOverlapOfInput = "pctOverlapOfInput"



class ImbuError(Exception):
  pass



class ImbuUnableToLoadModelError(ImbuError):
  def __init__(self, exc):
    self.exc = exc



def _loadJSON(jsonPath):
  try:
    with open(jsonPath, "rb") as fin:
      return json.load(fin)
  except IOError as e:
    print "Could not find JSON at '{}'.".format(jsonPath)
    raise e



def _loadNetworkConfig():
  root = (
    os.path.dirname(
      os.path.dirname(
        os.path.dirname(
          os.path.dirname(
            os.path.realpath(__file__)
          )
        )
      )
    )
  )
  return _loadJSON(os.path.join(root,
                                "projects/nlp/data/network_configs/imbu.json"))



class ImbuModels(object):

  defaultSimilarityMetric = ModelSimilarityMetrics.pctOverlapOfInput
  defaultModelType = "HTMNetwork"

  def __init__(self, cacheRoot, dataPath, loadPath, savePath,
      modelSimilarityMetric):
    self.cacheRoot = cacheRoot
    self.modelSimilarityMetric = (
      modelSimilarityMetric or self.defaultSimilarityMetric
    )
    self.loadPath = loadPath
    self.savePath = savePath
    self.dataPath = dataPath
    self.dataDict = self._loadTrainingData()


  def __repr__(self):
    return ("Imbu<cacheRoot={cacheRoot}, dataPath={dataPath}, "
            "loadPath={loadPath}, savePath={savePath}, "
            "modelSimilarityMetric={modelSimilarityMetric}>"
            .format(**self.__dict__))


  def _defaultModelFactoryKwargs(self):
    return dict(
      numLabels=len(self.dataDict),
      modelDir=self.savePath,
      classifierMetric=self.modelSimilarityMetric)


  def modelFactory(self, modelType, **kwargs):

    modelType = modelType or self.defaultModelType

    kwargs.update(**self._defaultModelFactoryKwargs())

    if modelType == "CioWordFingerprint":
      kwargs.update(retina=kwargs.get("retina"),
                    apiKey=kwargs.get("apiKey"),
                    fingerprintType=EncoderTypes.word,
                    cacheRoot=self.cacheRoot)

    elif modelType == "CioDocumentFingerprint":
      kwargs.update(retina=kwargs.get("retina"),
                    apiKey=kwargs.get("apiKey"),
                    fingerprintType=EncoderTypes.document,
                    cacheRoot=self.cacheRoot)
    elif modelType == "HTMNetwork":
      kwargs.update(networkConfig=_loadNetworkConfig(),
                    inputFilePath=None,
                    prepData=False,
                    retinaScaling=1.0)

    model = createModel(modelType, **kwargs)


    model.verbosity = 0

    return model


  def loadModel(self, *modelFactoryArgs, **modelFactoryKwargs):

    try:
      model = ClassificationModel.load(self.loadPath)
    except IOError as exc:
      model = self.modelFactory(*modelFactoryArgs, **modelFactoryKwargs)
      self.train(model)

    return model


  def _loadTrainingData(self):<commit_msg>line spacing<commit_after>import argparse
import os

import simplejson as json

from htmresearch.encoders import EncoderTypes
from htmresearch.frameworks.nlp.classification_model import ClassificationModel
from htmresearch.support.csv_helper import readCSV
from htmresearch.frameworks.nlp.model_factory import (
  ClassificationModelTypes,
  createModel)



class ModelSimilarityMetrics(object):
  pctOverlapOfInput = "pctOverlapOfInput"



class ImbuError(Exception):
  pass



class ImbuUnableToLoadModelError(ImbuError):
  def __init__(self, exc):
    self.exc = exc



def _loadJSON(jsonPath):
  try:
    with open(jsonPath, "rb") as fin:
      return json.load(fin)
  except IOError as e:
    print "Could not find JSON at '{}'.".format(jsonPath)
    raise e



def _loadNetworkConfig():
  root = (
    os.path.dirname(
      os.path.dirname(
        os.path.dirname(
          os.path.dirname(
            os.path.realpath(__file__)
          )
        )
      )
    )
  )
  return _loadJSON(os.path.join(root,
                                "projects/nlp/data/network_configs/imbu.json"))



class ImbuModels(object):

  defaultSimilarityMetric = ModelSimilarityMetrics.pctOverlapOfInput
  defaultModelType = "HTMNetwork"

  def __init__(self, cacheRoot, dataPath, loadPath, savePath,
      modelSimilarityMetric):
    self.cacheRoot = cacheRoot
    self.modelSimilarityMetric = (
      modelSimilarityMetric or self.defaultSimilarityMetric
    )
    self.loadPath = loadPath
    self.savePath = savePath
    self.dataPath = dataPath
    self.dataDict = self._loadTrainingData()


  def __repr__(self):
    return ("Imbu<cacheRoot={cacheRoot}, dataPath={dataPath}, "
            "loadPath={loadPath}, savePath={savePath}, "
            "modelSimilarityMetric={modelSimilarityMetric}>"
            .format(**self.__dict__))


  def _defaultModelFactoryKwargs(self):
    return dict(
      numLabels=len(self.dataDict),
      modelDir=self.savePath,
      classifierMetric=self.modelSimilarityMetric)


  def modelFactory(self, modelType, **kwargs):

    modelType = modelType or self.defaultModelType

    kwargs.update(**self._defaultModelFactoryKwargs())

    if modelType == "CioWordFingerprint":
      kwargs.update(retina=kwargs.get("retina"),
                    apiKey=kwargs.get("apiKey"),
                    fingerprintType=EncoderTypes.word,
                    cacheRoot=self.cacheRoot)

    elif modelType == "CioDocumentFingerprint":
      kwargs.update(retina=kwargs.get("retina"),
                    apiKey=kwargs.get("apiKey"),
                    fingerprintType=EncoderTypes.document,
                    cacheRoot=self.cacheRoot)

    elif modelType == "HTMNetwork":
      kwargs.update(networkConfig=_loadNetworkConfig(),
                    inputFilePath=None,
                    prepData=False,
                    retinaScaling=1.0)

    model = createModel(modelType, **kwargs)


    model.verbosity = 0

    return model


  def loadModel(self, *modelFactoryArgs, **modelFactoryKwargs):

    try:
      model = ClassificationModel.load(self.loadPath)
    except IOError as exc:
      model = self.modelFactory(*modelFactoryArgs, **modelFactoryKwargs)
      self.train(model)

    return model


  def _loadTrainingData(self):
{'fraction_non_alphanumeric': 0.0950920245398773, 'fraction_numerical': 0.0009202453987730061, 'mean_word_length': 2.935425467712734, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2867941', 'n_tokens_mistral': 2016, 'n_tokens_neox': 1881, 'n_words': 344}