<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import itertools
import os
import numpy
import random

from cortipy.cortical_client import CorticalClient
from cortipy.exceptions import UnsuccessfulEncodingError
from fluent.encoders.language_encoder import LanguageEncoder



class CioEncoder(LanguageEncoder):
  """
  A language encoder using the Cortical.io API.

  The encoder queries the Cortical.io REST API via the cortipy module, which
  returns data in the form of "fingerprints". These representations are
  converted to binary SDR arrays with this Cio encoder.
  """

  def __init__(self, w=128, h=128, cacheDir="./cache", verbosity=0):
    if 'CORTICAL_API_KEY' not in os.environ:
      print ("Missing CORTICAL_API_KEY environment variable. If you have a "
        "key, set it with $ export CORTICAL_API_KEY=api_key\n"
        "You can retrieve a key by registering for the REST API at "
        "http://www.cortical.io/resources_apikey.html")
      raise OSError("Missing API key.")

    self.apiKey         = os.environ['CORTICAL_API_KEY']
    self.client         = CorticalClient(self.apiKey, cacheDir=cacheDir)
    self.targetSparsity = 5.0
    self.w              = w
    self.h              = h
    self.n              = w*h
    self.verbosity      = verbosity


  def encode(self, text):
    """
    Encodes the input text w/ a cortipy client. The client returns a
    dictionary of "fingerprint" info, including the SDR bitmap.

    @param  text    (str)             A non-tokenized sample of text.
    @return         (dict)            Result from the cortipy client. The bitmap
                                      encoding is at
                                      encoding["fingerprint"]["positions"].
    """
    if not text:
      return None
    try:
      encoding = self.client.getTextBitmap(text)
    except UnsuccessfulEncodingError:
      if self.verbosity > 0:
        print ("\tThe client returned no encoding for the text \'{0}\', so "
               "we'll use the encoding of the token that is least frequent in "
               "the corpus.".format(text))
      encoding = self._subEncoding(text)

    return encoding


  def decode(self, encoding, numTerms=10):
    """
    Converts an SDR back into the most likely word or words.

    By default, the most likely term will be returned. If numTerms is
    specified, then the Cortical.io API will attempt to return that many;
    otherwise the standard is 10. The return value will be a sequence of
    (term, weight) tuples, where higher weights imply the corresponding term
    better matches the encoding.

    @param  encoding        (list)            Bitmap encoding.
    @param  numTerms        (int)             The max number of terms to return.
    @return                 (list)            List of dictionaries, where keys
                                              are terms and likelihood scores.
    """
    terms = client.bitmapToTerms(encoding, numTerms=numTerms)
    # Convert cortipy response to list of tuples (term, weight)
    return [((term["term"], term["score"])) for term in terms]


  def _subEncoding(self, text, method="df"):
    """
    @param text             (str)             A non-tokenized sample of text.
    @return encoding        (dict)            Fingerprint from cortipy client.
                                              An empty dictionary of the text
                                              could not be encoded.
    """
    tokens = list(itertools.chain.from_iterable(
      [t.split(',') for t in self.client.tokenize(text)]))
    try:
      if method == "df":
        encoding = min([self.client.getBitmap(t) for t in tokens],
                        key=lambda x: x["df"])
      elif method == "keyword":
        # Take a union of the bitmaps
        union = numpy.zeros(0)
        for t in tokens:
          bitmap = self.client.getBitmap(t)["fingerprint"]["positions"]
          union = numpy.union1d(bitmap, union)

        # Sample to remain sparse
        count = len(union)
        sparsity = int((self.targetSparsity / 100) * self.n)
        sampleIndices = random.sample(xrange(count), min(count, sparsity))

        # Populate encoding
        encoding = {}
        encoding["text"] = text
        encoding["sparsity"] = min(sparsity, count) * 100 / float(self.n)
        encoding["df"] = 0.0
        encoding["height"] = self.h
        encoding["width"] = self.w
        encoding["score"] = 0.0
        encoding["fingerprint"] = {}
        positions = union[sampleIndices]
        encoding["fingerprint"]["positions"] = numpy.sort(positions).tolist()
        encoding["pos_types"] = []
      else:
        raise TypeError("method must be either \'df\' or \'keyword\'")
    except UnsuccessfulEncodingError:
      if self.verbosity > 0:
        print ("\tThe client returned no substitute encoding for the text "
               "\'{0}\', so we encode with None.".format(text))
      encoding = None

    return encoding

<commit_msg>take the most common indices<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import itertools
import os
import numpy
import random

from cortipy.cortical_client import CorticalClient
from cortipy.exceptions import UnsuccessfulEncodingError
from fluent.encoders.language_encoder import LanguageEncoder

from collections import Counter



class CioEncoder(LanguageEncoder):
  """
  A language encoder using the Cortical.io API.

  The encoder queries the Cortical.io REST API via the cortipy module, which
  returns data in the form of "fingerprints". These representations are
  converted to binary SDR arrays with this Cio encoder.
  """

  def __init__(self, w=128, h=128, cacheDir="./cache", verbosity=0):
    if 'CORTICAL_API_KEY' not in os.environ:
      print ("Missing CORTICAL_API_KEY environment variable. If you have a "
        "key, set it with $ export CORTICAL_API_KEY=api_key\n"
        "You can retrieve a key by registering for the REST API at "
        "http://www.cortical.io/resources_apikey.html")
      raise OSError("Missing API key.")

    self.apiKey         = os.environ['CORTICAL_API_KEY']
    self.client         = CorticalClient(self.apiKey, cacheDir=cacheDir)
    self.targetSparsity = 5.0
    self.w              = w
    self.h              = h
    self.n              = w*h
    self.verbosity      = verbosity


  def encode(self, text):
    """
    Encodes the input text w/ a cortipy client. The client returns a
    dictionary of "fingerprint" info, including the SDR bitmap.

    @param  text    (str)             A non-tokenized sample of text.
    @return         (dict)            Result from the cortipy client. The bitmap
                                      encoding is at
                                      encoding["fingerprint"]["positions"].
    """
    if not text:
      return None
    try:
      encoding = self.client.getTextBitmap(text)
    except UnsuccessfulEncodingError:
      if self.verbosity > 0:
        print ("\tThe client returned no encoding for the text \'{0}\', so "
               "we'll use the encoding of the token that is least frequent in "
               "the corpus.".format(text))
      encoding = self._subEncoding(text)

    return encoding


  def decode(self, encoding, numTerms=10):
    """
    Converts an SDR back into the most likely word or words.

    By default, the most likely term will be returned. If numTerms is
    specified, then the Cortical.io API will attempt to return that many;
    otherwise the standard is 10. The return value will be a sequence of
    (term, weight) tuples, where higher weights imply the corresponding term
    better matches the encoding.

    @param  encoding        (list)            Bitmap encoding.
    @param  numTerms        (int)             The max number of terms to return.
    @return                 (list)            List of dictionaries, where keys
                                              are terms and likelihood scores.
    """
    terms = client.bitmapToTerms(encoding, numTerms=numTerms)
    # Convert cortipy response to list of tuples (term, weight)
    return [((term["term"], term["score"])) for term in terms]


  def _subEncoding(self, text, method="df"):
    """
    @param text             (str)             A non-tokenized sample of text.
    @return encoding        (dict)            Fingerprint from cortipy client.
                                              An empty dictionary of the text
                                              could not be encoded.
    """
    tokens = list(itertools.chain.from_iterable(
      [t.split(',') for t in self.client.tokenize(text)]))
    try:
      if method == "df":
        encoding = min([self.client.getBitmap(t) for t in tokens],
                        key=lambda x: x["df"])
      elif method == "keyword":
        # Take a union of the bitmaps
        union = numpy.zeros(0)
        counts = Counter()
        for t in tokens:
          bitmap = self.client.getBitmap(t)["fingerprint"]["positions"]
          counts.update(bitmap)
          union = numpy.union1d(bitmap, union)

        # Sample to remain sparse
        max_sparsity = int((self.targetSparsity / 100) * self.n)
        w = min(len(union), max_sparsity)
        sampleIndices = counts.most_common(w)

        # Populate encoding
        encoding = {}
        encoding["text"] = text
        encoding["sparsity"] = w * 100 / float(self.n)
        encoding["df"] = 0.0
        encoding["height"] = self.h
        encoding["width"] = self.w
        encoding["score"] = 0.0
        encoding["fingerprint"] = {}
        positions = union[sampleIndices]
        encoding["fingerprint"]["positions"] = numpy.sort(positions).tolist()
        encoding["pos_types"] = []
      else:
        raise TypeError("method must be either \'df\' or \'keyword\'")
    except UnsuccessfulEncodingError:
      if self.verbosity > 0:
        print ("\tThe client returned no substitute encoding for the text "
               "\'{0}\', so we encode with None.".format(text))
      encoding = None

    return encoding


{'fraction_non_alphanumeric': 0.10901187092459455, 'fraction_numerical': 0.005684668115699716, 'mean_word_length': 2.1243144424131626, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2243829', 'n_tokens_mistral': 3213, 'n_tokens_neox': 3103, 'n_words': 1165}