<commit_before>"""alloccli subcommand for viewing a list of tokens."""
from alloc import alloc

class ktokens(alloc):
  """Retrieve token information."""

  # Setup the options that this cli can accept
  ops = []
  ops.append((''  , 'help           ', 'Show this help.'))
  ops.append((''  , 'csv=[WHEN]     ', 'Return the results in CSV format. WHEN can be "auto",\n'
                                       '"never" or "always". If WHEN is omitted, assume "always".'))
  ops.append(('q' , 'quiet          ', 'Run with no output except errors.'))
  ops.append(('k:', 'key=KEY        ', 'An 8 character email subject line key.'))
  ops.append(('i:', 'id=ID          ', 'A comment id.'))

  # Specify some header and footer text for the help text
  help_text = "Usage: %s [OPTIONS] [FILE]\n"
  help_text += __doc__
  help_text += """\n\n%s

Examples:

# Print out a list of tokens
alloc token --key <KEY>"""

  def run(self, command_list):
    """Execute subcommand."""

    # Get the command line arguments into a dictionary
    o, remainder_ = self.get_args(command_list, self.ops, self.help_text)

    # Got this far, then authenticate
    self.authenticate()

    # Initialize some variables
    self.quiet = o['quiet']

    # This is the data format that is exported and imported
    fields = ["tokenEntity", "Entity", "tokenEntityID", "ID", "tokenHash", "Key"]
    searchops = {}
    searchops['tokenHash'] = o['key'] or ''
    searchops['tokenEntity'] = 'comment'
    searchops['tokenEntityID'] = o['id'] or ''

    if o['key'] or o['id']:
      tokens = self.get_list("token", searchops)
      if tokens:
        self.print_table("token", tokens, fields)

<commit_msg>Also fixed field list for ktokens.<commit_after>"""alloccli subcommand for viewing a list of tokens."""
from alloc import alloc

class ktokens(alloc):
  """Retrieve token information."""

  # Setup the options that this cli can accept
  ops = []
  ops.append((''  , 'help           ', 'Show this help.'))
  ops.append((''  , 'csv=[WHEN]     ', 'Return the results in CSV format. WHEN can be "auto",\n'
                                       '"never" or "always". If WHEN is omitted, assume "always".'))
  ops.append(('q' , 'quiet          ', 'Run with no output except errors.'))
  ops.append(('k:', 'key=KEY        ', 'An 8 character email subject line key.'))
  ops.append(('i:', 'id=ID          ', 'A comment id.'))

  # Specify some header and footer text for the help text
  help_text = "Usage: %s [OPTIONS] [FILE]\n"
  help_text += __doc__
  help_text += """\n\n%s

Examples:

# Print out a list of tokens
alloc token --key <KEY>"""

  def run(self, command_list):
    """Execute subcommand."""

    # Get the command line arguments into a dictionary
    o, remainder_ = self.get_args(command_list, self.ops, self.help_text)

    # Got this far, then authenticate
    self.authenticate()

    # Initialize some variables
    self.quiet = o['quiet']

    # This is the data format that is exported and imported
    fields = ["tokenEntity", "tokenEntityID", "tokenHash"]
    searchops = {}
    searchops['tokenHash'] = o['key'] or ''
    searchops['tokenEntity'] = 'comment'
    searchops['tokenEntityID'] = o['id'] or ''

    if o['key'] or o['id']:
      tokens = self.get_list("token", searchops)
      if tokens:
        self.print_table("token", tokens, fields)


{'fraction_non_alphanumeric': 0.1506276150627615, 'fraction_numerical': 0.0005977286312014345, 'mean_word_length': 2.9284037558685445, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '776030', 'n_tokens_mistral': 1061, 'n_tokens_neox': 1014, 'n_words': 344}