<commit_before>#!/usr/bin/python

# Main entry-point for the neutrino parser.


# Set up appropriate path. Bit of a hack but hey.
import sys
import os.path
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))


# Remaining imports.
import optparse
import parser
import plankton
import token


# Encapsulates stats relating to the main script.
class Main(object):

  def __init__(self):
    self.args = None
    self.flags = None

  # Builds and returns a new option parser for the flags understood by this
  # script.
  def build_option_parser(self):
    parser = optparse.OptionParser()
    parser.add_option('--expression', action='append', default=[])
    return parser

  # Parses the script arguments, storing the values in the appropriate fields.
  def parse_arguments(self):
    parser = self.build_option_parser()
    (self.flags, self.args) = parser.parse_args()

  # Main entry-point.
  def run(self):
    self.parse_arguments()
    self.run_expressions()

  # Processes any --expression arguments.
  def run_expressions(self):
    for expr in self.flags.expression:
      tokens = token.tokenize(expr)
      ast = parser.Parser(tokens).parse_expression()
      print plankton.base64encode(ast)


if __name__ == '__main__':
  Main().run()<commit_msg>Added a --filter mode to the main script that passes output through except for plankton which is converted to the encoded data.<commit_after>#!/usr/bin/python

# Main entry-point for the neutrino parser.


# Set up appropriate path. Bit of a hack but hey.
import sys
import os.path
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))


# Remaining imports.
import optparse
import parser
import plankton
import re
import token


# Encapsulates stats relating to the main script.
class Main(object):

  def __init__(self):
    self.args = None
    self.flags = None

  # Builds and returns a new option parser for the flags understood by this
  # script.
  def build_option_parser(self):
    parser = optparse.OptionParser()
    parser.add_option('--expression', action='append', default=[])
    parser.add_option('--filter', action='store_true', default=False)
    return parser

  # Parses the script arguments, storing the values in the appropriate fields.
  def parse_arguments(self):
    parser = self.build_option_parser()
    (self.flags, self.args) = parser.parse_args()

  # If the filter option is set, filters input and return True. Otherwise
  # returns False.
  def run_filter(self):
    if not self.flags.filter:
      return False
    pattern = re.compile(r'^p64/([a-zA-Z0-9=]+)$')
    for line in sys.stdin:
      match = pattern.match(line)
      if match:
        code = match.group(1)
        data = plankton.base64decode(code)
        print data
      else:
        print line
    return True

  # Main entry-point.
  def run(self):
    self.parse_arguments()
    if self.run_filter():
      return
    self.run_expressions()

  # Processes any --expression arguments.
  def run_expressions(self):
    for expr in self.flags.expression:
      tokens = token.tokenize(expr)
      ast = parser.Parser(tokens).parse_expression()
      print "p64/%s" % plankton.base64encode(ast)


if __name__ == '__main__':
  Main().run()
{'fraction_non_alphanumeric': 0.10627719080174021, 'fraction_numerical': 0.004039776258545681, 'mean_word_length': 3.3975409836065573, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2616592', 'n_tokens_mistral': 993, 'n_tokens_neox': 954, 'n_words': 299}