<commit_before>module Tritium
  module Serializer
    require_relative 'tritium.pb'

    def self.process_file(ts_file)
      filename = ts_file.split("/").last
      path = ts_file.split("/")[0..-2].join("/")
      parser = Tritium::Parser::Parser.new(:filename => filename, :path => path)
      root = parser.parse
      script = Script.new(:root => convert_instruction(root))
      result = script.encode
      puts result.to_s.size
      puts result.to_s.inspect
    end
    
    def self.convert_instruction(ins)
      obj = Script::Instruction.new
      set_name(obj, ins)
      obj["type"] = instruction_type(ins)
      obj.children = ins.statements.collect { |c| convert_instruction(c) }
      obj.arguments = ins.args.collect { |c| convert_instruction(c) } if ins.respond_to?("args")<commit_msg>Only serialize positional args<commit_after>module Tritium
  module Serializer
    require_relative 'tritium.pb'

    def self.process_file(ts_file)
      filename = ts_file.split("/").last
      path = ts_file.split("/")[0..-2].join("/")
      parser = Tritium::Parser::Parser.new(:filename => filename, :path => path)
      root = parser.parse
      script = Script.new(:root => convert_instruction(root))
      result = script.encode
      puts result.to_s.inspect
    end
    
    def self.convert_instruction(ins)
      obj = Script::Instruction.new
      set_name(obj, ins)
      obj["type"] = instruction_type(ins)
      obj.children = ins.statements.collect { |c| convert_instruction(c) }
      obj.arguments = ins.pos_args.collect { |c| convert_instruction(c) } if ins.respond_to?("args")
{'fraction_non_alphanumeric': 0.13616489693941286, 'fraction_numerical': 0.0024984384759525295, 'mean_word_length': 3.193717277486911, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6596074', 'n_tokens_mistral': 510, 'n_tokens_neox': 501, 'n_words': 94}