<commit_before># encoding: UTF-8

module Buildbox
  class Script
    MAGICAL_LINE_REGEX = /(buildbox:(?:begin|end)\:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}(?:\:\d+)?)/
    MAGICAL_LINE_DIVIDER_REGEX = /:/

    def self.parse(line)
      json = line.chomp.match(/buildbox-begin\:(.+)?\:buildbox-end/)[1]

      JSON.parse(json)
    end

    def self.matches?(line)
      !!line.chomp.match(/buildbox-begin\:(.+)\:buildbox-end/)
    end

    def self.split(chunk)
      chunk.split(/(buildbox-begin\:.+?\:buildbox-end)/)
    end

    def initialize
      @commands = []
    end

    def queue(identifier, command)
      @commands << { :identifier => identifier, :command => command }
    end

    def to_s
      buffer = []

      @commands.each do |item|
        buffer << magical_line(item.merge(:action => "begin"))
        buffer << item[:command]
        buffer << magical_line(item.merge(:action => "end", :exit_status => "$?"))
      end

      buffer.join("\n")
    end

    private

    def magical_line(json)
      line = [ "buildbox-begin", JSON.dump(json), "buildbox-end" ]

      %{echo #{line.join(":").inspect};}
    end
  end
end<commit_msg>Small change to the buildbox script generator<commit_after># encoding: UTF-8

module Buildbox
  class Script
    MAGICAL_LINE_REGEX = /(buildbox:(?:begin|end)\:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}(?:\:\d+)?)/
    MAGICAL_LINE_DIVIDER_REGEX = /:/

    def self.parse(line)
      json = line.chomp.match(/buildbox-begin\:(.+)?\:buildbox-end/)[1]

      JSON.parse(json)
    end

    def self.matches?(line)
      !!line.chomp.match(/buildbox-begin\:(.+)\:buildbox-end/)
    end

    def self.split(chunk)
      chunk.split(/(buildbox-begin\:.+?\:buildbox-end)/)
    end

    def initialize
      @commands = []
    end

    def queue(identifier, command)
      @commands << { :identifier => identifier, :command => command }
    end

    def to_s
      script = []

      @commands.each do |item|
        script << magical_line(item.merge(:action => "begin"))
        script << item[:command]
        script << magical_line(item.merge(:action => "end", :exit_status => "$?"))
      end

      script.join("\n")
    end

    private

    def magical_line(json)
      line = [ "buildbox-begin", JSON.dump(json), "buildbox-end" ]

      %{echo #{line.join(":").inspect};}
    end
  end
end
{'fraction_non_alphanumeric': 0.18674698795180722, 'fraction_numerical': 0.0068846815834767644, 'mean_word_length': 2.5173978819969745, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7354015', 'n_tokens_mistral': 838, 'n_tokens_neox': 795, 'n_words': 139}