<commit_before>module Armok
  class Entity
    attr_accessor :id, :subtype, :tokens

    def initialize(id='', subtype='', s='')
      @id = id
      @subtype = subtype
      @tokens = Array.new
      parse(s) unless s.empty?
    end

    def parse(s)
      s.scan(/#{SPACE}#{LB}([^\]]+)#{RB}#{SPACE}/m).each {|t|
        @tokens << Token.new(t[0])
      }
    end

    def to_s
      "[#{subtype}:#{id}]\n#{tokens.join('\n')}"
    end

  end
end<commit_msg>Use NEWLINE<commit_after>module Armok
  class Entity
    attr_accessor :id, :subtype, :tokens

    def initialize(id='', subtype='', s='')
      @id = id
      @subtype = subtype
      @tokens = Array.new
      parse(s) unless s.empty?
    end

    def parse(s)
      s.scan(/#{SPACE}#{LB}([^\]]+)#{RB}#{SPACE}/m).each {|t|
        @tokens << Token.new(t[0])
      }
    end

    def to_s
      "[#{subtype}:#{id}]#{NEWLINE}#{tokens.join(NEWLINE)}"
    end

  end
end
{'fraction_non_alphanumeric': 0.2, 'fraction_numerical': 0.002162162162162162, 'mean_word_length': 2.4044117647058822, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2928236', 'n_tokens_mistral': 358, 'n_tokens_neox': 362, 'n_words': 70}