<commit_before>#!/usr/bin/env ruby

module TmlTokenParser
  class MiscParser < GeneralParser

    @@dots = {
      'pt' => true,
    }

    @@divisions = {
      ' ' => "space",
      '[' => "left square bracket",
      ']' => "right square bracket",
      ';' => "barline",
    }

    def parse
      sym = nil
      args = {}

      err = catch :unrecognized do
        sym, args = do_misc
        nil
      end

      @builder.send(sym, args) unless err
      unrecognized(@token, err) if err

    end

    private

    # really simple for now, just match these verbatim
    def do_misc
      case
      when @@dots.has_key?(@token)
        return :dot, {'form' => 'div'}
      when @@divisions.has_key?(@token)
        return :comment, "#{@@divisions[@token]} in example"
      else
        throw :unrecognized, 'no_key'
      end
    end

  end
end<commit_msg>Adding <supplied> tag for brackets.<commit_after>#!/usr/bin/env ruby

module TmlTokenParser
  class MiscParser < GeneralParser

    # a bit kludgy, but this is the best I can come up with for now
    @@in_supplied = false

    @@dots = {
      'pt' => true,
    }

    @@divisions = {
      ' ' => "space",
      ';' => "barline",
    }

    def parse
      sym = nil
      args = {}

      skip_output = catch :no_output do
        sym, args = do_misc
        nil
      end

      return if skip_output || sym.nil?
      @builder.send(sym, args)

    end

    private

    # really simple for now, just match these verbatim
    def do_misc
      case
      when @@dots.has_key?(@token)
        return :dot, {'form' => 'div'}

      when @token == ']'
        @@in_supplied = false
        throw :no_output

      when @token == '['
        @@in_supplied = true
        start_supplied()
        throw :no_output

      when @@divisions.has_key?(@token)
        return :comment, "#{@@divisions[@token]} in example"

      else
        unrecognized(@token, 'no_key')
        throw :no_output
      end
    end

    def start_supplied
      @builder.supplied {
        @parent.parse_next until @@in_supplied == false
      }
    end



  end
end
{'fraction_non_alphanumeric': 0.12102515424774561, 'fraction_numerical': 0.0, 'mean_word_length': 1.8956043956043955, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2740246', 'n_tokens_mistral': 718, 'n_tokens_neox': 656, 'n_words': 184}