<commit_before># encoding: UTF-8

# Copyright 2012 Twitter, Inc
# http://www.apache.org/licenses/LICENSE-2.0

module TwitterCldr
  module Tokenizers
    class NumberTokenizer < Base
      VALID_TYPES = [:decimal, :percent, :currency]
      TOKEN_SPLITTER_REGEX = /([^0*#,\.]*)([0#,\.]+)([^0*#,\.]*)$/
      TOKEN_TYPE_REGEXES = {
        :pattern => { :regex => /[0?#,\.]*/ },
        :plaintext => { :regex => // }
      }

      def initialize(options = {})<commit_msg>Adding priorities to regexes in number tokenizer<commit_after># encoding: UTF-8

# Copyright 2012 Twitter, Inc
# http://www.apache.org/licenses/LICENSE-2.0

module TwitterCldr
  module Tokenizers
    class NumberTokenizer < Base
      VALID_TYPES = [:decimal, :percent, :currency]
      TOKEN_SPLITTER_REGEX = /([^0*#,\.]*)([0#,\.]+)([^0*#,\.]*)$/
      TOKEN_TYPE_REGEXES = {
        :pattern => { :regex => /[0?#,\.]*/, :priority => 1 },
        :plaintext => { :regex => //, :priority => 2 }
      }

      def initialize(options = {})
{'fraction_non_alphanumeric': 0.20416253716551042, 'fraction_numerical': 0.023785926660059464, 'mean_word_length': 2.840304182509506, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '369526', 'n_tokens_mistral': 378, 'n_tokens_neox': 360, 'n_words': 68}