<commit_before># -*- coding: utf-8 -*-

# :title: A simple Tokenizer for NLP Tasks.
# :main: README.rdoc

# A namespace for all project related stuff.
module Tokenizer

  class Tokenizer
    FS = Regexp.new('[[:blank:]]+')
#    PRE = '[{(\\`"‚„†‡‹‘’“”•–—›'
#    POST = %w| ] } ' ` " ) , ; : \ ! \ ? \ % ‚ „ … † ‡ ‰ ‹ ‘ ’ “ ” • – — › |
    
    SIMPLE_PRE = [] # spanish marks
    PAIR_PRE = ['(', '{', '[']
    SIMPLE_POST = ['!', '?', ',', ':', ';', '.']
    PAIR_POST = [')', '}', ']']
    PRE_N_POST = ['"', "'"]

    PRE = SIMPLE_PRE + PAIR_PRE
    POST = SIMPLE_POST + PAIR_POST
    
    def initialize(lang=:de)
      @lang = lang
      @braces = []
    end

    def tokenize(str)
      tokens = []
      token = ''
      output = ''
      fields = str.chomp.split(FS)
#      puts "Fields: #{fields}"
      if fields.empty?
        return [""]
      end<commit_msg>Remove unused valiables<commit_after># -*- coding: utf-8 -*-

# :title: A simple Tokenizer for NLP Tasks.
# :main: README.rdoc

# A namespace for all project related stuff.
module Tokenizer

  class Tokenizer
    FS = Regexp.new('[[:blank:]]+')
#    PRE = '[{(\\`"‚„†‡‹‘’“”•–—›'
#    POST = %w| ] } ' ` " ) , ; : \ ! \ ? \ % ‚ „ … † ‡ ‰ ‹ ‘ ’ “ ” • – — › |
    
    SIMPLE_PRE = [] # spanish marks
    PAIR_PRE = ['(', '{', '[']
    SIMPLE_POST = ['!', '?', ',', ':', ';', '.']
    PAIR_POST = [')', '}', ']']
    PRE_N_POST = ['"', "'"]

    PRE = SIMPLE_PRE + PAIR_PRE
    POST = SIMPLE_POST + PAIR_POST
    
    def initialize(lang=:de)
      @lang = lang
      @braces = []
    end

    def tokenize(str)
      output = ''
      fields = str.chomp.split(FS)
#      puts "Fields: #{fields}"
      if fields.empty?
        return [""]
      end
{'fraction_non_alphanumeric': 0.23529411764705882, 'fraction_numerical': 0.0011648223645894002, 'mean_word_length': 2.0954954954954954, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5255810', 'n_tokens_mistral': 707, 'n_tokens_neox': 690, 'n_words': 141}