<commit_before>#!/usr/bin/env ruby
# encoding: utf-8

origin = <<-END
 - Сынок, чего это     от тебя   зигами пахнет,
опять на Манежную площадь ходил?

 - Нет мама, я в метро ехал, там  назиговано было!!

  

Четырнадцать, восемьдесять восемь: 14/88.
Вот так блять
END
origin.chomp!

RU_LEX = /^[А-Яа-я]+$/u
EN_LEX = /^[A-Za-z]+$/u
EOL = /^\n+$/u
SEP = /^[*=_\/\\ ]$/u
PUN = /^(\.|\!|\?)$/u
SPUN = /^(\,|\[|\]|\(|\)|\-|:|;)$/u
DIG = /^[0-9]+$/u
DIL = /^[А-Яа-яA-Za-z0-9]+$/u
EMPTY = ''

class MetaArray < Array
  def [] id
    super(id) or begin
      self.class.new.tap do |element|
        self[id] = element
      end
    end
  end
end

def parse(origin)
  text = MetaArray.new

  paragraph_id = 0
  sentence_id = 0
  subsentence_id = 0

  token = ''

  origin.each_char do |c|
    puts "[#{token.inspect}] ← #{c.inspect}"
    case c
      when EOL then begin
        case token
          when EMPTY then token << c
          when EOL then begin
            token = ''
            paragraph_id += 1
            sentence_id = 0
            subsentence_id = 0
          end
          else
            text[paragraph_id][sentence_id][subsentence_id] << token
            token = c
        end
      end
      when SEP then begin
        case token
          when EMPTY
          else
            text[paragraph_id][sentence_id][subsentence_id] << token
            while text[paragraph_id][sentence_id][subsentence_id].last == c
              text[paragraph_id][sentence_id][subsentence_id].pop
            end
            text[paragraph_id][sentence_id][subsentence_id] << c
            token = ''
        end
      end
      when PUN then begin
        case token
          when EMPTY
          else
            text[paragraph_id][sentence_id][subsentence_id] << token
            text[paragraph_id][sentence_id][subsentence_id] << c
            token = ''
            sentence_id += 1
            subsentence_id = 0
        end
      end
      when SPUN then begin
        case token
          when EMPTY
          else
            text[paragraph_id][sentence_id][subsentence_id] << token
            text[paragraph_id][sentence_id][subsentence_id] << c
            token = ''
            subsentence_id += 1
        end
      end
      when RU_LEX then begin
        case token
          when EOL then begin
            text[paragraph_id][sentence_id][subsentence_id] << ' '
            token = c
          end
          else token << c
        end
      end
      when EN_LEX then begin
        case token
          when EOL then begin
            text[paragraph_id][sentence_id][subsentence_id] << ' '
            token = c
          end
          else token << c
        end
      end
      when DIG then begin
        case token
          when EOL then begin
            text[paragraph_id][sentence_id][subsentence_id] << ' '
            token = c
          end
          else token << c
        end
      end
      when DIL then begin
        case token
          when EOL then begin
            text[paragraph_id][sentence_id][subsentence_id] << token
            token = c
          end
          else token << c
        end<commit_msg>greeb-proto-parser: little code cleanup<commit_after>#!/usr/bin/env ruby
# encoding: utf-8

origin = <<-END
 - Сынок, чего это     от тебя   зигами пахнет,
опять на Манежную площадь ходил?

 - Нет мама, я в метро ехал, там  назиговано было!!

  

Четырнадцать, восемьдесять восемь: 14/88.
Вот так блять
END
origin.chomp!

RU_LEX = /^[А-Яа-я]+$/u
EN_LEX = /^[A-Za-z]+$/u
EOL = /^\n+$/u
SEP = /^[*=_\/\\ ]$/u
PUN = /^(\.|\!|\?)$/u
SPUN = /^(\,|\[|\]|\(|\)|\-|:|;)$/u
DIG = /^[0-9]+$/u
DIL = /^[А-Яа-яA-Za-z0-9]+$/u
EMPTY = ''

class MetaArray < Array
  def [] id
    super(id) or begin
      self.class.new.tap do |element|
        self[id] = element
      end
    end
  end
end

def parse(origin)
  text = MetaArray.new

  paragraph_id = 0
  sentence_id = 0
  subsentence_id = 0

  token = ''

  origin.each_char do |c|
    puts "[#{token.inspect}] ← #{c.inspect}"
    case c
      when EOL then begin
        case token
          when EMPTY then token << c
          when EOL then begin
            token = ''
            paragraph_id += 1
            sentence_id = 0
            subsentence_id = 0
          end
        else
          text[paragraph_id][sentence_id][subsentence_id] << token
          token = c
        end
      end
      when SEP then begin
        case token
          when EMPTY
        else
          text[paragraph_id][sentence_id][subsentence_id] << token
          while text[paragraph_id][sentence_id][subsentence_id].last == c
            text[paragraph_id][sentence_id][subsentence_id].pop
          end
          text[paragraph_id][sentence_id][subsentence_id] << c
          token = ''
        end
      end
      when PUN then begin
        case token
          when EMPTY
        else
          text[paragraph_id][sentence_id][subsentence_id] << token
          text[paragraph_id][sentence_id][subsentence_id] << c
          token = ''
          sentence_id += 1
          subsentence_id = 0
        end
      end
      when SPUN then begin
        case token
          when EMPTY
        else
          text[paragraph_id][sentence_id][subsentence_id] << token
          text[paragraph_id][sentence_id][subsentence_id] << c
          token = ''
          subsentence_id += 1
        end
      end
      when RU_LEX then begin
        case token
          when EOL then begin
            text[paragraph_id][sentence_id][subsentence_id] << ' '
            token = c
          end
        else
          token << c
        end
      end
      when EN_LEX then begin
        case token
          when EOL then begin
            text[paragraph_id][sentence_id][subsentence_id] << ' '
            token = c
          end
        else
          token << c
        end
      end
      when DIG then begin
        case token
          when EOL then begin
            text[paragraph_id][sentence_id][subsentence_id] << ' '
            token = c
          end
        else
          token << c
        end
      end
      when DIL then begin
        case token
          when EOL then begin
            text[paragraph_id][sentence_id][subsentence_id] << token
            token = c
          end
        else
          token << c
        end
{'fraction_non_alphanumeric': 0.10285261489698891, 'fraction_numerical': 0.005705229793977813, 'mean_word_length': 1.5427074939564867, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4514417', 'n_tokens_mistral': 2143, 'n_tokens_neox': 2023, 'n_words': 512}