<commit_before># frozen_string_literal: true

module Textbringer
  module Commands
    define_command(:fill_region,
                   doc: "Fill paragraph.") do
      |s = buffer.point, e = buffer.mark|
      s, e = Buffer.region_boundaries(s, e)
      buffer = Buffer.current
      buffer.save_excursion do
        str = buffer.substring(s, e)
        buffer.goto_char(s)
        pos = buffer.point
        buffer.beginning_of_line
        column = Buffer.display_width(buffer.substring(buffer.point, pos))
        input = StringIO.new(str)
        output = String.new
        fill_column = CONFIG[:fill_column]
        while c = input.getc
          if column < fill_column && c == "\n"
            next
          end
          w = Buffer.display_width(c)
          if c == "\n"
            output << c
            column = 0
          else
            if column + w > fill_column || column >= fill_column
              output << "\n"
              column = 0
            end
            output << c
            column += w
          end<commit_msg>Fill English text correctly.<commit_after># frozen_string_literal: true

module Textbringer
  module Commands
    define_command(:fill_region,
                   doc: "Fill paragraph.") do
      |s = buffer.point, e = buffer.mark|
      s, e = Buffer.region_boundaries(s, e)
      buffer = Buffer.current
      buffer.save_excursion do
        str = buffer.substring(s, e)
        buffer.goto_char(s)
        pos = buffer.point
        buffer.beginning_of_line
        column = Buffer.display_width(buffer.substring(buffer.point, pos))
        input = StringIO.new(str)
        output = String.new
        fill_column = CONFIG[:fill_column]
        prev_c = nil
        while c = input.getc
          if column < fill_column && c == "\n"
            if /\w/ =~ prev_c
              next_c = input.getc
              input.ungetc(next_c)
              if /\w/ =~ next_c
                output << " "
                column += 1
              end
            end
            next
          end
          if c == "\n"
            output << c
            column = 0
          else
            w = Buffer.display_width(c)
            if column + w > fill_column || column >= fill_column
              if /\w/ =~ prev_c && /\w/ =~ c
                if output.sub!(/([^\w\n])(\w*)\z/, "\\1\n")
                  output << $2
                  column = Buffer.display_width($2)
                end
              else
                output << "\n"
                column = 0
              end
            end
            output << c
            column += w
          end
          prev_c = c
{'fraction_non_alphanumeric': 0.09943074003795066, 'fraction_numerical': 0.0030360531309297912, 'mean_word_length': 1.2862098872506504, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5503607', 'n_tokens_mistral': 751, 'n_tokens_neox': 728, 'n_words': 187}