<commit_before># coding: utf-8

module PDF
  class Reader
    module Formatted
      # Converts an object graph of text data into a UTF8 string
      class LayoutHelper
        attr_reader :lines, :verbosity, :options

        @tg = nil

        def initialize(options = {})
          @lines = []
          @options = options
          @verbosity = options.fetch(:verbosity, 0)
          @tg = PageLayout::TextGroup.new(@verbosity)
        end

        def lines
          @lines.clear
          puts "Attempting 1 more simplify before writing page" if @verbosity > 1
          @tg.simplify
          @lines.concat(@tg.lines)
        end

        def add_lines_from_text_group(text_group)
          text_group.sorted_lines.each do |line|
            @tg.lines << line
          end
        end

        def to_s
          def_rows = @options.fetch(:number_of_rows, 100)
          def_cols = @options.fetch(:number_of_cols, 200)
          row_multiplier = @options.fetch(:row_scale, 8.0) # 800
          col_multiplier = @options.fetch(:col_scale, 3.0) # 600
          page = []
          def_value = ""
          def_cols.times { def_value << " " }
          def_rows.times { page << String.new(def_value) }
          self.lines.each do |line|
            unless line.is_empty?
              x_pos = (line.position.x / col_multiplier).round
              y_pos = def_rows - (line.position.y / row_multiplier).round
              str = line.text
              if y_pos <= def_rows && y_pos >= 0 && x_pos <= def_cols && x_pos >= 0
                $stderr.puts "{%3d, %3d} -- %s" % [x_pos, y_pos, str.dump] if @verbosity > 2
                page[y_pos][Range.new(x_pos, x_pos + str.length - 1)] = String.new(str)
                $stderr.puts "Page[#{y_pos}] #{page[y_pos]}" if @verbosity > 2
              else
                $stderr.puts "Layout Skipping Line off of page:\n#{line}" if @verbosity > 0
              end
            else
              $stderr.puts "Layout Skipping Empty Line:\n#{line}" if @verbosity > 0
            end
          end
          final_string = ""
          strip_empty_lines = @options.fetch(:strip_empty_lines, true)
          page.each do |line|
            final_string << line.rstrip << "\n" unless line.strip.length == 0 && strip_empty_lines
          end
          final_string
        end

      end<commit_msg>small refactor to use Enumerable methods<commit_after># coding: utf-8

module PDF
  class Reader
    module Formatted
      # Converts an object graph of text data into a UTF8 string
      class LayoutHelper
        attr_reader :lines, :verbosity, :options

        @tg = nil

        def initialize(options = {})
          @lines = []
          @options = options
          @verbosity = options.fetch(:verbosity, 0)
          @tg = PageLayout::TextGroup.new(@verbosity)
        end

        def lines
          @lines.clear
          puts "Attempting 1 more simplify before writing page" if @verbosity > 1
          @tg.simplify
          @lines.concat(@tg.lines)
        end

        def add_lines_from_text_group(text_group)
          text_group.sorted_lines.each do |line|
            @tg.lines << line
          end
        end

        def to_s
          def_rows = @options.fetch(:number_of_rows, 100)
          def_cols = @options.fetch(:number_of_cols, 200)
          row_multiplier = @options.fetch(:row_scale, 8.0) # 800
          col_multiplier = @options.fetch(:col_scale, 3.0) # 600
          page = []
          def_value = ""
          def_cols.times { def_value << " " }
          def_rows.times { page << String.new(def_value) }
          self.lines.each do |line|
            unless line.is_empty?
              x_pos = (line.position.x / col_multiplier).round
              y_pos = def_rows - (line.position.y / row_multiplier).round
              str = line.text
              if y_pos <= def_rows && y_pos >= 0 && x_pos <= def_cols && x_pos >= 0
                $stderr.puts "{%3d, %3d} -- %s" % [x_pos, y_pos, str.dump] if @verbosity > 2
                page[y_pos][Range.new(x_pos, x_pos + str.length - 1)] = String.new(str)
                $stderr.puts "Page[#{y_pos}] #{page[y_pos]}" if @verbosity > 2
              else
                $stderr.puts "Layout Skipping Line off of page:\n#{line}" if @verbosity > 0
              end
            else
              $stderr.puts "Layout Skipping Empty Line:\n#{line}" if @verbosity > 0
            end
          end
          if @options.fetch(:strip_empty_lines, true)
            page.select! { |line| line.strip.length > 0 }
          end
          page.map(&:rstrip).join("\n")
        end

      end
{'fraction_non_alphanumeric': 0.10964532871972318, 'fraction_numerical': 0.013408304498269897, 'mean_word_length': 1.7142018779342723, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3820225', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1399, 'n_words': 343}