<commit_before>require 'ndr_support/safe_file'
require_relative 'registry'

module NdrImport
  # This is one of a collection of file handlers that deal with individual formats of data.
  # They can be instantiated directly or via the factory method Registry.tables
  module File
    # This class is a text file handler that returns a single table.
    class Text < Base
      private

      def rows(&block)
        return enum_for(:rows) unless block

        ::File.new(@filename, 'rt').each do |line|
          block.call(line.sub(/\n\z/, ''))
        end

      rescue => e
        raise "Failed to read #{SafeFile.basename(@filename)} as text " \
              "[#{e.class}: #{e.message}]"
      end
    end

    Registry.register(Text, 'txt') # TODO: Add 'nontabular'?
  end
end<commit_msg># Security comment added with superfluous update<commit_after>require 'ndr_support/safe_file'
require_relative 'registry'

module NdrImport
  # This is one of a collection of file handlers that deal with individual formats of data.
  # They can be instantiated directly or via the factory method Registry.tables
  module File
    # This class is a text file handler that returns a single table.
    class Text < Base
      private

      def rows(&block)
        
        return enum_for(:rows) unless block

        # SECURE: TG 13 Oct 2015 SafeFile.safepath_to_string ensures that the path is SafePath.
        # TODO: SafeFile.new does not support the 'rt' switches
        ::File.new(SafeFile.safepath_to_string(@filename), 'rt').each do |line|
          block.call(line.sub(/\n\z/, ''))
        end

      rescue => e
        raise "Failed to read #{SafeFile.basename(@filename)} as text " \
              "[#{e.class}: #{e.message}]"
      end
    end

    Registry.register(Text, 'txt') # TODO: Add 'nontabular'?
  end
end
{'fraction_non_alphanumeric': 0.09698630136986301, 'fraction_numerical': 0.0032876712328767125, 'mean_word_length': 2.7113821138211383, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2822570', 'n_tokens_mistral': 543, 'n_tokens_neox': 516, 'n_words': 191}