<commit_before>module RDF
  ##
  # **`RDF::All`** Attempts to read and parse all formats.
  #
  # Documentation
  # -------------
  #
  # * {RDF::All::Format}
  # * {RDF::All::Reader}
  # * {RDF::NTriples::Writer}
  #
  # @example Requiring the `RDF::All` module explicitly
  #   require 'rdf/all'
  module All
    ##
    # Only matched with :format => :all
    class Format < RDF::Format
      reader { RDF::All::Reader }
    end
    
    ##
    # Generic reader, detects appropriate readers and passes to each one
    class Reader < RDF::Reader
      ##
      # Finds each appropriate reader and yields statements
      # from each reader found.
      #
      # Take a sample from the input and pass to each Reader which implements
      # a .detect class method
      #
      # @overload each_statement
      #   @yield  [statement]
      #     each statement
      #   @yieldparam  [RDF::Statement] statement
      #   @yieldreturn [void] ignored
      #   @return [void]
      #
      # @overload each_statement
      #   @return [Enumerator]
      #
      # @return [void]
      # @see    RDF::Enumerable#each_statement
      def each_statement(&block)
        if block_given?
          @input.rewind
          sample = @input.read(1000)
          @input.rewind

          RDF::Reader.each do |reader_class|
            if reader_class.detect(sample)
              #puts "detected #{reader_class.name}"
              begin
                reader_class.new(@input, @options) do |reader|
                  reader.each_statement do |statement|
                    block.call(statement)
                  end
                end
              rescue RDF::ReaderError
                # Ignore errors
              end
            end
          end
        end
        enum_for(:each_statement)<commit_msg>Return statement counts from each reader used in RDF::All::Reader.<commit_after>module RDF
  ##
  # **`RDF::All`** Attempts to read and parse all formats.
  #
  # Documentation
  # -------------
  #
  # * {RDF::All::Format}
  # * {RDF::All::Reader}
  # * {RDF::NTriples::Writer}
  #
  # @example Requiring the `RDF::All` module explicitly
  #   require 'rdf/all'
  module All
    ##
    # Only matched with :format => :all
    class Format < RDF::Format
      reader { RDF::All::Reader }
    end
    
    ##
    # Generic reader, detects appropriate readers and passes to each one
    class Reader < RDF::Reader
      ##
      # Returns a hash of the number of statements parsed by each reader.
      #
      # @attr [Hash<Class, Integer>] statement_count
      attr :statement_count

      ##
      # Finds each appropriate reader and yields statements
      # from each reader found.
      #
      # Take a sample from the input and pass to each Reader which implements
      # a .detect class method
      #
      # @overload each_statement
      #   @yield  [statement]
      #     each statement
      #   @yieldparam  [RDF::Statement] statement
      #   @yieldreturn [void] ignored
      #   @return [void]
      #
      # @overload each_statement
      #   @return [Enumerator]
      #
      # @return [void]
      # @see    RDF::Enumerable#each_statement
      def each_statement(&block)
        if block_given?
          @input.rewind
          sample = @input.read(1000)
          @input.rewind

          @statement_count = {}
          
          RDF::Reader.each do |reader_class|
            if reader_class.detect(sample)
              #puts "detected #{reader_class.name}"
              begin
                reader_class.new(@input, @options) do |reader|
                  reader.each_statement do |statement|
                    @statement_count[reader_class] ||= 0
                    @statement_count[reader_class] += 1
                    block.call(statement)
                  end
                end
              rescue RDF::ReaderError
                # Ignore errors
              end
            end
          end
        end
        enum_for(:each_statement)
{'fraction_non_alphanumeric': 0.1014021031547321, 'fraction_numerical': 0.0025037556334501754, 'mean_word_length': 1.7103120759837178, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '55446', 'n_tokens_mistral': 1151, 'n_tokens_neox': 1113, 'n_words': 298}