<commit_before>################################################################################
# PostgreSQLCursor: library class provides postgresql cursor for large result
# set processing. Requires ActiveRecord, but can be adapted to other DBI/ORM libraries.
# If you don't use AR, this assumes #connection and #instantiate methods are available.
#
# options     - Hash to control operation and loop breaks
#   connection: instance  - ActiveRecord connection to use
#   fraction: 0.1..1.0    - The cursor_tuple_fraction (default 1.0)
#   block_size: 1..n      - The number of rows to fetch per db block fetch
#   while: value          - Exits loop when block does not return this value.
#   until: value          - Exits loop when block returns this value.
#   with_hold: boolean    - Allows the query to remain open across commit points.
#
# Exmaples:
#   PostgreSQLCursor::Cursor.new("select ...").each { |hash| ... }
#   ActiveRecordModel.where(...).each_row { |hash| ... }
#   ActiveRecordModel.each_row_by_sql("select ...") { |hash| ... }
#   ActiveRecordModel.each_instance_by_sql("select ...") { |model| ... }
#
module PostgreSQLCursor
  class Cursor
    include Enumerable
    attr_reader :sql, :options, :connection, :count, :result

    # Public: Start a new PostgreSQL cursor query
    # sql     - The SQL statement with interpolated values
    # options - hash of processing controls
    #   while: value    - Exits loop when block does not return this value.
    #   until: value    - Exits loop when block returns this value.
    #   fraction: 0.1..1.0    - The cursor_tuple_fraction (default 1.0)
    #   block_size: 1..n      - The number of rows to fetch per db block fetch
    #                           Defaults to 1000
    #   with_hold       - Allows the query to remain open across commit points.
    #
    # Examples
    #
    #   PostgreSQLCursor::Cursor.new("select ....")
    #
    # Returns the cursor object when called with new.
    def initialize(sql, options={})
      @sql        = sql
      @options    = options
      @connection = @options.fetch(:connection) { ::ActiveRecord::Base.connection }
      @count      = 0
      @iterate    = options[:instances] ? :each_instance : :each_row
    end

    # Specify the type to instantiate, or reset to return a Hash
    def iterate_type(type=nil)<commit_msg>Fix OID reference for unknown column type<commit_after>require 'active_record/connection_adapters/postgresql/oid'

################################################################################
# PostgreSQLCursor: library class provides postgresql cursor for large result
# set processing. Requires ActiveRecord, but can be adapted to other DBI/ORM libraries.
# If you don't use AR, this assumes #connection and #instantiate methods are available.
#
# options     - Hash to control operation and loop breaks
#   connection: instance  - ActiveRecord connection to use
#   fraction: 0.1..1.0    - The cursor_tuple_fraction (default 1.0)
#   block_size: 1..n      - The number of rows to fetch per db block fetch
#   while: value          - Exits loop when block does not return this value.
#   until: value          - Exits loop when block returns this value.
#   with_hold: boolean    - Allows the query to remain open across commit points.
#
# Exmaples:
#   PostgreSQLCursor::Cursor.new("select ...").each { |hash| ... }
#   ActiveRecordModel.where(...).each_row { |hash| ... }
#   ActiveRecordModel.each_row_by_sql("select ...") { |hash| ... }
#   ActiveRecordModel.each_instance_by_sql("select ...") { |model| ... }
#


if ::ActiveRecord::VERSION::MAJOR <= 4
  OID = ActiveRecord::ConnectionAdapters::PostgreSQLAdapter::OID
else
  OID = ActiveRecord::ConnectionAdapters::PostgreSQL::OID
end

module PostgreSQLCursor
  class Cursor
    include Enumerable
    attr_reader :sql, :options, :connection, :count, :result

    # Public: Start a new PostgreSQL cursor query
    # sql     - The SQL statement with interpolated values
    # options - hash of processing controls
    #   while: value    - Exits loop when block does not return this value.
    #   until: value    - Exits loop when block returns this value.
    #   fraction: 0.1..1.0    - The cursor_tuple_fraction (default 1.0)
    #   block_size: 1..n      - The number of rows to fetch per db block fetch
    #                           Defaults to 1000
    #   with_hold       - Allows the query to remain open across commit points.
    #
    # Examples
    #
    #   PostgreSQLCursor::Cursor.new("select ....")
    #
    # Returns the cursor object when called with new.
    def initialize(sql, options={})
      @sql        = sql
      @options    = options
      @connection = @options.fetch(:connection) { ::ActiveRecord::Base.connection }
      @count      = 0
      @iterate    = options[:instances] ? :each_instance : :each_row
    end

    # Specify the type to instantiate, or reset to return a Hash
    def iterate_type(type=nil)
{'fraction_non_alphanumeric': 0.12641585760517798, 'fraction_numerical': 0.007888349514563107, 'mean_word_length': 2.8333333333333335, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1662283', 'n_tokens_mistral': 1414, 'n_tokens_neox': 1345, 'n_words': 475}