<commit_before>module MARC

  class Reader
    include Enumerable

    # The constructor which you may pass either a path
    #
    #   reader = MARC::Reader.new('marc.dat')
    #
    # or, if it's more convenient a File object:
    #
    #   fh = File.new('marc.dat')
    #   reader = MARC::Reader.new(fh)
    #
    # or really any object that responds to read(n)
    #
    #   # marc is a string with a bunch of records in it
    #   reader = MARC::Reader.new(StringIO.new(marc))
    #
    # If your data have non-standard control fields in them
    # (e.g., Aleph's 'FMT') you need to add them specifically
    # to the MARC::ControlField.control_tags Set object
    #
    #   MARC::ControlField.control_tags << 'FMT'
    #
    # Also, if your data encoded with non ascii/utf-8 encoding
    # (for ex. when reading RUSMARC data) and you use ruby 1.9
    # you can specify source data encoding with an option. 
    #
    #   reader = MARC::Reader.new('marc.dat', :external_encoding => 'cp866')
    #
    # or, you can pass IO, opened in the corresponding encoding
    #
    #   reader = MARC::Reader.new(File.new('marc.dat', 'r:cp866'))
    def initialize(file, options = {})      
      @encoding_options = {}
      # all can be nil
      [:internal_encoding, :external_encoding, :invalid, :replace, :validate_encoding].each do |key|
        @encoding_options[key] = options[key] if options.has_key?(key)
      end
            
      if file.is_a?(String)
        @handle = File.new(file)
      elsif file.respond_to?("read", 5)
        @handle = file
      else
        throw "must pass in path or file"
      end
      
      if (! @encoding_options[:external_encoding] ) && @handle.respond_to?(:external_encoding)
        # use file encoding only if we didn't already have an explicit one,
        # explicit one takes precedence. 
        #
        # Note, please don't use ruby's own internal_encoding transcode
        # with binary marc data, the transcode can mess up the byte count
        # and make it unreadable. 
        @encoding_options[:external_encoding] ||= @handle.external_encoding
      end      
    end

    # to support iteration:
    #   for record in reader
    #     print record
    #   end
    def each
      # while there is data left in the file
      while rec_length_s = @handle.read(5)
        # make sure the record length looks like an integer
        rec_length_i = rec_length_s.to_i
        if rec_length_i == 0
          raise MARC::Exception.new("invalid record length: #{rec_length_s}")
        end

        # get the raw MARC21 for a record back from the file
        # using the record length
        raw = rec_length_s + @handle.read(rec_length_i-5)


        # create a record from the data and return it
        #record = MARC::Record.new_from_marc(raw)
        record = MARC::Reader.decode(raw, @encoding_options)
        yield record
      end
    end


    # A static method for turning raw MARC data in transission
    # format into a MARC::Record object.
    # First argument is a String
    # options include:
    #   [:external_encoding]  encoding of MARC record data values
    #   [:forgiving]          needs more docs, true is some kind of forgiving 
    #                         of certain kinds of bad MARC. 
    def self.decode(marc, params={})
      if params.has_key?(:encoding)
        $stderr.puts "DEPRECATION WARNING: MARC::Reader.decode :encoding option deprecated, please use :external_encoding"
        params[:external_encoding] = params.delete(:encoding)
      end
      
      if (! params.has_key? :external_encoding ) && marc.respond_to?(:encoding)
        # If no forced external_encoding giving, respect the encoding
        # declared on the string passed in. 
        params[:external_encoding] = marc.encoding
      end
      
      record = Record.new()
      record.leader = marc[0..LEADER_LENGTH-1]

      # where the field data starts
      base_address = record.leader[12..16].to_i

      # get the byte offsets from the record directory
      directory = marc[LEADER_LENGTH..base_address-1]

      throw "invalid directory in record" if directory == nil

      # the number of fields in the record corresponds to
      # how many directory entries there are
      num_fields = directory.length / DIRECTORY_ENTRY_LENGTH

      # when operating in forgiving mode we just split on end of
      # field instead of using calculated byte offsets from the
      # directory
      if params[:forgiving]        
        marc_field_data = marc[base_address..-1]
        # It won't let us do the split on bad utf8 data, but
        # we haven't yet set the 'proper' encoding or used
        # our correction/replace options. So call it binary for now.
        marc_field_data.force_encoding("binary") if marc_field_data.respond_to?(:force_encoding)
        
        all_fields = marc_field_data.split(END_OF_FIELD)
      else
        mba =  marc.bytes.to_a
      end

      0.upto(num_fields-1) do |field_num|

        # pull the directory entry for a field out
        entry_start = field_num * DIRECTORY_ENTRY_LENGTH
        entry_end = entry_start + DIRECTORY_ENTRY_LENGTH
        entry = directory[entry_start..entry_end]

        # extract the tag
        tag = entry[0..2]

        # get the actual field data
        # if we were told to be forgiving we just use the
        # next available chuck of field data that we
        # split apart based on the END_OF_FIELD
        field_data = ''
        if params[:forgiving]
          field_data = all_fields.shift()

        # otherwise we actually use the byte offsets in
        # directory to figure out what field data to extract
        else
          length = entry[3..6].to_i
          offset = entry[7..11].to_i
          field_start = base_address + offset
          field_end = field_start + length - 1
          field_data = mba[field_start..field_end].pack("c*")
        end

        # remove end of field
        field_data.delete!(END_OF_FIELD)

        if field_data.respond_to?(:force_encoding)
          if params[:external_encoding]
            field_data = field_data.force_encoding(params[:external_encoding])
          end     
          
          # If we don't check this now to raise a InvalidByteSequenceError,
          # bad bytes will trigger ArgumentErrors from arbitrary part
          # of the ruby_marc stack _anyway_, we got to check now for
          # a predictable erorr message. 
          # pass on params for :replace and :invalid options. 
          if (params[:invalid] || params[:replace] || (params[:validate_encoding] == true))
            field_data = MARC::Reader.validate_encoding(field_data,  params)
          end
          
          if params[:internal_encoding]
            field_data = field_data.encode(params[:internal_encoding])
          end
          
          
        end
        # add a control field or data field
        if MARC::ControlField.control_tag?(tag)
          record.append(MARC::ControlField.new(tag,field_data))
        else
          field = MARC::DataField.new(tag)

          # get all subfields
          subfields = field_data.split(SUBFIELD_INDICATOR)

          # must have at least 2 elements (indicators, and 1 subfield)
          # TODO some sort of logging?
          next if subfields.length() < 2

          # get indicators
          indicators = subfields.shift()
          field.indicator1 = indicators[0,1]
          field.indicator2 = indicators[1,1]

          # add each subfield to the field
          subfields.each() do |data|
            subfield = MARC::Subfield.new(data[0,1],data[1..-1])
            field.append(subfield)
          end

          # add the field to the record
          record.append(field)
        end<commit_msg>Don't use our slow pure ruby encoding validator unless we are asked to, or have to<commit_after>module MARC

  class Reader
    include Enumerable

    # The constructor which you may pass either a path
    #
    #   reader = MARC::Reader.new('marc.dat')
    #
    # or, if it's more convenient a File object:
    #
    #   fh = File.new('marc.dat')
    #   reader = MARC::Reader.new(fh)
    #
    # or really any object that responds to read(n)
    #
    #   # marc is a string with a bunch of records in it
    #   reader = MARC::Reader.new(StringIO.new(marc))
    #
    # If your data have non-standard control fields in them
    # (e.g., Aleph's 'FMT') you need to add them specifically
    # to the MARC::ControlField.control_tags Set object
    #
    #   MARC::ControlField.control_tags << 'FMT'
    #
    # Also, if your data encoded with non ascii/utf-8 encoding
    # (for ex. when reading RUSMARC data) and you use ruby 1.9
    # you can specify source data encoding with an option. 
    #
    #   reader = MARC::Reader.new('marc.dat', :external_encoding => 'cp866')
    #
    # or, you can pass IO, opened in the corresponding encoding
    #
    #   reader = MARC::Reader.new(File.new('marc.dat', 'r:cp866'))
    def initialize(file, options = {})      
      @encoding_options = {}
      # all can be nil
      [:internal_encoding, :external_encoding, :invalid, :replace, :validate_encoding].each do |key|
        @encoding_options[key] = options[key] if options.has_key?(key)
      end
            
      if file.is_a?(String)
        @handle = File.new(file)
      elsif file.respond_to?("read", 5)
        @handle = file
      else
        throw "must pass in path or file"
      end
      
      if (! @encoding_options[:external_encoding] ) && @handle.respond_to?(:external_encoding)
        # use file encoding only if we didn't already have an explicit one,
        # explicit one takes precedence. 
        #
        # Note, please don't use ruby's own internal_encoding transcode
        # with binary marc data, the transcode can mess up the byte count
        # and make it unreadable. 
        @encoding_options[:external_encoding] ||= @handle.external_encoding
      end      
    end

    # to support iteration:
    #   for record in reader
    #     print record
    #   end
    def each
      # while there is data left in the file
      while rec_length_s = @handle.read(5)
        # make sure the record length looks like an integer
        rec_length_i = rec_length_s.to_i
        if rec_length_i == 0
          raise MARC::Exception.new("invalid record length: #{rec_length_s}")
        end

        # get the raw MARC21 for a record back from the file
        # using the record length
        raw = rec_length_s + @handle.read(rec_length_i-5)


        # create a record from the data and return it
        #record = MARC::Record.new_from_marc(raw)
        record = MARC::Reader.decode(raw, @encoding_options)
        yield record
      end
    end


    # A static method for turning raw MARC data in transission
    # format into a MARC::Record object.
    # First argument is a String
    # options include:
    #   [:external_encoding]  encoding of MARC record data values
    #   [:forgiving]          needs more docs, true is some kind of forgiving 
    #                         of certain kinds of bad MARC. 
    def self.decode(marc, params={})
      if params.has_key?(:encoding)
        $stderr.puts "DEPRECATION WARNING: MARC::Reader.decode :encoding option deprecated, please use :external_encoding"
        params[:external_encoding] = params.delete(:encoding)
      end
      
      if (! params.has_key? :external_encoding ) && marc.respond_to?(:encoding)
        # If no forced external_encoding giving, respect the encoding
        # declared on the string passed in. 
        params[:external_encoding] = marc.encoding
      end
      
      record = Record.new()
      record.leader = marc[0..LEADER_LENGTH-1]

      # where the field data starts
      base_address = record.leader[12..16].to_i

      # get the byte offsets from the record directory
      directory = marc[LEADER_LENGTH..base_address-1]

      throw "invalid directory in record" if directory == nil

      # the number of fields in the record corresponds to
      # how many directory entries there are
      num_fields = directory.length / DIRECTORY_ENTRY_LENGTH

      # when operating in forgiving mode we just split on end of
      # field instead of using calculated byte offsets from the
      # directory
      if params[:forgiving]        
        marc_field_data = marc[base_address..-1]
        # It won't let us do the split on bad utf8 data, but
        # we haven't yet set the 'proper' encoding or used
        # our correction/replace options. So call it binary for now.
        marc_field_data.force_encoding("binary") if marc_field_data.respond_to?(:force_encoding)
        
        all_fields = marc_field_data.split(END_OF_FIELD)
      else
        mba =  marc.bytes.to_a
      end

      0.upto(num_fields-1) do |field_num|

        # pull the directory entry for a field out
        entry_start = field_num * DIRECTORY_ENTRY_LENGTH
        entry_end = entry_start + DIRECTORY_ENTRY_LENGTH
        entry = directory[entry_start..entry_end]

        # extract the tag
        tag = entry[0..2]

        # get the actual field data
        # if we were told to be forgiving we just use the
        # next available chuck of field data that we
        # split apart based on the END_OF_FIELD
        field_data = ''
        if params[:forgiving]
          field_data = all_fields.shift()

        # otherwise we actually use the byte offsets in
        # directory to figure out what field data to extract
        else
          length = entry[3..6].to_i
          offset = entry[7..11].to_i
          field_start = base_address + offset
          field_end = field_start + length - 1
          field_data = mba[field_start..field_end].pack("c*")
        end

        # remove end of field
        field_data.delete!(END_OF_FIELD)

        if field_data.respond_to?(:force_encoding)
          if params[:external_encoding]
            field_data = field_data.force_encoding(params[:external_encoding])
          end     
          
          # If we're transcoding anyway, pass our invalid/replace options
          # on to String#encode, which will take care of them -- or raise
          # with illegal bytes without :replace=>:invalid. 
          #
          # If we're NOT transcoding, we need to use our own pure-ruby
          # implementation to do invalid byte replacements. OR to raise
          # a predicatable exception iff :validate_encoding, otherwise
          # for performance we won't check, and you may or may not
          # get an exception from inside ruby-marc, and it may change
          # in future implementations. 
          if params[:internal_encoding]
            field_data = field_data.encode(params[:internal_encoding], params)
          elsif (params[:invalid] || params[:replace] || (params[:validate_encoding] == true))
            field_data = MARC::Reader.validate_encoding(field_data,  params)
          end
          
        end
        # add a control field or data field
        if MARC::ControlField.control_tag?(tag)
          record.append(MARC::ControlField.new(tag,field_data))
        else
          field = MARC::DataField.new(tag)

          # get all subfields
          subfields = field_data.split(SUBFIELD_INDICATOR)

          # must have at least 2 elements (indicators, and 1 subfield)
          # TODO some sort of logging?
          next if subfields.length() < 2

          # get indicators
          indicators = subfields.shift()
          field.indicator1 = indicators[0,1]
          field.indicator2 = indicators[1,1]

          # add each subfield to the field
          subfields.each() do |data|
            subfield = MARC::Subfield.new(data[0,1],data[1..-1])
            field.append(subfield)
          end

          # add the field to the record
          record.append(field)
        end
{'fraction_non_alphanumeric': 0.08153235460903535, 'fraction_numerical': 0.00592275218952807, 'mean_word_length': 2.203229061553986, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '306053', 'n_tokens_mistral': 4631, 'n_tokens_neox': 4418, 'n_words': 1457}