<commit_before>module MARC  
  
  # Simply what the class name says.
  # The checksum is used to see if the FieldMap's array has changed.
  class HashWithChecksumAttribute < Hash
    attr_accessor :checksum
  end
  
  # The FieldMap is an Array of DataFields and Controlfields.
  # It also contains a HashWithChecksumAttribute with a Hash-based
  # representation of the fields for faster lookups
  class FieldMap < Array
    attr_reader :tags
    def initialize
      @tags = HashWithChecksumAttribute.new
    end
    
    # Checks to see if the HashWithChecksumAttribute is in sync
    # with the Array of fields
    def in_sync?
      @tags.checksum == self.hash
    end
    
    # Rebuild the HashWithChecksumAttribute with the current
    # values of the fields Array
    def reindex
      @tags = HashWithChecksumAttribute.new
      self.each_with_index do |field, i|
        @tags[field.tag] ||= []
        @tags[field.tag] << i
      end
      @tags.checksum = self.hash
    end
    
    # Returns an array of all of the tags that appear in the record (not in the order they appear, however).
    def tag_list
      reindex if @tags.empty?
      @tags.keys
    end
    
    # Returns an array of fields, in the order they appear, according to their tag.
    # The tags argument can be a string (e.g. '245'), an array (['100','700','800'])
    # or a range (('600'..'699')).
    def each_by_tag(tags)
      reindex if @tags.empty?
      indices = @tags.values_at(*(@tags.keys & [*tags])).flatten.sort
      return [] if indices.empty?
      self.values_at(*indices).each do |tag|
        yield tag
      end
    end
  end

  # A class that represents an individual MARC record. Every record
  # is made up of a collection of MARC::DataField objects. 
  #
  # MARC::Record mixes in Enumerable to enable access to constituent
  # DataFields. For example, to return a list of all subject DataFields:
  #
  #   record.find_all {|field| field.tag =~ /^6../}  
  # 
  # The accessor 'fields' is also an Array of MARC::DataField objects which
  # the client can access or modifyi if neccesary.
  #
  #   record.fields.delete(field)
  # 
  # Other accessor attribute: 'leader' for record leader as String
 
  class Record
    include Enumerable

    # the record fields
    #attr_reader :fields

    # the record leader
    attr_accessor :leader

    def initialize
      @fields = FieldMap.new
      # leader is 24 bytes
      @leader = ' ' * 24
      # leader defaults:
      # http://www.loc.gov/marc/bibliographic/ecbdldrd.html
      @leader[10..11] = '22'
      @leader[20..23] = '4500'
    end

    # add a field to the record
    #   record.append(MARC::DataField.new( '100', '2', '0', ['a', 'Fred']))

    def append(field)
      @fields.push(field)
    end

    # alias to append
    
    def <<(field)
      append(field)
    end

    # each() is here to support iterating and searching since MARC::Record
    # mixes in Enumerable
    #
    # iterating through the fields in a record:
    #   record.each { |f| print f }
    #
    # getting the 245
    #   title = record.find {|f| f.tag == '245'}
    #
    # getting all subjects
    #   subjects = record.find_all {|f| ('600'..'699') === f.tag}

    def each
      for field in @fields
        yield field
      end
    end
    
    # A more convenient way to iterate over each field with a given tag.  
    # The filter argument can be a string, array or range.
    def each_by_tag(filter)
      @fields.each_by_tag(filter) {|tag| yield tag }
    end

    # You can lookup fields using this shorthand:
    #   title = record['245']

    def [](tag)
      return self.find {|f| f.tag == tag}
    end
    
    # Provides a backwards compatible means to access the FieldMap.
    # No argument returns the FieldMap array in entirety.  Providing
    # a string, array or range of tags will return an array of fields
    # in the order they appear in the record.
    def fields(filter=nil)
      return @fields.to_a unless filter<commit_msg>Return the fieldmap array, not a copy of it<commit_after>module MARC  
  
  # Simply what the class name says.
  # The checksum is used to see if the FieldMap's array has changed.
  class HashWithChecksumAttribute < Hash
    attr_accessor :checksum
  end
  
  # The FieldMap is an Array of DataFields and Controlfields.
  # It also contains a HashWithChecksumAttribute with a Hash-based
  # representation of the fields for faster lookups
  class FieldMap < Array
    attr_reader :tags
    def initialize
      @tags = HashWithChecksumAttribute.new
    end
    
    # Checks to see if the HashWithChecksumAttribute is in sync
    # with the Array of fields
    def in_sync?
      @tags.checksum == self.hash
    end
    
    # Rebuild the HashWithChecksumAttribute with the current
    # values of the fields Array
    def reindex
      @tags = HashWithChecksumAttribute.new
      self.each_with_index do |field, i|
        @tags[field.tag] ||= []
        @tags[field.tag] << i
      end
      @tags.checksum = self.hash
    end
    
    # Returns an array of all of the tags that appear in the record (not in the order they appear, however).
    def tag_list
      reindex if @tags.empty?
      @tags.keys
    end
    
    # Returns an array of fields, in the order they appear, according to their tag.
    # The tags argument can be a string (e.g. '245'), an array (['100','700','800'])
    # or a range (('600'..'699')).
    def each_by_tag(tags)
      reindex if @tags.empty?
      indices = @tags.values_at(*(@tags.keys & [*tags])).flatten.sort
      return [] if indices.empty?
      self.values_at(*indices).each do |tag|
        yield tag
      end
    end
  end

  # A class that represents an individual MARC record. Every record
  # is made up of a collection of MARC::DataField objects. 
  #
  # MARC::Record mixes in Enumerable to enable access to constituent
  # DataFields. For example, to return a list of all subject DataFields:
  #
  #   record.find_all {|field| field.tag =~ /^6../}  
  # 
  # The accessor 'fields' is also an Array of MARC::DataField objects which
  # the client can access or modifyi if neccesary.
  #
  #   record.fields.delete(field)
  # 
  # Other accessor attribute: 'leader' for record leader as String
 
  class Record
    include Enumerable

    # the record fields
    #attr_reader :fields

    # the record leader
    attr_accessor :leader

    def initialize
      @fields = FieldMap.new
      # leader is 24 bytes
      @leader = ' ' * 24
      # leader defaults:
      # http://www.loc.gov/marc/bibliographic/ecbdldrd.html
      @leader[10..11] = '22'
      @leader[20..23] = '4500'
    end

    # add a field to the record
    #   record.append(MARC::DataField.new( '100', '2', '0', ['a', 'Fred']))

    def append(field)
      @fields.push(field)
    end

    # alias to append
    
    def <<(field)
      append(field)
    end

    # each() is here to support iterating and searching since MARC::Record
    # mixes in Enumerable
    #
    # iterating through the fields in a record:
    #   record.each { |f| print f }
    #
    # getting the 245
    #   title = record.find {|f| f.tag == '245'}
    #
    # getting all subjects
    #   subjects = record.find_all {|f| ('600'..'699') === f.tag}

    def each
      for field in @fields
        yield field
      end
    end
    
    # A more convenient way to iterate over each field with a given tag.  
    # The filter argument can be a string, array or range.
    def each_by_tag(filter)
      @fields.each_by_tag(filter) {|tag| yield tag }
    end

    # You can lookup fields using this shorthand:
    #   title = record['245']

    def [](tag)
      return self.find {|f| f.tag == tag}
    end
    
    # Provides a backwards compatible means to access the FieldMap.
    # No argument returns the FieldMap array in entirety.  Providing
    # a string, array or range of tags will return an array of fields
    # in the order they appear in the record.
    def fields(filter=nil)
      return @fields unless filter
{'fraction_non_alphanumeric': 0.09026305946889415, 'fraction_numerical': 0.014212691684328637, 'mean_word_length': 2.428205128205128, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4357517', 'n_tokens_mistral': 2571, 'n_tokens_neox': 2436, 'n_words': 907}