<commit_before>require 'external'

module Ms
  module Mascot
    module Dat
      
      # A hash of (section_name, SectionClass) pairs mapping section names
      # to section class.  Initially SectionClass may be a require path; if
      # so the path is required and the class looked up like:
      #
      #   Ms::Mascot::Dat.const_get(section_name.capitalize)
      #
      # Such that 'header' is mapped to Ms::Mascot::Dat::Header.
      CONTENT_TYPE_CLASSES = {}
      
      # currently unimplemented: unimod enzyme taxonomy mixture quantitation
      %w{header index masses parameters peptides proteins summary query
      }.each do |section_name|
        CONTENT_TYPE_CLASSES[section_name] = "ms/mascot/dat/#{section_name}"
      end 
      
      # Provides access to a Mascot dat file.
      class Archive < ExternalArchive
        module Utils
          module_function
          
          # Parses a hash of metadata (content_type, boundary, etc) from io.
          # parse_metadata does not reposition io.
          def parse_metadata(io)
            current_pos = io.pos
            io.rewind
            
            metadata = {}
            line = io.readline
            unless line =~ /MIME-Version: (\d+\.\d+) \(Generated by Mascot version (\d+\.\d+)\)/
              raise "could not parse mime-version or mascot-version: #{line}"
            end
            metadata[:mime_version] = $1
            metadata[:mascot_version] = $2
            
            line = io.readline
            unless line =~ /Content-Type: (.*?); boundary=(.*)/
              raise "could not parse content-type: #{line}"
            end
            metadata[:content_type] = $1
            metadata[:boundary] = $2
            
            io.pos = current_pos
            metadata
          end
          
          # Parses a mascot-style content type declaration.  This method uses
          # a simple regexp and is very brittle, but it works for all known
          # dat files.
          def parse_content_type(str)
            unless str =~ /^Content-Type: (.*?); name=\"(.*)\"/
              raise "unparseable content-type declaration: #{str.inspect}"
            end
            
            {:content_type => $1, :section_name => $2}
          end
          
          # Resolves a content type class from a hash of metadata like:
          #
          #   metadata = {
          #     :content_type => 'application/x-Mascot',
          #     :section_name => 'header'
          #   }
          #   Dat.content_type_class(metadata)   # => Ms::Mascot::Dat::Header
          #
          # Raises an error if the content type is not 'application/x-Mascot'
          # or if the name is not registered in CONTENT_TYPE_CLASSES.
          def content_type_class(metadata)
            unless metadata[:content_type] == 'application/x-Mascot'
              raise "unknown content_type: #{metadata.inspect}"
            end
            
            name = metadata[:section_name]
            name = 'query' if name =~ /^query(\d+)$/
            case const = CONTENT_TYPE_CLASSES[name]
            when String
              require const
              CONTENT_TYPE_CLASSES[name] = Dat.const_get(name.capitalize)
            else
              const
            end
          end
        end
        
        include Utils
        
        # A hash of metadata associated with this dat file.
        attr_reader :metadata
        
        def initialize(io=nil, io_index=nil)
          super(io)
          @metadata = parse_metadata(io)
          @section_names = []
        end
        
        # The boundary separating sections, typically '--gc0p4Jq0M2Yt08jU534c0p'.
        def boundary
          "--#{metadata[:boundary]}"
        end
        
        # Reindexes self.
        def reindex(&block)
          @section_names.clear
          reindex_by_sep(boundary, 
            :entry_follows_sep => true, 
            :exclude_sep => true,
          &block)

          # remove the first and last entries, which contain
          # the metadata and indicate the end of the multipart 
          # form data.
          io_index.shift
          io_index.pop

          self
        end

        # Converts str into an entry according to the content type header
        # which should be present at the start of the string.
        def str_to_entry(str)
          if ctc = content_type_class(parse_content_type(str))
            ctc.parse(str, self)
          else
            str
          end
        end
        
        # The section names corresponding to each entry in self.
        #
        # Normally section names are lazily parsed from the Content-Type header
        # of an entry as needed.  If resolve is true, all section names are
        # parsed and then returned; otherwise section_names may return a
        # partially-filled array.
        def section_names(resolve=true)
          resolve_sections if resolve
          @section_names
        end
        
        # Returns the entry for the named section.
        def section(name)
          self[section_index(name)]
        end

        # Returns the index of the named section.
        def section_index(name)
          0.upto(length - 1) do |index|
            return index if section_name(index) == name
          end
          nil
        end

        # Returns the section name for the entry at index.
        def section_name(index)
          # all sections must be resolved for negative indicies to
          # work correctly (since otherwise @section_names may not
          # have the same length as self)
          resolve_sections if index < 0
          @section_names[index] ||= parse_section_name(index)
        end
        
        # Returns the number of queries registered in self.
        def nqueries
          @nqueries ||= section_names.select {|name| name =~ /query/ }.length
        end
        
        # Yields each query to the block.
        def each_query
          1.upto(nqueries) do |n|
            yield(query(n))
          end
        end
        
        # Returns the specified query. 
        def query(num)
          if si = section_index("query#{num}")
            self[si]
          else
            nil
          end
        end

        private
        
        # resolves each section
        def resolve_sections # :nodoc:
          (@section_names.length).upto(length - 1) do |index| 
            section_name(index)
          end
        end
        
        # helper to go to the entry at index and parse the section name
        def parse_section_name(index) # :nodoc:
          return nil unless index = io_index[index]
          io.pos = index[0] + 1
          parse_content_type(io.readline)[:section_name]
        end
      end
    end
  end
end
<commit_msg>increased default block size for dat archive<commit_after>require 'external'

module Ms
  module Mascot
    module Dat
      
      # A hash of (section_name, SectionClass) pairs mapping section names
      # to section class.  Initially SectionClass may be a require path; if
      # so the path is required and the class looked up like:
      #
      #   Ms::Mascot::Dat.const_get(section_name.capitalize)
      #
      # Such that 'header' is mapped to Ms::Mascot::Dat::Header.
      CONTENT_TYPE_CLASSES = {}
      
      # currently unimplemented: unimod enzyme taxonomy mixture quantitation
      %w{header index masses parameters peptides proteins summary query
      }.each do |section_name|
        CONTENT_TYPE_CLASSES[section_name] = "ms/mascot/dat/#{section_name}"
      end 
      
      # Provides access to a Mascot dat file.
      class Archive < ExternalArchive
        module Utils
          module_function
          
          # Parses a hash of metadata (content_type, boundary, etc) from io.
          # parse_metadata does not reposition io.
          def parse_metadata(io)
            current_pos = io.pos
            io.rewind
            
            metadata = {}
            line = io.readline
            unless line =~ /MIME-Version: (\d+\.\d+) \(Generated by Mascot version (\d+\.\d+)\)/
              raise "could not parse mime-version or mascot-version: #{line}"
            end
            metadata[:mime_version] = $1
            metadata[:mascot_version] = $2
            
            line = io.readline
            unless line =~ /Content-Type: (.*?); boundary=(.*)/
              raise "could not parse content-type: #{line}"
            end
            metadata[:content_type] = $1
            metadata[:boundary] = $2
            
            io.pos = current_pos
            metadata
          end
          
          # Parses a mascot-style content type declaration.  This method uses
          # a simple regexp and is very brittle, but it works for all known
          # dat files.
          def parse_content_type(str)
            unless str =~ /^Content-Type: (.*?); name=\"(.*)\"/
              raise "unparseable content-type declaration: #{str.inspect}"
            end
            
            {:content_type => $1, :section_name => $2}
          end
          
          # Resolves a content type class from a hash of metadata like:
          #
          #   metadata = {
          #     :content_type => 'application/x-Mascot',
          #     :section_name => 'header'
          #   }
          #   Dat.content_type_class(metadata)   # => Ms::Mascot::Dat::Header
          #
          # Raises an error if the content type is not 'application/x-Mascot'
          # or if the name is not registered in CONTENT_TYPE_CLASSES.
          def content_type_class(metadata)
            unless metadata[:content_type] == 'application/x-Mascot'
              raise "unknown content_type: #{metadata.inspect}"
            end
            
            name = metadata[:section_name]
            name = 'query' if name =~ /^query(\d+)$/
            case const = CONTENT_TYPE_CLASSES[name]
            when String
              require const
              CONTENT_TYPE_CLASSES[name] = Dat.const_get(name.capitalize)
            else
              const
            end
          end
        end
        
        include Utils
        
        # A hash of metadata associated with this dat file.
        attr_reader :metadata
        
        def initialize(io=nil, io_index=nil)
          super(io)
          @metadata = parse_metadata(io)
          @section_names = []
        end
        
        # The boundary separating sections, typically '--gc0p4Jq0M2Yt08jU534c0p'.
        def boundary
          "--#{metadata[:boundary]}"
        end
        
        # Reindexes self.
        def reindex(&block)
          @section_names.clear
          reindex_by_sep(boundary, 
            :entry_follows_sep => true, 
            :exclude_sep => true,
            # :blksize => 8388608,  # default in ExternalArchive
            :blksize => 16777216,   # doubled the blksize
          &block)

          # remove the first and last entries, which contain
          # the metadata and indicate the end of the multipart 
          # form data.
          io_index.shift
          io_index.pop

          self
        end

        # Converts str into an entry according to the content type header
        # which should be present at the start of the string.
        def str_to_entry(str)
          if ctc = content_type_class(parse_content_type(str))
            ctc.parse(str, self)
          else
            str
          end
        end
        
        # The section names corresponding to each entry in self.
        #
        # Normally section names are lazily parsed from the Content-Type header
        # of an entry as needed.  If resolve is true, all section names are
        # parsed and then returned; otherwise section_names may return a
        # partially-filled array.
        def section_names(resolve=true)
          resolve_sections if resolve
          @section_names
        end
        
        # Returns the entry for the named section.
        def section(name)
          self[section_index(name)]
        end

        # Returns the index of the named section.
        def section_index(name)
          0.upto(length - 1) do |index|
            return index if section_name(index) == name
          end
          nil
        end

        # Returns the section name for the entry at index.
        def section_name(index)
          # all sections must be resolved for negative indicies to
          # work correctly (since otherwise @section_names may not
          # have the same length as self)
          resolve_sections if index < 0
          @section_names[index] ||= parse_section_name(index)
        end
        
        # Returns the number of queries registered in self.
        def nqueries
          @nqueries ||= section_names.select {|name| name =~ /query/ }.length
        end
        
        # Yields each query to the block.
        def each_query
          1.upto(nqueries) do |n|
            yield(query(n))
          end
        end
        
        # Returns the specified query. 
        def query(num)
          if si = section_index("query#{num}")
            self[si]
          else
            nil
          end
        end

        private
        
        # resolves each section
        def resolve_sections # :nodoc:
          (@section_names.length).upto(length - 1) do |index| 
            section_name(index)
          end
        end
        
        # helper to go to the entry at index and parse the section name
        def parse_section_name(index) # :nodoc:
          return nil unless index = io_index[index]
          io.pos = index[0] + 1
          parse_content_type(io.readline)[:section_name]
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.07496721550342415, 'fraction_numerical': 0.0044441206469473995, 'mean_word_length': 1.5614853517447285, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7363551', 'n_tokens_mistral': 3731, 'n_tokens_neox': 3606, 'n_words': 1146}