<commit_before>require 'yaml'
require 'rexml/document'

module SymbolizeKeys

  # converts any current string keys to symbol keys 
  def self.extended(hash)
    hash.each do |key,value|
      if key.is_a?(String)
        hash.delete key
        hash[key] = value #through overridden []= 
      end
      if value.is_a?(Hash)
        hash[key]=value.extend(SymbolizeKeys)
      elsif value.is_a?(Array)
        value.each do |val|
          if val.is_a?(Hash)
            val.extend(SymbolizeKeys)
          end
        end
      end
    end
  end

  # assigns a new key/value pair 
  # converts they key to a symbol if it is a string 
  def []=(*args)
    args[0] = args[0].to_sym if args[0].is_a?(String)
    super
  end

  # returns new hash which is the merge of self and other hashes 
  # the returned hash will also be extended by SymbolizeKeys 
  def merge(*other_hashes , &resolution_proc )
    merged = Hash.new.extend SymbolizeKeys
    merged.merge! self , *other_hashes , &resolution_proc
  end

  # merges the other hashes into self 
  # if a proc is submitted , it's return will be the value for the key 
  def merge!( *other_hashes , &resolution_proc )

    # default resolution: value of the other hash 
    resolution_proc ||= proc{ |key,oldval,newval| newval }

    # merge each hash into self 
    other_hashes.each do |hash|
      hash.each{ |k,v|
        # assign new k/v into self, resolving conflicts with resolution_proc 
        self[k] = self.has_key?(k) ? resolution_proc[k,self[k],v] : v
      }
    end

    self
  end
end

class Metadata
  attr_accessor :source
  REQUIRED_FIELDS = [:name, :version, :maintainer]

  # Cleans up a string to make it suitable for use in a filename
  def self.clean_for_filename(dirtystring)
    dirtystring.downcase.gsub(/[^\w]/, '')
  end

  def self.get_pkgs_metadata_from_yml_doc(yml_doc, metadata=nil, source=nil)
    metadata = {} if metadata.nil?
    metadata_lists = yml_doc.split("---")
    metadata_lists.each do | metadata_text |
      if metadata_text =~ /^:?name:(.+)/
        name = $1.strip
        metadata[name] = [] if !metadata[name]
        metadata[name] << Metadata.new(metadata_text,'yml', source)
      end
    end
    return metadata
  end

  # metadata_text = text representation of the metadata
  # format = yml, xml, json, etc.
  def initialize(metadata_text, format, source=nil)
    @hash = nil
    @metadata_text = metadata_text
    @format = format
    @source = source
  end

  def [](key)
    return to_hash[key]
  end

  def []=(key,value)
    to_hash[key]=value
  end

  def to_hash
    if @hash  
      return @hash 
    end

    if @format == 'yml'
      hash = YAML::load(@metadata_text)
      @hash = hash.extend(SymbolizeKeys)
    else
      @hash = metadata_xml_to_hash
    end
    return @hash
  end

  def write(file)
    YAML::dump(to_hash, file)
  end

  def get_files_list
  end

  def generate_package_filename
    name = to_hash[:name]
    version = to_hash[:version]
    packageversion = nil
    if to_hash[:package_version] && !to_hash[:package_version].to_s.empty?
      packageversion = to_hash[:package_version]
    end<commit_msg>Fix bug with native dependency when using yaml<commit_after>require 'yaml'
require 'rexml/document'

module SymbolizeKeys

  # converts any current string keys to symbol keys 
  def self.extended(hash)
    hash.each do |key,value|
      if key.is_a?(String)
        hash.delete key
        hash[key] = value #through overridden []= 
      end
      if value.is_a?(Hash)
        hash[key]=value.extend(SymbolizeKeys)
      elsif value.is_a?(Array)
        value.each do |val|
          if val.is_a?(Hash)
            val.extend(SymbolizeKeys)
          end
        end
      end
    end
  end

  # assigns a new key/value pair 
  # converts they key to a symbol if it is a string 
  def []=(*args)
    args[0] = args[0].to_sym if args[0].is_a?(String)
    super
  end

  # returns new hash which is the merge of self and other hashes 
  # the returned hash will also be extended by SymbolizeKeys 
  def merge(*other_hashes , &resolution_proc )
    merged = Hash.new.extend SymbolizeKeys
    merged.merge! self , *other_hashes , &resolution_proc
  end

  # merges the other hashes into self 
  # if a proc is submitted , it's return will be the value for the key 
  def merge!( *other_hashes , &resolution_proc )

    # default resolution: value of the other hash 
    resolution_proc ||= proc{ |key,oldval,newval| newval }

    # merge each hash into self 
    other_hashes.each do |hash|
      hash.each{ |k,v|
        # assign new k/v into self, resolving conflicts with resolution_proc 
        self[k] = self.has_key?(k) ? resolution_proc[k,self[k],v] : v
      }
    end

    self
  end
end

class Metadata
  attr_accessor :source
  REQUIRED_FIELDS = [:name, :version, :maintainer]

  # Cleans up a string to make it suitable for use in a filename
  def self.clean_for_filename(dirtystring)
    dirtystring.downcase.gsub(/[^\w]/, '')
  end

  def self.get_pkgs_metadata_from_yml_doc(yml_doc, metadata=nil, source=nil)
    metadata = {} if metadata.nil?
    metadata_lists = yml_doc.split("---")
    metadata_lists.each do | metadata_text |
      if metadata_text =~ /^:?name:(.+)/
        name = $1.strip
        metadata[name] = [] if !metadata[name]
        metadata[name] << Metadata.new(metadata_text,'yml', source)
      end
    end
    return metadata
  end

  # metadata_text = text representation of the metadata
  # format = yml, xml, json, etc.
  def initialize(metadata_text, format, source=nil)
    @hash = nil
    @metadata_text = metadata_text
    @format = format
    @source = source
  end

  def [](key)
    return to_hash[key]
  end

  def []=(key,value)
    to_hash[key]=value
  end

  def to_hash
    if @hash  
      return @hash 
    end

    if @format == 'yml'
      hash = YAML::load(@metadata_text)
      @hash = hash.extend(SymbolizeKeys)

      # We need this for backward compatibility. With xml, we specify
      # native dependency as type: :native rather then native: true
      @hash[:dependencies].each do | dep |
        dep[:type] = :native if dep[:native]
      end if @hash[:dependencies]
    else
      @hash = metadata_xml_to_hash
    end
    return @hash
  end

  def write(file)
    YAML::dump(to_hash, file)
  end

  def get_files_list
  end

  def generate_package_filename
    name = to_hash[:name]
    version = to_hash[:version]
    packageversion = nil
    if to_hash[:package_version] && !to_hash[:package_version].to_s.empty?
      packageversion = to_hash[:package_version]
    end
{'fraction_non_alphanumeric': 0.0985101854667072, 'fraction_numerical': 0.0012161751292186075, 'mean_word_length': 2.7572815533980584, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2448536', 'n_tokens_mistral': 2205, 'n_tokens_neox': 2108, 'n_words': 617}