<commit_before>module Miletus::Merge

  class Facet < ActiveRecord::Base
    include Miletus::NamespaceHelper

    self.table_name = 'merge_facets'

    attr_accessible :metadata
    belongs_to :concept, :touch => true

    before_validation :update_key
    after_save :reindex_concept

    def self.find_existing(xml)
      find_by_key(global_key(xml))
    end

    def reindex_concept
      concept.update_indexed_attributes_from_facet_rifcs
    end

    def to_rif
      doc = Nokogiri::XML(metadata)
      ns_by_prefix('rif').schema.valid?(doc) ?
        clean_rifcs_doc(doc).root.to_s : nil
    end

    private

    def update_key
      write_attribute(:key, self.class.global_key(metadata))
    end

    def self.global_key(xml)
      return nil if xml.nil?
      doc = Nokogiri::XML(xml)
      key_e = doc.at_xpath('//rif:registryObject/rif:key',
        Miletus::NamespaceHelper.ns_decl)
      return nil if key_e.nil?
      key_e.content.strip
    end

    def clean_rifcs_doc(doc)
      translate_old_elements(doc)
      update_datetime(doc)
      doc
    end

    def update_datetime(rifcs_doc)
      types = %w{collection party activity service}
      pattern = types.map { |e| "//rif:#{e}"}.join(' | ')
      rifcs_doc.xpath(pattern, ns_decl).each do |e|
        e['dateModified'] = (updated_at or Time.now).utc.iso8601
      end
    end

    def translate_old_elements(rifcs_doc)
      types = {'rights' => 'rightsStatement', 'accessRights' => 'accessRights'}
      # For those description elements we can translate...
      find_description_elements_with_types(rifcs_doc, types.keys).each do |e|
        # Get the rights element or create it if necessary
        rights = find_or_create_rifcs_rights_element(e.parent)
        # Create new element based on attribute name and insert it
        node = Nokogiri::XML::Node.new(types[e['type'].to_s], rifcs_doc)
        node.content = e.content
        rights << node
        e.remove
      end
    end

    def find_description_elements_with_types(rifcs_doc, types)
      pattern = types.map do |t|
        "//rif:description[@type=\"%s\"]" % t
      end.join("|")
      rifcs_doc.xpath(pattern, ns_decl)
    end

    def find_or_create_rifcs_rights_element(node)
      rights = node.at_xpath("rif:rights", ns_decl)
      rights ||= Nokogiri::XML::Node.new('rights', node.document)
      node << rights
      rights
    end

  end

end<commit_msg>Changing RIF-CS generation so that cleaning steps have a chance to make the document valid.<commit_after>module Miletus::Merge

  class Facet < ActiveRecord::Base
    include Miletus::NamespaceHelper

    self.table_name = 'merge_facets'

    attr_accessible :metadata
    belongs_to :concept, :touch => true

    before_validation :update_key
    after_save :reindex_concept

    def self.find_existing(xml)
      find_by_key(global_key(xml))
    end

    def reindex_concept
      concept.update_indexed_attributes_from_facet_rifcs
    end

    def to_rif
      doc = clean_rifcs_doc(Nokogiri::XML(metadata))
      ns_by_prefix('rif').schema.valid?(doc) ? doc.root.to_s : nil
    end

    private

    def update_key
      write_attribute(:key, self.class.global_key(metadata))
    end

    def self.global_key(xml)
      return nil if xml.nil?
      doc = Nokogiri::XML(xml)
      key_e = doc.at_xpath('//rif:registryObject/rif:key',
        Miletus::NamespaceHelper.ns_decl)
      return nil if key_e.nil?
      key_e.content.strip
    end

    def clean_rifcs_doc(doc)
      translate_old_elements(doc)
      update_datetime(doc)
      strip_schema_location(doc)
      doc
    end

    def update_datetime(rifcs_doc)
      types = %w{collection party activity service}
      pattern = types.map { |e| "//rif:#{e}"}.join(' | ')
      rifcs_doc.xpath(pattern, ns_decl).each do |e|
        e['dateModified'] = (updated_at or Time.now).utc.iso8601
      end
    end

    def translate_old_elements(rifcs_doc)
      types = {'rights' => 'rightsStatement', 'accessRights' => 'accessRights'}
      # For those description elements we can translate...
      find_description_elements_with_types(rifcs_doc, types.keys).each do |e|
        # Get the rights element or create it if necessary
        rights = find_or_create_rifcs_rights_element(e.parent)
        # Create new element based on attribute name and insert it
        node = Nokogiri::XML::Node.new(types[e['type'].to_s], rifcs_doc)
        node.content = e.content
        rights << node
        e.remove
      end
    end

    def strip_schema_location(rifcs_doc)
      n = rifcs_doc.root
      n.remove_attribute('schemaLocation') unless n.nil?
    end

    def find_description_elements_with_types(rifcs_doc, types)
      pattern = types.map do |t|
        "//rif:description[@type=\"%s\"]" % t
      end.join("|")
      rifcs_doc.xpath(pattern, ns_decl)
    end

    def find_or_create_rifcs_rights_element(node)
      rights = node.at_xpath("rif:rights", ns_decl)
      rights ||= Nokogiri::XML::Node.new('rights', node.document)
      node << rights
      rights
    end

  end

end
{'fraction_non_alphanumeric': 0.1022749752720079, 'fraction_numerical': 0.001582591493570722, 'mean_word_length': 2.904247104247104, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7430200', 'n_tokens_mistral': 1691, 'n_tokens_neox': 1631, 'n_words': 344}