<commit_before># -*- encoding: utf-8 -*-
# frozen_string_literal: true
require 'set'

module JSON::LD
  ##
  # Expand module, used as part of API
  module Expand
    include Utils

    ##
    # Expand an Array or Object given an active context and performing local context expansion.
    #
    # @param [Array, Hash] input
    # @param [String] active_property
    # @param [Context] context
    # @param [Boolean] ordered (true)
    #   Ensure output objects have keys ordered properly
    # @return [Array<Hash{String => Object}>]
    def expand(input, active_property, context, ordered: true)
      framing = @options[:processingMode].include?("expand-frame")
      #log_debug("expand") {"input: #{input.inspect}, active_property: #{active_property.inspect}, context: #{context.inspect}"}
      result = case input
      when Array
        # If element is an array,
        is_list = context.container(active_property) == %w(@list)
        value = input.each_with_object([]) do |v, memo|
          # Initialize expanded item to the result of using this algorithm recursively, passing active context, active property, and item as element.
          v = expand(v, active_property, context, ordered: ordered)

          # If the active property is @list or its container mapping is set to @list, the expanded item must not be an array or a list object, otherwise a list of lists error has been detected and processing is aborted.
          raise JsonLdError::ListOfLists,
                "A list may not contain another list" if
                is_list && (v.is_a?(Array) || list?(v))
          case v
          when nil then nil
          when Array then memo.concat(v)
          else            memo << v
          end
        end

        value
      when Hash
        # If element contains the key @context, set active context to the result of the Context Processing algorithm, passing active context and the value of the @context key as local context.
        if input.has_key?('@context')
          context = context.parse(input.delete('@context'))
          #log_debug("expand") {"context: #{context.inspect}"}
        end

        output_object = {}

        # See if keys mapping to @type have terms with a local context
        input.each_pair do |key, val|
          next unless context.expand_iri(key, vocab: true) == '@type'
          Array(val).each do |term|
            term_context = context.term_definitions[term].context if context.term_definitions[term]
            context = term_context ? context.parse(term_context) : context
          end
        end

        # Process each key and value in element. Ignores @nesting content
        expand_object(input, active_property, context, output_object, ordered: ordered)

        #log_debug("output object") {output_object.inspect}

        # If result contains the key @value:
        if value?(output_object)
          unless (output_object.keys - %w(@value @language @type @index)).empty? &&
                 !(output_object.key?('@language') && output_object.key?('@type'))
            # The result must not contain any keys other than @value, @language, @type, and @index. It must not contain both the @language key and the @type key. Otherwise, an invalid value object error has been detected and processing is aborted.
            raise JsonLdError::InvalidValueObject,
            "value object has unknown keys: #{output_object.inspect}"
          end

          output_object.delete('@language') if output_object.key?('@language') && Array(output_object['@language']).empty?
          output_object.delete('@type') if output_object.key?('@type') && Array(output_object['@type']).empty?

          # If the value of result's @value key is null, then set result to null.
          ary = Array(output_object['@value'])
          return nil if ary.empty?

          if !ary.all? {|v| v.is_a?(String) || v.is_a?(Hash) && v.empty?} && output_object.has_key?('@language')
            # Otherwise, if the value of result's @value member is not a string and result contains the key @language, an invalid language-tagged value error has been detected (only strings can be language-tagged) and processing is aborted.
            raise JsonLdError::InvalidLanguageTaggedValue,
                  "when @language is used, @value must be a string: #{output_object.inspect}"
          elsif !Array(output_object.fetch('@type', "")).all? {|t|
                  t.is_a?(String) && context.expand_iri(t, vocab: true, log_depth: @options[:log_depth]).is_a?(RDF::URI) ||
                  t.is_a?(Hash) && t.empty?}
            # Otherwise, if the result has a @type member and its value is not an IRI, an invalid typed value error has been detected and processing is aborted.
            raise JsonLdError::InvalidTypedValue,
                  "value of @type must be an IRI: #{output_object.inspect}"
          end
        elsif !output_object.fetch('@type', []).is_a?(Array)
          # Otherwise, if result contains the key @type and its associated value is not an array, set it to an array containing only the associated value.
          output_object['@type'] = [output_object['@type']]
        elsif output_object.key?('@set') || output_object.key?('@list')
          # Otherwise, if result contains the key @set or @list:
          # The result must contain at most one other key and that key must be @index. Otherwise, an invalid set or list object error has been detected and processing is aborted.
          raise JsonLdError::InvalidSetOrListObject,
                "@set or @list may only contain @index: #{output_object.keys.inspect}" unless
                (output_object.keys - %w(@set @list @index)).empty?

          # If result contains the key @set, then set result to the key's associated value.
          return output_object['@set'] if output_object.key?('@set')
        end

        # If result contains only the key @language, set result to null.
        return nil if output_object.length == 1 && output_object.key?('@language')

        # If active property is null or @graph, drop free-floating values as follows:
        if (active_property || '@graph') == '@graph' &&
          (output_object.key?('@value') || output_object.key?('@list') ||
           (output_object.keys - %w(@id)).empty? && !framing)
          #log_debug(" =>") { "empty top-level: " + output_object.inspect}
          return nil
        end

        # Re-order result keys if ordering
        if ordered
          output_object.keys.kw_sort.each_with_object({}) {|kk, memo| memo[kk] = output_object[kk]}
        else
          output_object
        end
      else
        # Otherwise, unless the value is a number, expand the value according to the Value Expansion rules, passing active property.
        return nil if input.nil? || active_property.nil? || active_property == '@graph'
        context.expand_value(active_property, input, log_depth: @options[:log_depth])
      end

      #log_debug {" => #{result.inspect}"}
      result
    end

  private
    CONTAINER_MAPPING_INDEX_ID_TYPE = Set.new(%w(@index @id @type)).freeze

    # Expand each key and value of element adding them to result
    def expand_object(input, active_property, context, output_object, ordered: false)
      framing = @options[:processingMode].include?("expand-frame")
      nests = []

      # Then, proceed and process each property and value in element as follows:
      keys = ordered ? input.keys.kw_sort : input.keys
      keys.each do |key|
        # For each key and value in element, ordered lexicographically by key:
        value = input[key]
        expanded_property = context.expand_iri(key, vocab: true, quiet: true)

        # If expanded property is null or it neither contains a colon (:) nor it is a keyword, drop key by continuing to the next key.
        next if expanded_property.is_a?(RDF::URI) && expanded_property.relative?
        expanded_property = expanded_property.to_s if expanded_property.is_a?(RDF::Resource)

        #log_debug("expand property") {"ap: #{active_property.inspect}, expanded: #{expanded_property.inspect}, value: #{value.inspect}"}

        if expanded_property.nil?
          #log_debug(" => ") {"skip nil property"}
          next
        end

        if KEYWORDS.include?(expanded_property)
          # If active property equals @reverse, an invalid reverse property map error has been detected and processing is aborted.
          raise JsonLdError::InvalidReversePropertyMap,
                "@reverse not appropriate at this point" if active_property == '@reverse'

          # If result has already an expanded property member, an colliding keywords error has been detected and processing is aborted.
          raise JsonLdError::CollidingKeywords,
                "#{expanded_property} already exists in result" if output_object.has_key?(expanded_property)

          expanded_value = case expanded_property
          when '@id'
            # If expanded property is @id and value is not a string, an invalid @id value error has been detected and processing is aborted
            e_id = case value
            when String
              context.expand_iri(value, documentRelative: true, quiet: true).to_s
            when Array
              # Framing allows an array of IRIs, and always puts values in an array
              raise JsonLdError::InvalidIdValue,
                    "value of @id must be a string, array of string or hash if framing: #{value.inspect}" unless framing
              context.expand_iri(value, documentRelative: true, quiet: true).to_s
              value.map do |v|
                raise JsonLdError::InvalidTypeValue,
                      "@id value must be a string or array of strings for framing: #{v.inspect}" unless v.is_a?(String)
                context.expand_iri(v, documentRelative: true, quiet: true,).to_s
              end
            when Hash
              raise JsonLdError::InvalidIdValue,
                    "value of @id must be a string unless framing: #{value.inspect}" unless framing
              raise JsonLdError::InvalidTypeValue,
                    "value of @id must be a an empty object for framing: #{value.inspect}" unless
                    value.empty? && framing
              [{}]
            else
              raise JsonLdError::InvalidIdValue,
                    "value of @id must be a string or hash if framing: #{value.inspect}"
            end

            # Use array form if framing
            if framing && !e_id.is_a?(Array)
              [e_id]
            else
              e_id
            end
          when '@type'
            # If expanded property is @type and value is neither a string nor an array of strings, an invalid type value error has been detected and processing is aborted. Otherwise, set expanded value to the result of using the IRI Expansion algorithm, passing active context, true for vocab, and true for document relative to expand the value or each of its items.
            #log_debug("@type") {"value: #{value.inspect}"}<commit_msg>Improve exception messages on invalid framing values when expanding.<commit_after># -*- encoding: utf-8 -*-
# frozen_string_literal: true
require 'set'

module JSON::LD
  ##
  # Expand module, used as part of API
  module Expand
    include Utils

    ##
    # Expand an Array or Object given an active context and performing local context expansion.
    #
    # @param [Array, Hash] input
    # @param [String] active_property
    # @param [Context] context
    # @param [Boolean] ordered (true)
    #   Ensure output objects have keys ordered properly
    # @return [Array<Hash{String => Object}>]
    def expand(input, active_property, context, ordered: true)
      framing = @options[:processingMode].include?("expand-frame")
      #log_debug("expand") {"input: #{input.inspect}, active_property: #{active_property.inspect}, context: #{context.inspect}"}
      result = case input
      when Array
        # If element is an array,
        is_list = context.container(active_property) == %w(@list)
        value = input.each_with_object([]) do |v, memo|
          # Initialize expanded item to the result of using this algorithm recursively, passing active context, active property, and item as element.
          v = expand(v, active_property, context, ordered: ordered)

          # If the active property is @list or its container mapping is set to @list, the expanded item must not be an array or a list object, otherwise a list of lists error has been detected and processing is aborted.
          raise JsonLdError::ListOfLists,
                "A list may not contain another list" if
                is_list && (v.is_a?(Array) || list?(v))
          case v
          when nil then nil
          when Array then memo.concat(v)
          else            memo << v
          end
        end

        value
      when Hash
        # If element contains the key @context, set active context to the result of the Context Processing algorithm, passing active context and the value of the @context key as local context.
        if input.has_key?('@context')
          context = context.parse(input.delete('@context'))
          #log_debug("expand") {"context: #{context.inspect}"}
        end

        output_object = {}

        # See if keys mapping to @type have terms with a local context
        input.each_pair do |key, val|
          next unless context.expand_iri(key, vocab: true) == '@type'
          Array(val).each do |term|
            term_context = context.term_definitions[term].context if context.term_definitions[term]
            context = term_context ? context.parse(term_context) : context
          end
        end

        # Process each key and value in element. Ignores @nesting content
        expand_object(input, active_property, context, output_object, ordered: ordered)

        #log_debug("output object") {output_object.inspect}

        # If result contains the key @value:
        if value?(output_object)
          unless (output_object.keys - %w(@value @language @type @index)).empty? &&
                 !(output_object.key?('@language') && output_object.key?('@type'))
            # The result must not contain any keys other than @value, @language, @type, and @index. It must not contain both the @language key and the @type key. Otherwise, an invalid value object error has been detected and processing is aborted.
            raise JsonLdError::InvalidValueObject,
            "value object has unknown keys: #{output_object.inspect}"
          end

          output_object.delete('@language') if output_object.key?('@language') && Array(output_object['@language']).empty?
          output_object.delete('@type') if output_object.key?('@type') && Array(output_object['@type']).empty?

          # If the value of result's @value key is null, then set result to null.
          ary = Array(output_object['@value'])
          return nil if ary.empty?

          if !ary.all? {|v| v.is_a?(String) || v.is_a?(Hash) && v.empty?} && output_object.has_key?('@language')
            # Otherwise, if the value of result's @value member is not a string and result contains the key @language, an invalid language-tagged value error has been detected (only strings can be language-tagged) and processing is aborted.
            raise JsonLdError::InvalidLanguageTaggedValue,
                  "when @language is used, @value must be a string: #{output_object.inspect}"
          elsif !Array(output_object.fetch('@type', "")).all? {|t|
                  t.is_a?(String) && context.expand_iri(t, vocab: true, log_depth: @options[:log_depth]).is_a?(RDF::URI) ||
                  t.is_a?(Hash) && t.empty?}
            # Otherwise, if the result has a @type member and its value is not an IRI, an invalid typed value error has been detected and processing is aborted.
            raise JsonLdError::InvalidTypedValue,
                  "value of @type must be an IRI: #{output_object.inspect}"
          end
        elsif !output_object.fetch('@type', []).is_a?(Array)
          # Otherwise, if result contains the key @type and its associated value is not an array, set it to an array containing only the associated value.
          output_object['@type'] = [output_object['@type']]
        elsif output_object.key?('@set') || output_object.key?('@list')
          # Otherwise, if result contains the key @set or @list:
          # The result must contain at most one other key and that key must be @index. Otherwise, an invalid set or list object error has been detected and processing is aborted.
          raise JsonLdError::InvalidSetOrListObject,
                "@set or @list may only contain @index: #{output_object.keys.inspect}" unless
                (output_object.keys - %w(@set @list @index)).empty?

          # If result contains the key @set, then set result to the key's associated value.
          return output_object['@set'] if output_object.key?('@set')
        end

        # If result contains only the key @language, set result to null.
        return nil if output_object.length == 1 && output_object.key?('@language')

        # If active property is null or @graph, drop free-floating values as follows:
        if (active_property || '@graph') == '@graph' &&
          (output_object.key?('@value') || output_object.key?('@list') ||
           (output_object.keys - %w(@id)).empty? && !framing)
          #log_debug(" =>") { "empty top-level: " + output_object.inspect}
          return nil
        end

        # Re-order result keys if ordering
        if ordered
          output_object.keys.kw_sort.each_with_object({}) {|kk, memo| memo[kk] = output_object[kk]}
        else
          output_object
        end
      else
        # Otherwise, unless the value is a number, expand the value according to the Value Expansion rules, passing active property.
        return nil if input.nil? || active_property.nil? || active_property == '@graph'
        context.expand_value(active_property, input, log_depth: @options[:log_depth])
      end

      #log_debug {" => #{result.inspect}"}
      result
    end

  private
    CONTAINER_MAPPING_INDEX_ID_TYPE = Set.new(%w(@index @id @type)).freeze

    # Expand each key and value of element adding them to result
    def expand_object(input, active_property, context, output_object, ordered: false)
      framing = @options[:processingMode].include?("expand-frame")
      nests = []

      # Then, proceed and process each property and value in element as follows:
      keys = ordered ? input.keys.kw_sort : input.keys
      keys.each do |key|
        # For each key and value in element, ordered lexicographically by key:
        value = input[key]
        expanded_property = context.expand_iri(key, vocab: true, quiet: true)

        # If expanded property is null or it neither contains a colon (:) nor it is a keyword, drop key by continuing to the next key.
        next if expanded_property.is_a?(RDF::URI) && expanded_property.relative?
        expanded_property = expanded_property.to_s if expanded_property.is_a?(RDF::Resource)

        #log_debug("expand property") {"ap: #{active_property.inspect}, expanded: #{expanded_property.inspect}, value: #{value.inspect}"}

        if expanded_property.nil?
          #log_debug(" => ") {"skip nil property"}
          next
        end

        if KEYWORDS.include?(expanded_property)
          # If active property equals @reverse, an invalid reverse property map error has been detected and processing is aborted.
          raise JsonLdError::InvalidReversePropertyMap,
                "@reverse not appropriate at this point" if active_property == '@reverse'

          # If result has already an expanded property member, an colliding keywords error has been detected and processing is aborted.
          raise JsonLdError::CollidingKeywords,
                "#{expanded_property} already exists in result" if output_object.has_key?(expanded_property)

          expanded_value = case expanded_property
          when '@id'
            # If expanded property is @id and value is not a string, an invalid @id value error has been detected and processing is aborted
            e_id = case value
            when String
              context.expand_iri(value, documentRelative: true, quiet: true).to_s
            when Array
              # Framing allows an array of IRIs, and always puts values in an array
              raise JsonLdError::InvalidIdValue,
                    "value of @id must be a string unless framing: #{value.inspect}" unless framing
              context.expand_iri(value, documentRelative: true, quiet: true).to_s
              value.map do |v|
                raise JsonLdError::InvalidTypeValue,
                      "@id value must be a string or array of strings for framing: #{v.inspect}" unless v.is_a?(String)
                context.expand_iri(v, documentRelative: true, quiet: true,).to_s
              end
            when Hash
              raise JsonLdError::InvalidIdValue,
                    "value of @id must be a string unless framing: #{value.inspect}" unless framing
              raise JsonLdError::InvalidTypeValue,
                    "value of @id must be a an empty object for framing: #{value.inspect}" unless
                    value.empty?
              [{}]
            else
              raise JsonLdError::InvalidIdValue,
                    "value of @id must be a string or hash if framing: #{value.inspect}"
            end

            # Use array form if framing
            if framing && !e_id.is_a?(Array)
              [e_id]
            else
              e_id
            end
          when '@type'
            # If expanded property is @type and value is neither a string nor an array of strings, an invalid type value error has been detected and processing is aborted. Otherwise, set expanded value to the result of using the IRI Expansion algorithm, passing active context, true for vocab, and true for document relative to expand the value or each of its items.
            #log_debug("@type") {"value: #{value.inspect}"}
{'fraction_non_alphanumeric': 0.0947639951124587, 'fraction_numerical': 0.00018102004797031272, 'mean_word_length': 2.570528356761997, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5071533', 'n_tokens_mistral': 5859, 'n_tokens_neox': 5763, 'n_words': 2159}