<commit_before>module JsonValidation
  class Schema
    extend Forwardable
    def_delegators :@schema_data, :keys, :fetch, :all?, :any?, :has_key?, :size

    attr_reader :schema_data, :base_uri, :fragment_elements, :resolution_scope

    def initialize(schema_data, base_uri = nil, resolution_scope = nil, fragment_elements = [])
      @schema_data = schema_data

      if base_uri
        @base_uri = Addressable::URI.parse(base_uri)
      else
        @base_uri = generate_base_uri
      end
      raise "base_uri has fragment" unless @base_uri.fragment.nil?

      if schema_data["id"] && schema_data["id"].is_a?(String)
        if resolution_scope
          @resolution_scope = resolution_scope.join(Addressable::URI.parse(schema_data["id"]))
        else
          @resolution_scope = Addressable::URI.parse(schema_data["id"])
        end
      else
        if resolution_scope
          @resolution_scope = resolution_scope
        else
          @resolution_scope = @base_uri
        end
      end

      @fragment_elements = fragment_elements
    end

    def generate_base_uri
      Addressable::URI.parse(Digest::SHA1.hexdigest(schema_data.to_json))
    end

    def uri
      if fragment_elements.empty?
        fragment = ""
      else
        fragment = "/" + fragment_elements.join("/")
      end
      Addressable::URI.new(base_uri.to_hash.merge(fragment: fragment))
    end

    def [](fragment)
      return self if fragment.nil?

      key, remainder = fragment.split("/", 2)

      if key == "" || key.nil?
        value = self
      else
        escaped_key = key.gsub('~0', '~').gsub('~1', '/').gsub('%25', '%')
        value = schema_data[escaped_key]
        if value.is_a?(Hash)
          value = Schema.new(value, base_uri, resolution_scope, fragment_elements + [key.to_s])
        elsif value.is_a?(Array) && value[0].is_a?(Hash)
          value = value.each_with_index.map {|v, ix|
            Schema.new(v, base_uri, resolution_scope, fragment_elements + [key.to_s] + [ix.to_s])
          }
        end
      end

      if remainder.nil?
        value
      else
        case value
        when Schema
          value[remainder]
        when Array
          value[remainder.to_i]
        else
          raise "Unexpected type for #{value}"
        end
      end
    end

    def map(&block)
      schema_data.map {|key, subschema|
        block.call(key, Schema.new(subschema, base_uri, resolution_scope, fragment_elements + [key]))
      }
    end
  end
end<commit_msg>Factor out some common code<commit_after>module JsonValidation
  class Schema
    extend Forwardable
    def_delegators :@schema_data, :keys, :fetch, :all?, :any?, :has_key?, :size

    attr_reader :schema_data, :base_uri, :fragment_elements, :resolution_scope

    def initialize(schema_data, base_uri = nil, resolution_scope = nil, fragment_elements = [])
      @schema_data = schema_data

      if base_uri
        @base_uri = Addressable::URI.parse(base_uri)
      else
        @base_uri = generate_base_uri
      end
      raise "base_uri has fragment" unless @base_uri.fragment.nil?

      if schema_data["id"] && schema_data["id"].is_a?(String)
        if resolution_scope
          @resolution_scope = resolution_scope.join(Addressable::URI.parse(schema_data["id"]))
        else
          @resolution_scope = Addressable::URI.parse(schema_data["id"])
        end
      else
        if resolution_scope
          @resolution_scope = resolution_scope
        else
          @resolution_scope = @base_uri
        end
      end

      @fragment_elements = fragment_elements
    end

    def generate_base_uri
      Addressable::URI.parse(Digest::SHA1.hexdigest(schema_data.to_json))
    end

    def uri
      if fragment_elements.empty?
        fragment = ""
      else
        fragment = "/" + fragment_elements.join("/")
      end
      Addressable::URI.new(base_uri.to_hash.merge(fragment: fragment))
    end

    def [](fragment)
      return self if fragment.nil?

      key, remainder = fragment.split("/", 2)

      if key == "" || key.nil?
        value = self
      else
        escaped_key = key.gsub('~0', '~').gsub('~1', '/').gsub('%25', '%')
        value = schema_data[escaped_key]
        if value.is_a?(Hash)
          value = build_subschema(value, [key])
        elsif value.is_a?(Array) && value[0].is_a?(Hash)
          value = value.each_with_index.map {|v, ix|
            build_subschema(v, [key, ix.to_s])
          }
        end
      end

      if remainder.nil?
        value
      else
        case value
        when Schema
          value[remainder]
        when Array
          value[remainder.to_i]
        else
          raise "Unexpected type for #{value}"
        end
      end
    end

    def build_subschema(subschema_data, extra_fragment_elements)
      Schema.new(subschema_data, base_uri, resolution_scope, fragment_elements + extra_fragment_elements)
    end

    def map(&block)
      schema_data.map {|key, subschema_data|
        block.call(key, build_subschema(subschema_data, [key]))
      }
    end
  end
end
{'fraction_non_alphanumeric': 0.10865270644014223, 'fraction_numerical': 0.002765705254839984, 'mean_word_length': 2.3730846102598266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7633303', 'n_tokens_mistral': 1579, 'n_tokens_neox': 1542, 'n_words': 327}