<commit_before># The Schema defines a collection of attributes.
# It is responsible for returning its attributes' values back to its Graph.

require "net/http"
require "json"
require "representation"
require "representation/attribute"

module Representation
  class Schema
    attr_accessor :title,
      :type,
      :attributes,
      :schema

    # The schema need to know where its getting a value from
    # an Attribute, e.g. { foo: "bar" } => { foo: { type: String } }
    # an Archetype, e.g. [1,2,3] => { type: Integer } # could this be a pass-through?
    # an Attribute-Schema, e.g. { foo: { bar: "baz" } } => { foo: { type: Object, attributes: { bar: { type: String } } }
    # an Attribute-Array, e.g. [{ foo: "bar" }] => { type: Array, elements: { type: Object, attributes: { foo: { type: String } } } }
    # and we need to identify when given { ids: [1,2] }, parental_mapping { ids: id } means to make 2 calls
    def initialize(schema_or_title)
      self.schema = schema_or_title.is_a?(Hash) ? schema_or_title : find_schema(schema_or_title)
      self.title = schema[:title]
      self.type = Representation.parse_type(schema[:type])

      if array_of_schemas?(schema[:elements])
        self.attributes = [Attribute.new({ elements: schema[:elements] })]
      else
        definitions = schema[:attributes] || schema[:elements]
        self.attributes = definitions.collect do |key, value|
          Attribute.new({ key => value })
        end
      end
    end

    def array_of_schemas?(definition = {})
      return false unless definition
      definition.keys.include?(:attributes)
    end

    def pull(options = {})
      endpoint = options.delete(:endpoint)
      data = Representation.live ? request(endpoint) : options
      { title.to_sym => serialize(data) }
    end

    def serialize(data)
      # data.is_a?(Array) ? to_array(data) : to_hash(data)
      (type == Array) ? to_array(data) : to_hash(data)
    end

    private

    def find_schema(title)
      loaded_schema = Representation.schemas[title.to_sym]
      raise Errors::Schema::NotFound.new(title) if !loaded_schema
      loaded_schema
    end

    def request(endpoint)
      response = Net::HTTP.get_response(URI(endpoint))
      if response.kind_of?(Net::HTTPSuccess)
        response = JSON.parse(response.body).recursive_symbolize_keys!
        if (response.is_a?(Array) || (response[title.to_sym] == nil))
          response
        else
          response[title.to_sym]
        end
      else
        raise Errors::Schema::RequestError.new(endpoint, response)
      end
    rescue JSON::ParserError => error
      raise Errors::Schema::InvalidResponse.new(endpoint, response.body)
    end

    def to_array(elements)
      # elements.first.is_a?(Hash) ? to_array_of_schemas(elements) : to_array_of_elements(elements)
      attributes.first.schema_value? ? to_array_of_schemas(elements) : to_array_of_elements(elements)
    end

    def to_array_of_elements(elements)
      (elements = stubbed_array) if !Representation.live && !elements
      archetype = attributes.first # there should only be one in the event that it's an array schema
      elements.collect do |element|
        archetype.serialize(element)
      end
    end

    def to_array_of_schemas(elements)
      (elements = stubbed_array) if !Representation.live && !elements<commit_msg>[bug] generate samples for arrays of schemas<commit_after># The Schema defines a collection of attributes.
# It is responsible for returning its attributes' values back to its Graph.

require "net/http"
require "json"
require "representation"
require "representation/attribute"

module Representation
  class Schema
    attr_accessor :title,
      :type,
      :attributes,
      :schema

    # The schema need to know where its getting a value from
    # an Attribute, e.g. { foo: "bar" } => { foo: { type: String } }
    # an Archetype, e.g. [1,2,3] => { type: Integer } # could this be a pass-through?
    # an Attribute-Schema, e.g. { foo: { bar: "baz" } } => { foo: { type: Object, attributes: { bar: { type: String } } }
    # an Attribute-Array, e.g. [{ foo: "bar" }] => { type: Array, elements: { type: Object, attributes: { foo: { type: String } } } }
    # and we need to identify when given { ids: [1,2] }, parental_mapping { ids: id } means to make 2 calls
    def initialize(schema_or_title)
      self.schema = schema_or_title.is_a?(Hash) ? schema_or_title : find_schema(schema_or_title)
      self.title = schema[:title]
      self.type = Representation.parse_type(schema[:type])

      if array_of_schemas?(schema[:elements])
        self.attributes = [Attribute.new({ elements: schema[:elements] })]
      else
        definitions = schema[:attributes] || schema[:elements]
        self.attributes = definitions.collect do |key, value|
          Attribute.new({ key => value })
        end
      end
    end

    def array_of_schemas?(definition = {})
      return false unless definition
      definition.keys.include?(:attributes)
    end

    def pull(options = {})
      endpoint = options.delete(:endpoint)
      data = Representation.live ? request(endpoint) : options
      { title.to_sym => serialize(data) }
    end

    def serialize(data)
      # data.is_a?(Array) ? to_array(data) : to_hash(data)
      (type == Array) ? to_array(data) : to_hash(data)
    end

    private

    def find_schema(title)
      loaded_schema = Representation.schemas[title.to_sym]
      raise Errors::Schema::NotFound.new(title) if !loaded_schema
      loaded_schema
    end

    def request(endpoint)
      response = Net::HTTP.get_response(URI(endpoint))
      if response.kind_of?(Net::HTTPSuccess)
        response = JSON.parse(response.body).recursive_symbolize_keys!
        if (response.is_a?(Array) || (response[title.to_sym] == nil))
          response
        else
          response[title.to_sym]
        end
      else
        raise Errors::Schema::RequestError.new(endpoint, response)
      end
    rescue JSON::ParserError => error
      raise Errors::Schema::InvalidResponse.new(endpoint, response.body)
    end

    def to_array(elements)
      # elements.first.is_a?(Hash) ? to_array_of_schemas(elements) : to_array_of_elements(elements)
      attributes.first.schema_value? ? to_array_of_schemas(elements) : to_array_of_elements(elements)
    end

    def to_array_of_elements(elements)
      (elements = stubbed_array) if !Representation.live && (!elements || elements.empty?)
      archetype = attributes.first # there should only be one in the event that it's an array schema
      elements.collect do |element|
        archetype.serialize(element)
      end
    end

    def to_array_of_schemas(elements)
      (elements = stubbed_array) if !Representation.live && (!elements || elements.empty?)
{'fraction_non_alphanumeric': 0.11480005902316659, 'fraction_numerical': 0.0017706949977866313, 'mean_word_length': 3.135448444173276, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5368078', 'n_tokens_mistral': 2038, 'n_tokens_neox': 2014, 'n_words': 515}