<commit_before>def dereference(schemata, data)
  if data.has_key?('$ref')
    begin
      schema_id, key = data['$ref'].split('#')
      schema_id = schema_id.gsub(%r{^/}, '') # drop leading slash if one exists
      definition = key.gsub('/definitions/', '')
      schemata[schema_id]['definitions'][definition]
    rescue => error
      $stderr.puts("Failed to dereference #{data}")
      raise(error)
    end
  else
    expand_references(schemata, data)
  end
end

def expand_references(schemata, data)
  data.keys.each do |key|
    value = data[key]
    data[key] = case value
    when Hash
      dereference(schemata, value)
    when Array
      if key == 'anyOf'
        value
      else
        value.map do |item|
          if item.is_a?(Hash)
            dereference(schemata, item)
          else
            item
          end
        end
      end
    else
      value
    end
  end
  data
end

def extract_attributes(schemata, properties)
  attributes = []
  properties.each do |key, value|
    # found a reference to another element:
    if value.has_key?('anyOf')
      descriptions = []
      examples = []

      # sort anyOf! always show unique identifier first
      anyof = value['anyOf'].sort_by do |property|
        property['$ref'].split('/').last.gsub('id', 'a')
      end

      anyof.each do |ref|
        nested_field = dereference(schemata, ref)
        descriptions << nested_field['description']
        examples << nested_field['example']
      end

      # avoid repetition :}
      if descriptions.size > 1
        descriptions.first.gsub!(/ of (this )?.*/, "")
        descriptions[1..-1].map { |d| d.gsub!(/unique /, "") }
      end
      description = descriptions.join(" or ")
      example = doc_example(*examples)
      attributes << [key, "string", description, example]

    # found a nested object
    elsif value['type'] == ['object'] && value['properties']
      properties = value['properties'].sort_by { |k, v| k }

      properties.each do |prop_name, prop_value|
        new_key = "#{key}:#{prop_name}"
        attributes << [new_key, doc_type(prop_value),
          prop_value['description'], doc_example(prop_value['example'])]
      end

    # just a regular attribute
    else
      example = doc_example(value['example'])
      attributes << [key, doc_type(value),
        value['description'], example]
    end
  end
  return attributes
end

def doc_type(property)
  schema_type = property["type"].dup<commit_msg>add support for enum type<commit_after>def dereference(schemata, data)
  if data.has_key?('$ref')
    begin
      schema_id, key = data['$ref'].split('#')
      schema_id = schema_id.gsub(%r{^/}, '') # drop leading slash if one exists
      definition = key.gsub('/definitions/', '')
      schemata[schema_id]['definitions'][definition]
    rescue => error
      $stderr.puts("Failed to dereference #{data}")
      raise(error)
    end
  else
    expand_references(schemata, data)
  end
end

def expand_references(schemata, data)
  data.keys.each do |key|
    value = data[key]
    data[key] = case value
    when Hash
      dereference(schemata, value)
    when Array
      if key == 'anyOf'
        value
      else
        value.map do |item|
          if item.is_a?(Hash)
            dereference(schemata, item)
          else
            item
          end
        end
      end
    else
      value
    end
  end
  data
end

def extract_attributes(schemata, properties)
  attributes = []
  properties.each do |key, value|
    # found a reference to another element:
    if value.has_key?('anyOf')
      descriptions = []
      examples = []

      # sort anyOf! always show unique identifier first
      anyof = value['anyOf'].sort_by do |property|
        property['$ref'].split('/').last.gsub('id', 'a')
      end

      anyof.each do |ref|
        nested_field = dereference(schemata, ref)
        descriptions << nested_field['description']
        examples << nested_field['example']
      end

      # avoid repetition :}
      if descriptions.size > 1
        descriptions.first.gsub!(/ of (this )?.*/, "")
        descriptions[1..-1].map { |d| d.gsub!(/unique /, "") }
      end
      description = descriptions.join(" or ")
      example = doc_example(*examples)
      attributes << [key, "string", description, example]

    # found a nested object
    elsif value['type'] == ['object'] && value['properties']
      properties = value['properties'].sort_by { |k, v| k }

      properties.each do |prop_name, prop_value|
        new_key = "#{key}:#{prop_name}"
        attributes << [new_key, doc_type(prop_value),
          prop_value['description'], doc_example(prop_value['example'])]
      end

    #found an enum
    elsif value['type'] == ['enum']
      example = doc_example(*value['oneOf'])
      attributes << [key, 'enum',
        value['description'], example]

    # just a regular attribute
    else
      example = doc_example(value['example'])
      attributes << [key, doc_type(value),
        value['description'], example]
    end
  end
  return attributes
end

def doc_type(property)
  schema_type = property["type"].dup
{'fraction_non_alphanumeric': 0.13315111284654432, 'fraction_numerical': 0.0011714174150722373, 'mean_word_length': 2.533103448275862, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2077844', 'n_tokens_mistral': 1551, 'n_tokens_neox': 1529, 'n_words': 384}