<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.avro;

import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import org.apache.avro.util.internal.JacksonUtils;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.node.DoubleNode;

/** An abstract data type.
 * <p>A schema may be one of:
 * <ul>
 * <li>A <i>record</i>, mapping field names to field value data;
 * <li>An <i>enum</i>, containing one of a small set of symbols;
 * <li>An <i>array</i> of values, all of the same schema;
 * <li>A <i>map</i>, containing string/value pairs, of a declared schema;
 * <li>A <i>union</i> of other schemas;
 * <li>A <i>fixed</i> sized binary object;
 * <li>A unicode <i>string</i>;
 * <li>A sequence of <i>bytes</i>;
 * <li>A 32-bit signed <i>int</i>;
 * <li>A 64-bit signed <i>long</i>;
 * <li>A 32-bit IEEE single-<i>float</i>; or
 * <li>A 64-bit IEEE <i>double</i>-float; or
 * <li>A <i>boolean</i>; or
 * <li><i>null</i>.
 * </ul>
 *
 * A schema can be constructed using one of its static <tt>createXXX</tt>
 * methods, or more conveniently using {@link SchemaBuilder}. The schema objects are
 * <i>logically</i> immutable.
 * There are only three mutating methods - {@link #setFields(List)},
 * {@link #addProp(String, String)}, and {@link #setLogicalType(LogicalType)}.
 * The following restrictions apply on these methods.
 * <ul>
 * <li> {@link #setFields(List)}, can be called at most once. This method exists
 * in order to enable clients to build recursive schemas.
 * <li> {@link #addProp(String, String)} can be called with property names
 * that are not present already. It is not possible to change or delete an
 * existing property.
 * <li> {@link #setLogicalType(LogicalType)} can be called at most once.
 * </ul>
 */
public abstract class Schema extends JsonProperties implements Serializable {

  private static final long serialVersionUID = 1L;

  protected Object writeReplace() {
    SerializableSchema ss = new SerializableSchema();
    ss.schemaString = toString();
    return ss;
  }

  private static final class SerializableSchema implements Serializable {

    private static final long serialVersionUID = 1L;

    private String schemaString;

    private Object readResolve() {
      return new Schema.Parser().parse(schemaString);
    }
  }

  public static final JsonFactory FACTORY = new JsonFactory();
  public static final ObjectMapper MAPPER = new ObjectMapper(FACTORY);

  private static final int NO_HASHCODE = Integer.MIN_VALUE;

  public static final Schema fromString(final String schemaJson) {
    return new Schema.Parser().parse(schemaJson);
  }

  public static final Schema fromString(final String schemaJson, final String ... more) {
    return new Schema.Parser().parse(schemaJson, more);
  }

  static {
    FACTORY.enable(JsonParser.Feature.ALLOW_COMMENTS);
    FACTORY.setCodec(MAPPER);
  }

  /** The type of a schema. */
  public enum Type {
    RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES,
      INT, LONG, FLOAT, DOUBLE, BOOLEAN, NULL;
    private String name;
    private Type() { this.name = this.name().toLowerCase(Locale.ENGLISH); }
    public String getName() { return name; }
  };

  private final Type type;
  private LogicalType logicalType;

  Schema(Type type) {
    super(SCHEMA_RESERVED);
    this.type = type;
    this.logicalType = null;
  }

  /** Create a schema for a primitive type. */
  public static Schema create(Type type) {
    switch (type) {
    case STRING:  return new StringSchema();
    case BYTES:   return new BytesSchema();
    case INT:     return new IntSchema();
    case LONG:    return new LongSchema();
    case FLOAT:   return new FloatSchema();
    case DOUBLE:  return new DoubleSchema();
    case BOOLEAN: return new BooleanSchema();
    case NULL:    return new NullSchema();
    default: throw new AvroRuntimeException("Can't create a: "+type);
    }
  }

  private static final Set<String> SCHEMA_RESERVED = new HashSet<String>();
  static {
    Collections.addAll(SCHEMA_RESERVED,
                       "doc", "fields", "items", "name", "namespace",
                       "size", "symbols", "values", "type", "aliases");
  }
  private static final Set<String> ENUM_RESERVED = new HashSet<>();
  static {
    ENUM_RESERVED.add("default");
    ENUM_RESERVED.addAll(SCHEMA_RESERVED);
  }

  int hashCode = NO_HASHCODE;

  public Schema withProp(String name, String value) {
    addProp(name, value);
    return this;
  }

  public Schema withProp(String name, Object value) {
    addProp(name, value);
    return this;
  }

  /** Create an anonymous record schema. */
  public static Schema createRecord(List<Field> fields) {
    Schema result = createRecord(null, null, null, false);
    result.setFields(fields);
    return result;
  }

  /** Create a named record schema. */
  public static Schema createRecord(String name, String doc, String namespace,
                                    boolean isError) {
    return new RecordSchema(new Name(name, namespace), doc, isError);
  }

  /** Create a named record schema with fields already set. */
  public static Schema createRecord(String name, String doc, String namespace,
                                    boolean isError, List<Field> fields) {
    return new RecordSchema(new Name(name, namespace), doc, isError, fields);
  }

  /** Create an enum schema. */
  public static Schema createEnum(String name, String doc, String namespace,
                                  List<String> values) {
    return new EnumSchema(new Name(name, namespace), doc,
        new LockableArrayList<String>(values), null);
  }

  /** Create an enum schema. */
  public static Schema createEnum(String name, String doc, String namespace,
                                  List<String> values, final String defaultEnumVal) {
    return new EnumSchema(new Name(name, namespace), doc,
        new LockableArrayList<String>(values), defaultEnumVal);
  }

  /** Create an array schema. */
  public static Schema createArray(Schema elementType) {
    return new ArraySchema(elementType);
  }

  /** Create a map schema. */
  public static Schema createMap(Schema valueType) {
    return new MapSchema(valueType);
  }

  /** Create a union schema. */
  public static Schema createUnion(List<Schema> types) {
    return new UnionSchema(new LockableArrayList<Schema>(types));
  }

  /** Create a union schema. */
  public static Schema createUnion(Schema... types) {
    return createUnion(new LockableArrayList<Schema>(types));
  }

  /** Create a union schema. */
  public static Schema createFixed(String name, String doc, String space,
      int size) {
    return new FixedSchema(new Name(name, space), doc, size);
  }

  /** Return the type of this schema. */
  @Nonnull
  public Type getType() { return type; }

  /** Return the logical type annotation for this schema */
  public LogicalType getLogicalType() { return logicalType; }

  /** Set the logical type annotation for this schema */
  public void setLogicalType(LogicalType<?> logicalType) {
    if (this.logicalType != null) {
      throw new IllegalArgumentException(
          "Cannot replace existing logical type");
    }
    this.logicalType = logicalType;
    for (Map.Entry<String, Object> prop : logicalType.getProperties().entrySet()) {
      this.addProp(prop.getKey(), prop.getValue());
    }
    this.hashCode = NO_HASHCODE;
  }

  protected boolean equalLogicalTypes(@Nullable Schema other) {
    if (logicalType == null) {
      return other.logicalType == null;
    }
    return logicalType.equals(other.logicalType);
  }

  /**
   * If this is a record, returns the Field with the
   * given name <tt>fieldName</tt>. If there is no field by that name, a
   * <tt>null</tt> is returned.
   */
  public Field getField(String fieldname) {
    throw new AvroRuntimeException("Not a record: "+this);
  }

  /**
   * If this is a record, returns the fields in it. The returned
   * list is in the order of their positions.
   */
  public List<Field> getFields() {
    throw new AvroRuntimeException("Not a record: "+this);
  }

  /**
   * If this is a record, set its fields. The fields can be set
   * only once in a schema.
   */
  public void setFields(List<Field> fields) {
    throw new AvroRuntimeException("Not a record: "+this);
  }

  /** If this is an enum, return its symbols. */
  public List<String> getEnumSymbols() {
    throw new AvroRuntimeException("Not an enum: "+this);
  }

  @Nullable
  public String getEnumDefault() {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  public String getEnumStringSymbol(final String symbol) {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  public  Map<String, Set<String>> getEnumSymbolAliases()  {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  public List<String> getEnumStringSymbols() {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  /** If this is an enum, return a symbol's ordinal value. */
  public int getEnumOrdinal(String symbol) {
    throw new AvroRuntimeException("Not an enum: "+this);
  }

  /** If this is an enum, returns true if it contains given symbol. */
  public boolean hasEnumSymbol(String symbol) {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  public boolean hasEnumSymbolOrAlias(String symbol) {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  public int getEnumSymbolOrAliasOrdinal(String soa) {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  /** If this is a record, enum or fixed, returns its name, otherwise the name
   * of the primitive type. */
  public String getName() { return type.name; }

  /** If this is a record, enum, or fixed, returns its docstring,
   * if available.  Otherwise, returns null. */
  @Nullable
  public String getDoc() {
    return null;
  }

  /** If this is a record, enum or fixed, returns its namespace, if any. */
  public String getNamespace() {
    throw new AvroRuntimeException("Not a named type: "+this);
  }

  /** If this is a record, enum or fixed, returns its namespace-qualified name,
   * otherwise returns the name of the primitive type. */
  public String getFullName() {
    return getName();
  }

  /** If this is a record, enum or fixed, add an alias. */
  public void addAlias(String alias) {
    throw new AvroRuntimeException("Not a named type: "+this);
  }

  /** If this is a record, enum or fixed, add an alias. */
  public void addAlias(String alias, String space) {
    throw new AvroRuntimeException("Not a named type: "+this);
  }

  /** If this is a record, enum or fixed, return its aliases, if any. */
  public Set<String> getAliases() {
    throw new AvroRuntimeException("Not a named type: "+this);
  }

  /** Returns true if this record is an error type. */
  public boolean isError() {
    throw new AvroRuntimeException("Not a record: "+this);
  }

  /** If this is an array, returns its element type. */
  public Schema getElementType() {
    throw new AvroRuntimeException("Not an array: "+this);
  }

  /** If this is a map, returns its value type. */
  public Schema getValueType() {
    throw new AvroRuntimeException("Not a map: "+this);
  }

  /** If this is a union, returns its types. */
  public List<Schema> getTypes() {
    throw new AvroRuntimeException("Not a union: "+this);
  }

  /** If this is a union, return the branch with the provided full name. */
  public Integer getIndexNamed(String name) {
    throw new AvroRuntimeException("Not a union: "+this);
  }

  /** If this is fixed, returns its size. */
  public int getFixedSize() {
    throw new AvroRuntimeException("Not fixed: "+this);
  }

  /** Render this as <a href="http://json.org/">JSON</a>.*/
  @Override
  public String toString() { return toString(false); }

  /** Render this as <a href="http://json.org/">JSON</a>.
   * @param pretty if true, pretty-print JSON.
   */
  public String toString(boolean pretty) {
    try {
      StringWriter writer = new StringWriter();
      JsonGenerator gen = FACTORY.createJsonGenerator(writer);
      if (pretty) gen.useDefaultPrettyPrinter();
      toJson(new Names(), gen);
      gen.flush();
      return writer.toString();
    } catch (IOException e) {
      throw new AvroRuntimeException(e);
    }
  }

  void toJson(Names names, JsonGenerator gen) throws IOException {
    if (props.size() == 0 && logicalType == null) { // no props defined
      gen.writeString(getName());                   // just write name
    } else {
      gen.writeStartObject();
      gen.writeStringField("type", getName());
      writeProps(gen);
      gen.writeEndObject();
    }
  }

  void fieldsToJson(Names names, JsonGenerator gen) throws IOException {
    throw new AvroRuntimeException("Not a record: "+this);
  }

  public boolean equals(Object o) {
    if (o == this) {
      return true;
    }
    if (o == null) {
      return false;
    }
    if (o.getClass() != this.getClass()) {
      return false;
    }
    Schema that = (Schema) o;
    if (!(this.type == that.type)) return false;
    return equalCachedHash(that)
        && equalLogicalTypes(that)
        && props.equals(that.props);
  }
  public final int hashCode() {
    if (hashCode == NO_HASHCODE) {
      hashCode = computeHash();
    }
    return hashCode;
  }

  int computeHash() {
    return (logicalType != null ?  23 * logicalType.hashCode() : getType().hashCode());
  }

  final boolean equalCachedHash(Schema other) {
    return (hashCode == other.hashCode)
           || (hashCode == NO_HASHCODE)
           || (other.hashCode == NO_HASHCODE);
  }

  private static final Set<String> FIELD_RESERVED = new HashSet<String>();
  static {
    Collections.addAll(FIELD_RESERVED,
                       "default","doc","name","order","type","aliases");
  }

  /** A field within a record. */
  public static class Field extends JsonProperties {

    /** How values of this field should be ordered when sorting records. */
    public enum Order {
      ASCENDING, DESCENDING, IGNORE;
      private String name;
      private Order() { this.name = this.name().toLowerCase(Locale.ENGLISH); }
    };

    private final String name;    // name of the field.
    private int position = -1;
    private final Schema schema;
    private final String doc;
    private final JsonNode defaultValue;
    private final Object defaultVal;
    private final Order order;
    private Set<String> aliases;

    /** @deprecated use {@link #Field(String, Schema, String, Object)} */
    @Deprecated
    public Field(String name, Schema schema, String doc,
        JsonNode defaultValue) {
      this(name, schema, doc, defaultValue, JacksonUtils.toObject(defaultValue, schema), true, Order.ASCENDING);
    }

    @Deprecated
    public Field(String name, Schema schema, String doc,
        JsonNode defaultValue, Order order) {
      this(name, schema, doc, defaultValue, JacksonUtils.toObject(defaultValue, schema), true, order);
    }

    @Deprecated
    public Field(String name, Schema schema, String doc,
        JsonNode defaultValue, boolean validateDefault, Order order) {
      this(name, schema, doc, defaultValue, JacksonUtils.toObject(defaultValue, schema), validateDefault, order);
    }

    /** @deprecated use {@link #Field(String, Schema, String, Object, Order)} */
    @Deprecated
    public Field(String name, Schema schema, String doc,
        JsonNode defaultValue, Object defaultVal, boolean validateDefault, Order order) {
      super(FIELD_RESERVED);
      this.name = validateName(name);
      this.schema = schema;
      this.doc = doc;
      this.defaultValue = validateDefault ? validateDefault(name, schema, defaultValue) : defaultValue;
      this.defaultVal = defaultVal;
      this.order = order;
    }
    /**
     * @param defaultValue the default value for this field specified using the mapping
     * null means no default value.
     * JsonProperties.NULL represents the null value
     *  in {@link JsonProperties}
     */
    public Field(String name, Schema schema, String doc,
        Object defaultValue) {
      this(name, schema, doc, defaultValue, Order.ASCENDING);
    }
    /**
     * @param defaultValue the default value for this field specified using the mapping
     *  in {@link JsonProperties}
     */
    public Field(String name, Schema schema, String doc,
        Object defaultValue, Order order) {
      this(name, schema, doc, JacksonUtils.toJsonNode(defaultValue), defaultValue, true, order);
    }
    public String name() { return name; };
    /** The position of this field within the record. */
    public int pos() { return position; }
    /** This field's {@link Schema}. */
    public Schema schema() { return schema; }
    /** Field's documentation within the record, if set. May return null. */
    public String doc() { return doc; }
    /** @deprecated use {@link #defaultVal() } */
    @Deprecated public JsonNode defaultValue() { return defaultValue; }
    /**
     * @return the default value for this field specified using the mapping
     *  in {@link JsonProperties}
     */
    public Object defaultVal() { return defaultVal; }
    public Order order() { return order; }
    @Deprecated public Map<String,String> props() { return getProps(); }
    public void addAlias(String alias) {
      if (aliases == null)
        this.aliases = new LinkedHashSet<String>();
      aliases.add(alias);
    }
    /** Return the defined aliases as an unmodifieable Set. */
    public Set<String> aliases() {
      if (aliases == null)
        return Collections.emptySet();
      return Collections.unmodifiableSet(aliases);
    }
    public boolean equals(Object other) {
      if (other == this) return true;
      if (!(other instanceof Field)) return false;
      Field that = (Field) other;
      return (name.equals(that.name)) &&
        (schema.equals(that.schema)) &&
        defaultValueEquals(that.defaultValue) &&
        (order == that.order) &&
        props.equals(that.props);
    }
    public int hashCode() { return name.hashCode() + 7 * schema.hashCode(); }

    private boolean defaultValueEquals(JsonNode thatDefaultValue) {
      if (defaultValue == null)
        return thatDefaultValue == null;
      if (thatDefaultValue == null)
        return false;
      if (Double.isNaN(defaultValue.getDoubleValue()))
        return Double.isNaN(thatDefaultValue.getDoubleValue());
      return defaultValue.equals(thatDefaultValue);
    }

    @Override
    public String toString() {
      return name + " type:" + schema.type + " pos:" + position;
    }
  }

  static class Name {
    private final String name;
    private final String space;
    private final String full;
    public Name(String name, String space) {
      if (name == null) {                         // anonymous
        this.name = this.space = this.full = null;
        return;
      }
      int lastDot = name.lastIndexOf('.');
      if (lastDot < 0) {                          // unqualified name
        this.name = validateName(name);
      } else {                                    // qualified name
        space = name.substring(0, lastDot);       // ordinal space from name
        this.name = validateName(name.substring(lastDot+1, name.length()));
      }
      if ("".equals(space))
        space = null;
      this.space = space;
      this.full = (this.space == null) ? this.name : this.space+"."+this.name;
    }
    public boolean equals(Object o) {
      if (o == this) return true;
      if (!(o instanceof Name)) return false;
      Name that = (Name)o;
      return full==null ? that.full==null : full.equals(that.full);
    }
    public int hashCode() {
      return full==null ? 0 : full.hashCode();
    }
    public String toString() { return full; }
    public void writeName(Names names, JsonGenerator gen) throws IOException {
      if (name != null) gen.writeStringField("name", name);
      if (space != null) {
        if (!space.equals(names.space()))
          gen.writeStringField("namespace", space);
      } else if (names.space() != null) {         // null within non-null<commit_msg>[fix] handle schema naming better to support $ in name-spaces.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.avro;

import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import org.apache.avro.util.internal.JacksonUtils;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.node.DoubleNode;

/** An abstract data type.
 * <p>A schema may be one of:
 * <ul>
 * <li>A <i>record</i>, mapping field names to field value data;
 * <li>An <i>enum</i>, containing one of a small set of symbols;
 * <li>An <i>array</i> of values, all of the same schema;
 * <li>A <i>map</i>, containing string/value pairs, of a declared schema;
 * <li>A <i>union</i> of other schemas;
 * <li>A <i>fixed</i> sized binary object;
 * <li>A unicode <i>string</i>;
 * <li>A sequence of <i>bytes</i>;
 * <li>A 32-bit signed <i>int</i>;
 * <li>A 64-bit signed <i>long</i>;
 * <li>A 32-bit IEEE single-<i>float</i>; or
 * <li>A 64-bit IEEE <i>double</i>-float; or
 * <li>A <i>boolean</i>; or
 * <li><i>null</i>.
 * </ul>
 *
 * A schema can be constructed using one of its static <tt>createXXX</tt>
 * methods, or more conveniently using {@link SchemaBuilder}. The schema objects are
 * <i>logically</i> immutable.
 * There are only three mutating methods - {@link #setFields(List)},
 * {@link #addProp(String, String)}, and {@link #setLogicalType(LogicalType)}.
 * The following restrictions apply on these methods.
 * <ul>
 * <li> {@link #setFields(List)}, can be called at most once. This method exists
 * in order to enable clients to build recursive schemas.
 * <li> {@link #addProp(String, String)} can be called with property names
 * that are not present already. It is not possible to change or delete an
 * existing property.
 * <li> {@link #setLogicalType(LogicalType)} can be called at most once.
 * </ul>
 */
public abstract class Schema extends JsonProperties implements Serializable {

  private static final long serialVersionUID = 1L;

  protected Object writeReplace() {
    SerializableSchema ss = new SerializableSchema();
    ss.schemaString = toString();
    return ss;
  }

  private static final class SerializableSchema implements Serializable {

    private static final long serialVersionUID = 1L;

    private String schemaString;

    private Object readResolve() {
      return new Schema.Parser().parse(schemaString);
    }
  }

  public static final JsonFactory FACTORY = new JsonFactory();
  public static final ObjectMapper MAPPER = new ObjectMapper(FACTORY);

  private static final int NO_HASHCODE = Integer.MIN_VALUE;

  public static final Schema fromString(final String schemaJson) {
    return new Schema.Parser().parse(schemaJson);
  }

  public static final Schema fromString(final String schemaJson, final String ... more) {
    return new Schema.Parser().parse(schemaJson, more);
  }

  static {
    FACTORY.enable(JsonParser.Feature.ALLOW_COMMENTS);
    FACTORY.setCodec(MAPPER);
  }

  /** The type of a schema. */
  public enum Type {
    RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES,
      INT, LONG, FLOAT, DOUBLE, BOOLEAN, NULL;
    private String name;
    private Type() { this.name = this.name().toLowerCase(Locale.ENGLISH); }
    public String getName() { return name; }
  };

  private final Type type;
  private LogicalType logicalType;

  Schema(Type type) {
    super(SCHEMA_RESERVED);
    this.type = type;
    this.logicalType = null;
  }

  /** Create a schema for a primitive type. */
  public static Schema create(Type type) {
    switch (type) {
    case STRING:  return new StringSchema();
    case BYTES:   return new BytesSchema();
    case INT:     return new IntSchema();
    case LONG:    return new LongSchema();
    case FLOAT:   return new FloatSchema();
    case DOUBLE:  return new DoubleSchema();
    case BOOLEAN: return new BooleanSchema();
    case NULL:    return new NullSchema();
    default: throw new AvroRuntimeException("Can't create a: "+type);
    }
  }

  private static final Set<String> SCHEMA_RESERVED = new HashSet<String>();
  static {
    Collections.addAll(SCHEMA_RESERVED,
                       "doc", "fields", "items", "name", "namespace",
                       "size", "symbols", "values", "type", "aliases");
  }
  private static final Set<String> ENUM_RESERVED = new HashSet<>();
  static {
    ENUM_RESERVED.add("default");
    ENUM_RESERVED.addAll(SCHEMA_RESERVED);
  }

  int hashCode = NO_HASHCODE;

  public Schema withProp(String name, String value) {
    addProp(name, value);
    return this;
  }

  public Schema withProp(String name, Object value) {
    addProp(name, value);
    return this;
  }

  /** Create an anonymous record schema. */
  public static Schema createRecord(List<Field> fields) {
    Schema result = createRecord(null, null, null, false);
    result.setFields(fields);
    return result;
  }

  /** Create a named record schema. */
  public static Schema createRecord(String name, String doc, String namespace,
                                    boolean isError) {
    return new RecordSchema(new Name(name, namespace), doc, isError);
  }

  /** Create a named record schema with fields already set. */
  public static Schema createRecord(String name, String doc, String namespace,
                                    boolean isError, List<Field> fields) {
    return new RecordSchema(new Name(name, namespace), doc, isError, fields);
  }

  /** Create an enum schema. */
  public static Schema createEnum(String name, String doc, String namespace,
                                  List<String> values) {
    return new EnumSchema(new Name(name, namespace), doc,
        new LockableArrayList<String>(values), null);
  }

  /** Create an enum schema. */
  public static Schema createEnum(String name, String doc, String namespace,
                                  List<String> values, final String defaultEnumVal) {
    return new EnumSchema(new Name(name, namespace), doc,
        new LockableArrayList<String>(values), defaultEnumVal);
  }

  /** Create an array schema. */
  public static Schema createArray(Schema elementType) {
    return new ArraySchema(elementType);
  }

  /** Create a map schema. */
  public static Schema createMap(Schema valueType) {
    return new MapSchema(valueType);
  }

  /** Create a union schema. */
  public static Schema createUnion(List<Schema> types) {
    return new UnionSchema(new LockableArrayList<Schema>(types));
  }

  /** Create a union schema. */
  public static Schema createUnion(Schema... types) {
    return createUnion(new LockableArrayList<Schema>(types));
  }

  /** Create a union schema. */
  public static Schema createFixed(String name, String doc, String space,
      int size) {
    return new FixedSchema(new Name(name, space), doc, size);
  }

  /** Return the type of this schema. */
  @Nonnull
  public Type getType() { return type; }

  /** Return the logical type annotation for this schema */
  public LogicalType getLogicalType() { return logicalType; }

  /** Set the logical type annotation for this schema */
  public void setLogicalType(LogicalType<?> logicalType) {
    if (this.logicalType != null) {
      throw new IllegalArgumentException(
          "Cannot replace existing logical type");
    }
    this.logicalType = logicalType;
    for (Map.Entry<String, Object> prop : logicalType.getProperties().entrySet()) {
      this.addProp(prop.getKey(), prop.getValue());
    }
    this.hashCode = NO_HASHCODE;
  }

  protected boolean equalLogicalTypes(@Nullable Schema other) {
    if (logicalType == null) {
      return other.logicalType == null;
    }
    return logicalType.equals(other.logicalType);
  }

  /**
   * If this is a record, returns the Field with the
   * given name <tt>fieldName</tt>. If there is no field by that name, a
   * <tt>null</tt> is returned.
   */
  public Field getField(String fieldname) {
    throw new AvroRuntimeException("Not a record: "+this);
  }

  /**
   * If this is a record, returns the fields in it. The returned
   * list is in the order of their positions.
   */
  public List<Field> getFields() {
    throw new AvroRuntimeException("Not a record: "+this);
  }

  /**
   * If this is a record, set its fields. The fields can be set
   * only once in a schema.
   */
  public void setFields(List<Field> fields) {
    throw new AvroRuntimeException("Not a record: "+this);
  }

  /** If this is an enum, return its symbols. */
  public List<String> getEnumSymbols() {
    throw new AvroRuntimeException("Not an enum: "+this);
  }

  @Nullable
  public String getEnumDefault() {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  public String getEnumStringSymbol(final String symbol) {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  public  Map<String, Set<String>> getEnumSymbolAliases()  {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  public List<String> getEnumStringSymbols() {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  /** If this is an enum, return a symbol's ordinal value. */
  public int getEnumOrdinal(String symbol) {
    throw new AvroRuntimeException("Not an enum: "+this);
  }

  /** If this is an enum, returns true if it contains given symbol. */
  public boolean hasEnumSymbol(String symbol) {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  public boolean hasEnumSymbolOrAlias(String symbol) {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  public int getEnumSymbolOrAliasOrdinal(String soa) {
    throw new AvroRuntimeException("Not an enum: " + this);
  }

  /** If this is a record, enum or fixed, returns its name, otherwise the name
   * of the primitive type. */
  public String getName() { return type.name; }

  /** If this is a record, enum, or fixed, returns its docstring,
   * if available.  Otherwise, returns null. */
  @Nullable
  public String getDoc() {
    return null;
  }

  /** If this is a record, enum or fixed, returns its namespace, if any. */
  public String getNamespace() {
    throw new AvroRuntimeException("Not a named type: "+this);
  }

  /** If this is a record, enum or fixed, returns its namespace-qualified name,
   * otherwise returns the name of the primitive type. */
  public String getFullName() {
    return getName();
  }

  /** If this is a record, enum or fixed, add an alias. */
  public void addAlias(String alias) {
    throw new AvroRuntimeException("Not a named type: "+this);
  }

  /** If this is a record, enum or fixed, add an alias. */
  public void addAlias(String alias, String space) {
    throw new AvroRuntimeException("Not a named type: "+this);
  }

  /** If this is a record, enum or fixed, return its aliases, if any. */
  public Set<String> getAliases() {
    throw new AvroRuntimeException("Not a named type: "+this);
  }

  /** Returns true if this record is an error type. */
  public boolean isError() {
    throw new AvroRuntimeException("Not a record: "+this);
  }

  /** If this is an array, returns its element type. */
  public Schema getElementType() {
    throw new AvroRuntimeException("Not an array: "+this);
  }

  /** If this is a map, returns its value type. */
  public Schema getValueType() {
    throw new AvroRuntimeException("Not a map: "+this);
  }

  /** If this is a union, returns its types. */
  public List<Schema> getTypes() {
    throw new AvroRuntimeException("Not a union: "+this);
  }

  /** If this is a union, return the branch with the provided full name. */
  public Integer getIndexNamed(String name) {
    throw new AvroRuntimeException("Not a union: "+this);
  }

  /** If this is fixed, returns its size. */
  public int getFixedSize() {
    throw new AvroRuntimeException("Not fixed: "+this);
  }

  /** Render this as <a href="http://json.org/">JSON</a>.*/
  @Override
  public String toString() { return toString(false); }

  /** Render this as <a href="http://json.org/">JSON</a>.
   * @param pretty if true, pretty-print JSON.
   */
  public String toString(boolean pretty) {
    try {
      StringWriter writer = new StringWriter();
      JsonGenerator gen = FACTORY.createJsonGenerator(writer);
      if (pretty) gen.useDefaultPrettyPrinter();
      toJson(new Names(), gen);
      gen.flush();
      return writer.toString();
    } catch (IOException e) {
      throw new AvroRuntimeException(e);
    }
  }

  void toJson(Names names, JsonGenerator gen) throws IOException {
    if (props.size() == 0 && logicalType == null) { // no props defined
      gen.writeString(getName());                   // just write name
    } else {
      gen.writeStartObject();
      gen.writeStringField("type", getName());
      writeProps(gen);
      gen.writeEndObject();
    }
  }

  void fieldsToJson(Names names, JsonGenerator gen) throws IOException {
    throw new AvroRuntimeException("Not a record: "+this);
  }

  public boolean equals(Object o) {
    if (o == this) {
      return true;
    }
    if (o == null) {
      return false;
    }
    if (o.getClass() != this.getClass()) {
      return false;
    }
    Schema that = (Schema) o;
    if (!(this.type == that.type)) return false;
    return equalCachedHash(that)
        && equalLogicalTypes(that)
        && props.equals(that.props);
  }
  public final int hashCode() {
    if (hashCode == NO_HASHCODE) {
      hashCode = computeHash();
    }
    return hashCode;
  }

  int computeHash() {
    return (logicalType != null ?  23 * logicalType.hashCode() : getType().hashCode());
  }

  final boolean equalCachedHash(Schema other) {
    return (hashCode == other.hashCode)
           || (hashCode == NO_HASHCODE)
           || (other.hashCode == NO_HASHCODE);
  }

  private static final Set<String> FIELD_RESERVED = new HashSet<String>();
  static {
    Collections.addAll(FIELD_RESERVED,
                       "default","doc","name","order","type","aliases");
  }

  /** A field within a record. */
  public static class Field extends JsonProperties {

    /** How values of this field should be ordered when sorting records. */
    public enum Order {
      ASCENDING, DESCENDING, IGNORE;
      private String name;
      private Order() { this.name = this.name().toLowerCase(Locale.ENGLISH); }
    };

    private final String name;    // name of the field.
    private int position = -1;
    private final Schema schema;
    private final String doc;
    private final JsonNode defaultValue;
    private final Object defaultVal;
    private final Order order;
    private Set<String> aliases;

    /** @deprecated use {@link #Field(String, Schema, String, Object)} */
    @Deprecated
    public Field(String name, Schema schema, String doc,
        JsonNode defaultValue) {
      this(name, schema, doc, defaultValue, JacksonUtils.toObject(defaultValue, schema), true, Order.ASCENDING);
    }

    @Deprecated
    public Field(String name, Schema schema, String doc,
        JsonNode defaultValue, Order order) {
      this(name, schema, doc, defaultValue, JacksonUtils.toObject(defaultValue, schema), true, order);
    }

    @Deprecated
    public Field(String name, Schema schema, String doc,
        JsonNode defaultValue, boolean validateDefault, Order order) {
      this(name, schema, doc, defaultValue, JacksonUtils.toObject(defaultValue, schema), validateDefault, order);
    }

    /** @deprecated use {@link #Field(String, Schema, String, Object, Order)} */
    @Deprecated
    public Field(String name, Schema schema, String doc,
        JsonNode defaultValue, Object defaultVal, boolean validateDefault, Order order) {
      super(FIELD_RESERVED);
      this.name = validateName(name);
      this.schema = schema;
      this.doc = doc;
      this.defaultValue = validateDefault ? validateDefault(name, schema, defaultValue) : defaultValue;
      this.defaultVal = defaultVal;
      this.order = order;
    }
    /**
     * @param defaultValue the default value for this field specified using the mapping
     * null means no default value.
     * JsonProperties.NULL represents the null value
     *  in {@link JsonProperties}
     */
    public Field(String name, Schema schema, String doc,
        Object defaultValue) {
      this(name, schema, doc, defaultValue, Order.ASCENDING);
    }
    /**
     * @param defaultValue the default value for this field specified using the mapping
     *  in {@link JsonProperties}
     */
    public Field(String name, Schema schema, String doc,
        Object defaultValue, Order order) {
      this(name, schema, doc, JacksonUtils.toJsonNode(defaultValue), defaultValue, true, order);
    }
    public String name() { return name; };
    /** The position of this field within the record. */
    public int pos() { return position; }
    /** This field's {@link Schema}. */
    public Schema schema() { return schema; }
    /** Field's documentation within the record, if set. May return null. */
    public String doc() { return doc; }
    /** @deprecated use {@link #defaultVal() } */
    @Deprecated public JsonNode defaultValue() { return defaultValue; }
    /**
     * @return the default value for this field specified using the mapping
     *  in {@link JsonProperties}
     */
    public Object defaultVal() { return defaultVal; }
    public Order order() { return order; }
    @Deprecated public Map<String,String> props() { return getProps(); }
    public void addAlias(String alias) {
      if (aliases == null)
        this.aliases = new LinkedHashSet<String>();
      aliases.add(alias);
    }
    /** Return the defined aliases as an unmodifieable Set. */
    public Set<String> aliases() {
      if (aliases == null)
        return Collections.emptySet();
      return Collections.unmodifiableSet(aliases);
    }
    public boolean equals(Object other) {
      if (other == this) return true;
      if (!(other instanceof Field)) return false;
      Field that = (Field) other;
      return (name.equals(that.name)) &&
        (schema.equals(that.schema)) &&
        defaultValueEquals(that.defaultValue) &&
        (order == that.order) &&
        props.equals(that.props);
    }
    public int hashCode() { return name.hashCode() + 7 * schema.hashCode(); }

    private boolean defaultValueEquals(JsonNode thatDefaultValue) {
      if (defaultValue == null)
        return thatDefaultValue == null;
      if (thatDefaultValue == null)
        return false;
      if (Double.isNaN(defaultValue.getDoubleValue()))
        return Double.isNaN(thatDefaultValue.getDoubleValue());
      return defaultValue.equals(thatDefaultValue);
    }

    @Override
    public String toString() {
      return name + " type:" + schema.type + " pos:" + position;
    }
  }

  static class Name {
    private final String name;
    private final String space;
    private final String full;
    /**
     * @param name if name is a fully qualified name, the space parameter value is being ignored.
     * @param space namespace in case the name is not fully qualified.
     */
    public Name(String name, String space) {
      if (name == null) {                         // anonymous
        this.name = this.space = this.full = null;
        return;
      }
      int lastDol = name.lastIndexOf('$');
      if (lastDol >= 0) {
        space = name.substring(0, lastDol + 1);       // ordinal space from name
        this.name = validateName(name.substring(lastDol + 1, name.length()));
      } else {
        int lastDot = name.lastIndexOf('.');
        if (lastDot < 0) {                          // unqualified name
          this.name = validateName(name);
        } else {                                    // qualified name
          space = name.substring(0, lastDot);       // ordinal space from name
          this.name = validateName(name.substring(lastDot + 1, name.length()));
        }
      }
      if ("".equals(space))
        space = null;
      this.space = space;
      this.full = (this.space == null) ? this.name
              : this.space.endsWith("$") ? this.space + this.name : this.space  + "." + this.name;
    }
    public boolean equals(Object o) {
      if (o == this) return true;
      if (!(o instanceof Name)) return false;
      Name that = (Name)o;
      return full==null ? that.full==null : full.equals(that.full);
    }
    public int hashCode() {
      return full==null ? 0 : full.hashCode();
    }
    public String toString() { return full; }
    public void writeName(Names names, JsonGenerator gen) throws IOException {
      if (name != null) gen.writeStringField("name", name);
      if (space != null) {
        if (!space.equals(names.space()))
          gen.writeStringField("namespace", space);
      } else if (names.space() != null) {         // null within non-null
{'fraction_non_alphanumeric': 0.11342392047786239, 'fraction_numerical': 0.001139938899274999, 'mean_word_length': 3.0833178179110035, 'pattern_counts': {'":': 0, '<': 187, '<?xml version=': 0, '>': 186, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '933218', 'n_tokens_mistral': 12576, 'n_tokens_neox': 12324, 'n_words': 4214}