<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/

package org.apache.kylin.metadata.datatype;

import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum;

/**
 */
@SuppressWarnings("serial")
public class DataType implements Serializable {

    private static final LinkedHashSet<String> VALID_TYPES = new LinkedHashSet<String>();
    private static final LinkedHashSet<String> COMPLEX_TYPES = new LinkedHashSet<String>();

    private static Pattern TYPE_PATTERN = null;
    private static Pattern COMPLEX_TYPE_PATTERN = null;
    private static final String TYPE_PATTEN_TAIL = "\\s*" //
            + "(?:" + "[(]" + "([\\d\\s,]+)" + "[)]" + ")?";

    public static synchronized void register(String... typeNames) {
        for (String typeName : typeNames) {
            VALID_TYPES.add(typeName);
        }

        TYPE_PATTERN = Pattern.compile(//
                "(" + StringUtils.join(VALID_TYPES, "|") + ")" //
                        + TYPE_PATTEN_TAIL,
                Pattern.CASE_INSENSITIVE);
    }

    public static synchronized void registerComplex(String... typeNames) {
        for (String typeName : typeNames) {
            COMPLEX_TYPES.add(typeName);
        }
        COMPLEX_TYPE_PATTERN = Pattern.compile(//
                "(" + StringUtils.join(COMPLEX_TYPES, "|") + ")" //
                        + TYPE_PATTEN_TAIL,
                Pattern.CASE_INSENSITIVE);
    }

    static {
        // standard sql types, ref: http://www.w3schools.com/sql/sql_datatypes_general.asp
        register("any", "char", "varchar", "string", //
                "boolean", "byte", "binary", //
                "int", "short", "long", "integer", "tinyint", "smallint", "bigint", //
                "int4", "long8", // for test only
                "float", "real", "double", "decimal", "numeric", //
                "date", "time", "datetime", "timestamp", //
                InnerDataTypeEnum.LITERAL.getDataType(), InnerDataTypeEnum.DERIVED.getDataType());

        registerComplex("array\\<.*\\>");
    }

    public static final Set<String> INTEGER_FAMILY = new HashSet<String>();
    public static final Set<String> NUMBER_FAMILY = new HashSet<String>();
    public static final Set<String> DATETIME_FAMILY = new HashSet<String>();
    public static final Set<String> STRING_FAMILY = new HashSet<String>();
    private static final Map<String, String> LEGACY_TYPE_MAP = new HashMap<String, String>();
    static {
        INTEGER_FAMILY.add("tinyint");
        INTEGER_FAMILY.add("smallint");
        INTEGER_FAMILY.add("integer");
        INTEGER_FAMILY.add("bigint");
        INTEGER_FAMILY.add("int4");
        INTEGER_FAMILY.add("long8");

        NUMBER_FAMILY.addAll(INTEGER_FAMILY);
        NUMBER_FAMILY.add("float");
        NUMBER_FAMILY.add("double");
        NUMBER_FAMILY.add("decimal");
        NUMBER_FAMILY.add("real");
        NUMBER_FAMILY.add("numeric");

        DATETIME_FAMILY.add("date");
        DATETIME_FAMILY.add("time");
        DATETIME_FAMILY.add("datetime");
        DATETIME_FAMILY.add("timestamp");

        STRING_FAMILY.add("varchar");
        STRING_FAMILY.add("char");

        LEGACY_TYPE_MAP.put("byte", "tinyint");
        LEGACY_TYPE_MAP.put("int", "integer");
        LEGACY_TYPE_MAP.put("short", "smallint");
        LEGACY_TYPE_MAP.put("long", "bigint");
        LEGACY_TYPE_MAP.put("string", "varchar");
        LEGACY_TYPE_MAP.put("hllc10", "hllc(10)");
        LEGACY_TYPE_MAP.put("hllc12", "hllc(12)");
        LEGACY_TYPE_MAP.put("hllc14", "hllc(14)");
        LEGACY_TYPE_MAP.put("hllc15", "hllc(15)");
        LEGACY_TYPE_MAP.put("hllc16", "hllc(16)");
    }

    private static final ConcurrentMap<DataType, DataType> CACHE = new ConcurrentHashMap<DataType, DataType>();

    public static final DataType ANY = DataType.getType("any");

    static {
        //to ensure the MeasureTypeFactory class has initialized
        MeasureTypeFactory.getUDAFs();
    }

    public static boolean isComplexType(DataType type) {
        Matcher m = COMPLEX_TYPE_PATTERN.matcher(type.getName());
        return m.matches();
    }

    public static DataType getType(String type) {
        if (type == null)
            return null;

        DataType dataType = new DataType(type);
        DataType cached = CACHE.get(dataType);
        if (cached == null) {
            CACHE.put(dataType, dataType);
            cached = dataType;
        }
        return cached;
    }

    public static boolean isNumberFamily(String name) {
        return NUMBER_FAMILY.contains(name);
    }

    // ============================================================================

    private String name;
    private int precision;
    private int scale;
    private transient DataTypeOrder order;

    public DataType(String name, int precision, int scale) {
        this.name = name;
        this.precision = precision;<commit_msg>KYLIN-3505 Fix wrong usage of cache in DataType<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/

package org.apache.kylin.metadata.datatype;

import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum;

/**
 */
@SuppressWarnings("serial")
public class DataType implements Serializable {

    private static final LinkedHashSet<String> VALID_TYPES = new LinkedHashSet<String>();
    private static final LinkedHashSet<String> COMPLEX_TYPES = new LinkedHashSet<String>();

    private static Pattern TYPE_PATTERN = null;
    private static Pattern COMPLEX_TYPE_PATTERN = null;
    private static final String TYPE_PATTEN_TAIL = "\\s*" //
            + "(?:" + "[(]" + "([\\d\\s,]+)" + "[)]" + ")?";

    public static synchronized void register(String... typeNames) {
        for (String typeName : typeNames) {
            VALID_TYPES.add(typeName);
        }

        TYPE_PATTERN = Pattern.compile(//
                "(" + StringUtils.join(VALID_TYPES, "|") + ")" //
                        + TYPE_PATTEN_TAIL,
                Pattern.CASE_INSENSITIVE);
    }

    public static synchronized void registerComplex(String... typeNames) {
        for (String typeName : typeNames) {
            COMPLEX_TYPES.add(typeName);
        }
        COMPLEX_TYPE_PATTERN = Pattern.compile(//
                "(" + StringUtils.join(COMPLEX_TYPES, "|") + ")" //
                        + TYPE_PATTEN_TAIL,
                Pattern.CASE_INSENSITIVE);
    }

    static {
        // standard sql types, ref: http://www.w3schools.com/sql/sql_datatypes_general.asp
        register("any", "char", "varchar", "string", //
                "boolean", "byte", "binary", //
                "int", "short", "long", "integer", "tinyint", "smallint", "bigint", //
                "int4", "long8", // for test only
                "float", "real", "double", "decimal", "numeric", //
                "date", "time", "datetime", "timestamp", //
                InnerDataTypeEnum.LITERAL.getDataType(), InnerDataTypeEnum.DERIVED.getDataType());

        registerComplex("array\\<.*\\>");
    }

    public static final Set<String> INTEGER_FAMILY = new HashSet<String>();
    public static final Set<String> NUMBER_FAMILY = new HashSet<String>();
    public static final Set<String> DATETIME_FAMILY = new HashSet<String>();
    public static final Set<String> STRING_FAMILY = new HashSet<String>();
    private static final Map<String, String> LEGACY_TYPE_MAP = new HashMap<String, String>();
    static {
        INTEGER_FAMILY.add("tinyint");
        INTEGER_FAMILY.add("smallint");
        INTEGER_FAMILY.add("integer");
        INTEGER_FAMILY.add("bigint");
        INTEGER_FAMILY.add("int4");
        INTEGER_FAMILY.add("long8");

        NUMBER_FAMILY.addAll(INTEGER_FAMILY);
        NUMBER_FAMILY.add("float");
        NUMBER_FAMILY.add("double");
        NUMBER_FAMILY.add("decimal");
        NUMBER_FAMILY.add("real");
        NUMBER_FAMILY.add("numeric");

        DATETIME_FAMILY.add("date");
        DATETIME_FAMILY.add("time");
        DATETIME_FAMILY.add("datetime");
        DATETIME_FAMILY.add("timestamp");

        STRING_FAMILY.add("varchar");
        STRING_FAMILY.add("char");

        LEGACY_TYPE_MAP.put("byte", "tinyint");
        LEGACY_TYPE_MAP.put("int", "integer");
        LEGACY_TYPE_MAP.put("short", "smallint");
        LEGACY_TYPE_MAP.put("long", "bigint");
        LEGACY_TYPE_MAP.put("string", "varchar");
        LEGACY_TYPE_MAP.put("hllc10", "hllc(10)");
        LEGACY_TYPE_MAP.put("hllc12", "hllc(12)");
        LEGACY_TYPE_MAP.put("hllc14", "hllc(14)");
        LEGACY_TYPE_MAP.put("hllc15", "hllc(15)");
        LEGACY_TYPE_MAP.put("hllc16", "hllc(16)");
    }

    private static final ConcurrentMap<String, DataType> CACHE = new ConcurrentHashMap<String, DataType>();

    public static final DataType ANY = DataType.getType("any");

    static {
        //to ensure the MeasureTypeFactory class has initialized
        MeasureTypeFactory.getUDAFs();
    }

    public static boolean isComplexType(DataType type) {
        Matcher m = COMPLEX_TYPE_PATTERN.matcher(type.getName());
        return m.matches();
    }

    public static DataType getType(String type) {
        if (type == null)
            return null;

        DataType cached = CACHE.get(type);
        if (cached == null) {
            DataType dataType = new DataType(type);
            CACHE.put(type, dataType);
            cached = dataType;
        }
        return cached;
    }

    public static boolean isNumberFamily(String name) {
        return NUMBER_FAMILY.contains(name);
    }

    // ============================================================================

    private String name;
    private int precision;
    private int scale;
    private transient DataTypeOrder order;

    public DataType(String name, int precision, int scale) {
        this.name = name;
        this.precision = precision;
{'fraction_non_alphanumeric': 0.13422442772530815, 'fraction_numerical': 0.0049623819433328, 'mean_word_length': 2.975501113585746, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6746218', 'n_tokens_mistral': 3922, 'n_tokens_neox': 3719, 'n_words': 850}