<commit_before>/*
 * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
 * license agreements.  See the NOTICE file distributed with this work for
 * additional information regarding copyright ownership.  Crate licenses
 * this file to you under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 * However, if you have executed another commercial license agreement
 * with Crate these terms will supersede the license and you may use the
 * software solely pursuant to the terms of the relevant commercial agreement.
 */

package io.crate.analyze;

import com.google.common.collect.ImmutableMap;
import io.crate.blob.v2.BlobIndicesService;
import io.crate.metadata.settings.NumberOfReplicasSetting;
import io.crate.metadata.settings.Validators;
import io.crate.metadata.table.ColumnPolicies;
import io.crate.sql.tree.ColumnPolicy;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.mapper.MapperService;

import javax.annotation.concurrent.Immutable;
import javax.annotation.concurrent.ThreadSafe;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * Container for the supported settings that can be used in the `WITH` clause of `CREATE TABLE` statements
 * or which can be updated via `ALTER TABLE SET` statements.
 *
 */
@Immutable
@ThreadSafe
public class TableParameters {

    // all available table settings
    static final NumberOfReplicasSetting NUMBER_OF_REPLICAS = new NumberOfReplicasSetting();
    static final Setting<String> COLUMN_POLICY = new Setting<>(
        new Setting.SimpleKey(ColumnPolicies.ES_MAPPING_NAME),
        s -> ColumnPolicy.STRICT.lowerCaseName(),
        s -> ColumnPolicies.encodeMappingValue(ColumnPolicy.of(s)),
        o -> {
            if (ColumnPolicies.encodeMappingValue(ColumnPolicy.IGNORED).equals(o)) {
                throw new IllegalArgumentException("Invalid value for argument '" + ColumnPolicies.CRATE_NAME + "'");
            }
        },
        Setting.Property.IndexScope
    );

    // all available table mapping keys

    private static final List<Setting<?>> SUPPORTED_SETTINGS =
        List.of(
            NUMBER_OF_REPLICAS,
            IndexSettings.INDEX_REFRESH_INTERVAL_SETTING,
            IndexMetaData.INDEX_READ_ONLY_SETTING,
            IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING,
            IndexMetaData.INDEX_BLOCKS_READ_SETTING,
            IndexMetaData.INDEX_BLOCKS_WRITE_SETTING,
            IndexMetaData.INDEX_BLOCKS_METADATA_SETTING,
            IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING,
            EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING,
            IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,
            IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING,
            ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING,
            MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
            IndexSettings.INDEX_WARMER_ENABLED_SETTING,
            UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING,
            IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS,
            MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY,
            IndexSettings.MAX_NGRAM_DIFF_SETTING,
            IndexSettings.MAX_SHINGLE_DIFF_SETTING,
            IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING,
            IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING,
            IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING,
            IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING,
            EngineConfig.INDEX_CODEC_SETTING,
            IndexModule.INDEX_STORE_TYPE_SETTING,

            // this setting is needed for tests and is not documented. see ClusterDisruptionIT for usages.
            IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING
        );

    /**
     * Settings which are not included in table default settings
     */
    static final Set<Setting> SETTINGS_NOT_INCLUDED_IN_DEFAULT = Set.of(
        IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING,
        IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING
    );

    private static final Map<String, Setting<?>> SUPPORTED_SETTINGS_DEFAULT
        = SUPPORTED_SETTINGS
            .stream()
            .collect(ImmutableMap.toImmutableMap((s) -> stripDotSuffix(stripIndexPrefix(s.getKey())), s -> s));

    private static final Set<Setting<?>> EXCLUDED_SETTING_FOR_METADATA_IMPORT = Set.of(NUMBER_OF_REPLICAS);

    private static final Map<String, Setting<?>> SUPPORTED_SETTINGS_INCL_SHARDS
        = ImmutableMap.<String, Setting<?>>builder()
            .putAll(SUPPORTED_SETTINGS_DEFAULT)
            .put(stripIndexPrefix(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey()), IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING)
            .build();
<commit_msg>Exclude `node.store.allow_mmap` from table/partition defaults.<commit_after>/*
 * Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
 * license agreements.  See the NOTICE file distributed with this work for
 * additional information regarding copyright ownership.  Crate licenses
 * this file to you under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 * However, if you have executed another commercial license agreement
 * with Crate these terms will supersede the license and you may use the
 * software solely pursuant to the terms of the relevant commercial agreement.
 */

package io.crate.analyze;

import com.google.common.collect.ImmutableMap;
import io.crate.blob.v2.BlobIndicesService;
import io.crate.metadata.settings.NumberOfReplicasSetting;
import io.crate.metadata.settings.Validators;
import io.crate.metadata.table.ColumnPolicies;
import io.crate.sql.tree.ColumnPolicy;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.mapper.MapperService;

import javax.annotation.concurrent.Immutable;
import javax.annotation.concurrent.ThreadSafe;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * Container for the supported settings that can be used in the `WITH` clause of `CREATE TABLE` statements
 * or which can be updated via `ALTER TABLE SET` statements.
 *
 */
@Immutable
@ThreadSafe
public class TableParameters {

    // all available table settings
    static final NumberOfReplicasSetting NUMBER_OF_REPLICAS = new NumberOfReplicasSetting();
    static final Setting<String> COLUMN_POLICY = new Setting<>(
        new Setting.SimpleKey(ColumnPolicies.ES_MAPPING_NAME),
        s -> ColumnPolicy.STRICT.lowerCaseName(),
        s -> ColumnPolicies.encodeMappingValue(ColumnPolicy.of(s)),
        o -> {
            if (ColumnPolicies.encodeMappingValue(ColumnPolicy.IGNORED).equals(o)) {
                throw new IllegalArgumentException("Invalid value for argument '" + ColumnPolicies.CRATE_NAME + "'");
            }
        },
        Setting.Property.IndexScope
    );

    // all available table mapping keys

    private static final List<Setting<?>> SUPPORTED_SETTINGS =
        List.of(
            NUMBER_OF_REPLICAS,
            IndexSettings.INDEX_REFRESH_INTERVAL_SETTING,
            IndexMetaData.INDEX_READ_ONLY_SETTING,
            IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING,
            IndexMetaData.INDEX_BLOCKS_READ_SETTING,
            IndexMetaData.INDEX_BLOCKS_WRITE_SETTING,
            IndexMetaData.INDEX_BLOCKS_METADATA_SETTING,
            IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING,
            EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING,
            IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,
            IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING,
            ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING,
            MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
            IndexSettings.INDEX_WARMER_ENABLED_SETTING,
            UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING,
            IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS,
            MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY,
            IndexSettings.MAX_NGRAM_DIFF_SETTING,
            IndexSettings.MAX_SHINGLE_DIFF_SETTING,
            IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING,
            IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING,
            IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING,
            IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING,
            EngineConfig.INDEX_CODEC_SETTING,
            IndexModule.INDEX_STORE_TYPE_SETTING,

            // this setting is needed for tests and is not documented. see ClusterDisruptionIT for usages.
            IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING
        );

    /**
     * Settings which are not included in table default settings
     */
    static final Set<Setting> SETTINGS_NOT_INCLUDED_IN_DEFAULT = Set.of(
        IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING,
        IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING,
        IndexModule.NODE_STORE_ALLOW_MMAP
    );

    private static final Map<String, Setting<?>> SUPPORTED_SETTINGS_DEFAULT
        = SUPPORTED_SETTINGS
            .stream()
            .collect(ImmutableMap.toImmutableMap((s) -> stripDotSuffix(stripIndexPrefix(s.getKey())), s -> s));

    private static final Set<Setting<?>> EXCLUDED_SETTING_FOR_METADATA_IMPORT = Set.of(NUMBER_OF_REPLICAS);

    private static final Map<String, Setting<?>> SUPPORTED_SETTINGS_INCL_SHARDS
        = ImmutableMap.<String, Setting<?>>builder()
            .putAll(SUPPORTED_SETTINGS_DEFAULT)
            .put(stripIndexPrefix(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey()), IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING)
            .build();

{'fraction_non_alphanumeric': 0.07001321003963012, 'fraction_numerical': 0.0008256274768824307, 'mean_word_length': 3.934012219959267, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1213685', 'n_tokens_mistral': 3327, 'n_tokens_neox': 3460, 'n_words': 721}