<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.metadata.repository.graph;

import com.google.inject.Singleton;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.EdgeLabel;
import com.thinkaurelius.titan.core.Order;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.Mapping;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.discovery.SearchIndexer;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.repository.RepositoryException;
import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.IDataType;
import org.apache.hadoop.metadata.typesystem.types.StructType;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;

/**
 * Adds index for properties of a given type when its added before any instances are added.
 */
@Singleton
public class GraphBackedSearchIndexer implements SearchIndexer {

    private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);

    private final TitanGraph titanGraph;

    @Inject
    public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider)
        throws RepositoryException {

        this.titanGraph = graphProvider.get();

        initialize();
    }

    /**
     * Initializes the indices for the graph - create indices for Global Vertex Keys
     */
    private void initialize() {
        TitanManagement management = titanGraph.getManagementSystem();
        if (management.containsPropertyKey(Constants.GUID_PROPERTY_KEY)) {
            LOG.info("Global indexes already exist for graph");
            return;
        }

        LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
        management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
                .buildMixedIndex(Constants.BACKING_INDEX);
        management.buildIndex(Constants.EDGE_INDEX, Edge.class)
                .buildMixedIndex(Constants.BACKING_INDEX);
        management.commit();

        // create a composite index for guid as its unique
        createCompositeIndex(Constants.GUID_INDEX,
                Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE);

        // create a composite and mixed index for type since it can be combined with other keys
        createCompositeAndMixedIndex(Constants.ENTITY_TYPE_INDEX,
                Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE);

        // create a composite and mixed index for type since it can be combined with other keys
        createCompositeAndMixedIndex(Constants.SUPER_TYPES_INDEX,
                Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET);

        // create a composite and mixed index for traitNames since it can be combined with other
        // keys. Traits must be a set and not a list.
        createCompositeAndMixedIndex(Constants.TRAIT_NAMES_INDEX,
                Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET);

        // Index for full text search
        createFullTextIndex();

        //Indexes for graph backed type system store
        createTypeStoreIndexes();

        LOG.info("Index creation for global keys complete.");
    }

    private void createFullTextIndex() {
        TitanManagement management = titanGraph.getManagementSystem();
        PropertyKey fullText =
                management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY).dataType(String.class).make();

        management.buildIndex(Constants.FULLTEXT_INDEX, Vertex.class)
                .addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
                .buildMixedIndex(Constants.BACKING_INDEX);
        management.commit();
        LOG.info("Created mixed index for {}", Constants.ENTITY_TEXT_PROPERTY_KEY);
    }

    private void createTypeStoreIndexes() {
        //Create unique index on typeName
        createCompositeIndex(Constants.TYPENAME_PROPERTY_KEY, Constants.TYPENAME_PROPERTY_KEY, String.class,
                true, Cardinality.SINGLE);

        //create index on vertex type
        createCompositeIndex(Constants.VERTEX_TYPE_PROPERTY_KEY, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class,
                false, Cardinality.SINGLE);

    }

    /**
     * This is upon adding a new type to Store.
     *
     * @param typeName type name
     * @param dataType data type
     * @throws org.apache.hadoop.metadata.MetadataException
     */
    @Override
    public void onAdd(String typeName, IDataType dataType) throws MetadataException {
        LOG.info("Creating indexes for type name={}, definition={}", typeName, dataType);

        try {
            addIndexForType(dataType);
            LOG.info("Index creation for type {} complete", typeName);

        } catch (Throwable throwable) {
            // gets handle to currently open transaction
            titanGraph.getManagementSystem().rollback();
            LOG.error("Error creating index for type {}", dataType, throwable);
        }
    }

    private void addIndexForType(IDataType dataType) {
        switch (dataType.getTypeCategory()) {
            case PRIMITIVE:
            case ENUM:
            case ARRAY:
            case MAP:
                // do nothing since these are only attributes
                // and not types like structs, traits or classes
                break;
<commit_msg>Fixed exception handing while creating index for a fails<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.metadata.repository.graph;

import com.google.inject.Singleton;
import com.thinkaurelius.titan.core.Cardinality;
import com.thinkaurelius.titan.core.EdgeLabel;
import com.thinkaurelius.titan.core.Order;
import com.thinkaurelius.titan.core.PropertyKey;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.schema.Mapping;
import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
import com.thinkaurelius.titan.core.schema.TitanManagement;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.hadoop.metadata.MetadataException;
import org.apache.hadoop.metadata.discovery.SearchIndexer;
import org.apache.hadoop.metadata.repository.Constants;
import org.apache.hadoop.metadata.repository.RepositoryException;
import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
import org.apache.hadoop.metadata.typesystem.types.ClassType;
import org.apache.hadoop.metadata.typesystem.types.DataTypes;
import org.apache.hadoop.metadata.typesystem.types.IDataType;
import org.apache.hadoop.metadata.typesystem.types.StructType;
import org.apache.hadoop.metadata.typesystem.types.TraitType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;

/**
 * Adds index for properties of a given type when its added before any instances are added.
 */
@Singleton
public class GraphBackedSearchIndexer implements SearchIndexer {

    private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);

    private final TitanGraph titanGraph;

    @Inject
    public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider)
        throws RepositoryException {

        this.titanGraph = graphProvider.get();

        initialize();
    }

    /**
     * Initializes the indices for the graph - create indices for Global Vertex Keys
     */
    private void initialize() {
        TitanManagement management = titanGraph.getManagementSystem();
        if (management.containsPropertyKey(Constants.GUID_PROPERTY_KEY)) {
            LOG.info("Global indexes already exist for graph");
            return;
        }

        LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
        management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
                .buildMixedIndex(Constants.BACKING_INDEX);
        management.buildIndex(Constants.EDGE_INDEX, Edge.class)
                .buildMixedIndex(Constants.BACKING_INDEX);
        management.commit();

        // create a composite index for guid as its unique
        createCompositeIndex(Constants.GUID_INDEX,
                Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE);

        // create a composite and mixed index for type since it can be combined with other keys
        createCompositeAndMixedIndex(Constants.ENTITY_TYPE_INDEX,
                Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE);

        // create a composite and mixed index for type since it can be combined with other keys
        createCompositeAndMixedIndex(Constants.SUPER_TYPES_INDEX,
                Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET);

        // create a composite and mixed index for traitNames since it can be combined with other
        // keys. Traits must be a set and not a list.
        createCompositeAndMixedIndex(Constants.TRAIT_NAMES_INDEX,
                Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET);

        // Index for full text search
        createFullTextIndex();

        //Indexes for graph backed type system store
        createTypeStoreIndexes();

        LOG.info("Index creation for global keys complete.");
    }

    private void createFullTextIndex() {
        TitanManagement management = titanGraph.getManagementSystem();
        PropertyKey fullText =
                management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY).dataType(String.class).make();

        management.buildIndex(Constants.FULLTEXT_INDEX, Vertex.class)
                .addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
                .buildMixedIndex(Constants.BACKING_INDEX);
        management.commit();
        LOG.info("Created mixed index for {}", Constants.ENTITY_TEXT_PROPERTY_KEY);
    }

    private void createTypeStoreIndexes() {
        //Create unique index on typeName
        createCompositeIndex(Constants.TYPENAME_PROPERTY_KEY, Constants.TYPENAME_PROPERTY_KEY, String.class,
                true, Cardinality.SINGLE);

        //create index on vertex type
        createCompositeIndex(Constants.VERTEX_TYPE_PROPERTY_KEY, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class,
                false, Cardinality.SINGLE);

    }

    /**
     * This is upon adding a new type to Store.
     *
     * @param typeName type name
     * @param dataType data type
     * @throws org.apache.hadoop.metadata.MetadataException
     */
    @Override
    public void onAdd(String typeName, IDataType dataType) throws MetadataException {
        LOG.info("Creating indexes for type name={}, definition={}", typeName, dataType);

        try {
            addIndexForType(dataType);
            LOG.info("Index creation for type {} complete", typeName);

        } catch (Throwable throwable) {
            LOG.error("Error creating index for type {}", dataType, throwable);
            throw new MetadataException("Error while creating index for type " + dataType, throwable);
        }
    }

    private void addIndexForType(IDataType dataType) {
        switch (dataType.getTypeCategory()) {
            case PRIMITIVE:
            case ENUM:
            case ARRAY:
            case MAP:
                // do nothing since these are only attributes
                // and not types like structs, traits or classes
                break;

{'fraction_non_alphanumeric': 0.08064050778995961, 'fraction_numerical': 0.0008655510675129833, 'mean_word_length': 3.246554364471669, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4235618', 'n_tokens_mistral': 3722, 'n_tokens_neox': 3646, 'n_words': 1005}