<commit_before>/*
 * Copyright (c) 2012 - 2020 Splice Machine, Inc.
 *
 * This file is part of Splice Machine.
 * Splice Machine is free software: you can redistribute it and/or modify it under the terms of the
 * GNU Affero General Public License as published by the Free Software Foundation, either
 * version 3, or (at your option) any later version.
 * Splice Machine is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU Affero General Public License for more details.
 * You should have received a copy of the GNU Affero General Public License along with Splice Machine.
 * If not, see <http://www.gnu.org/licenses/>.
 */

package com.splicemachine.access.hbase;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.sql.SQLException;
import java.sql.SQLWarning;
import java.util.*;
import java.util.concurrent.TimeUnit;

import com.google.common.collect.Lists;
import com.google.protobuf.InvalidProtocolBufferException;
import com.splicemachine.access.api.ReplicationPeerDescription;
import com.splicemachine.access.configuration.HBaseConfiguration;
import com.splicemachine.db.iapi.error.StandardException;
import com.splicemachine.db.iapi.sql.Activation;
import com.splicemachine.db.iapi.sql.conn.LanguageConnectionContext;
import com.splicemachine.db.impl.jdbc.EmbedConnection;
import com.splicemachine.db.jdbc.InternalDriver;
import com.splicemachine.db.shared.common.reference.SQLState;
import com.splicemachine.primitives.Bytes;
import com.splicemachine.replication.HBaseReplicationPlatformUtil;
import com.splicemachine.si.constants.SIConstants;
import com.splicemachine.storage.ClientPartition;
import com.splicemachine.utils.SpliceLogUtils;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
import org.apache.hadoop.hbase.ipc.HBaseRpcController;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.security.access.AccessControlClient;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.security.access.UserPermission;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
import org.apache.hadoop.hbase.util.HBaseFsckRepair;
import org.apache.log4j.Logger;
import org.spark_project.guava.base.Function;
import org.apache.hadoop.hbase.*;
import org.spark_project.guava.collect.Collections2;

import com.splicemachine.access.api.PartitionAdmin;
import com.splicemachine.access.api.PartitionCreator;
import com.splicemachine.access.api.TableDescriptor;
import com.splicemachine.concurrent.Clock;
import com.splicemachine.si.impl.driver.SIDriver;
import com.splicemachine.storage.LazyPartitionServer;
import com.splicemachine.storage.Partition;
import com.splicemachine.storage.PartitionInfoCache;
import com.splicemachine.storage.PartitionServer;
import com.splicemachine.storage.RangedClientPartition;

import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor;
/**
 * @author <NAME>
 *         Date: 12/31/15
 */
public class HBasePartitionAdmin implements PartitionAdmin{

    private static final Logger LOG = Logger.getLogger(HBasePartitionAdmin.class);
    private final Admin admin;
    private final Clock timeKeeper;
    private final HBaseTableInfoFactory tableInfoFactory;
    private final PartitionInfoCache<TableName> partitionInfoCache;

    public HBasePartitionAdmin(Admin admin,
                               Clock timeKeeper,
                               HBaseTableInfoFactory tableInfoFactory,
                               PartitionInfoCache<TableName> partitionInfoCache
                             ){
        this.admin=admin;
        this.timeKeeper=timeKeeper;
        this.tableInfoFactory = tableInfoFactory;
        this.partitionInfoCache = partitionInfoCache;
    }

    @Override
    public PartitionCreator newPartition() throws IOException{
        HBaseConnectionFactory instance=HBaseConnectionFactory.getInstance(SIDriver.driver().getConfiguration());
        HColumnDescriptor dataFamily = instance.createDataFamily();
        return new HPartitionCreator(tableInfoFactory,admin.getConnection(),timeKeeper,dataFamily,partitionInfoCache);
    }

    @Override
    public void deleteTable(String tableName) throws IOException{

        TableName tn = tableInfoFactory.getTableInfo(tableName);
        try{
            if (admin.isTableEnabled(tn)) {
                admin.disableTable(tn);
            }
            admin.deleteTable(tn);
        }catch(TableNotFoundException ignored){
            /*
             * If the table isn't there, then we probably have two concurrent Vacuums going on. In that
             * situation, we just ignore the error, since the delete already works.
             */
            SpliceLogUtils.warn(LOG, "Table %s not found. It may be deleted by another running vacuum.", tableName);
        }catch (Exception e) {
            SpliceLogUtils.warn(LOG, "Deleting table %s failed", tableName);
            if (!admin.isTableEnabled(tn)){
                SpliceLogUtils.warn(LOG, "Re-enable table %s", tableName);
                admin.enableTable(tn);
            }
            throw e;
        }
    }

    /**
     * Split a table. Asynchronous operation.
     *
     * @param tableName table to split
     * @param splitPoints the explicit positions to split on. If null, HBase chooses.
     * @throws IOException if a remote or network exception occurs
     */
    @Override
    public void splitTable(String tableName,byte[]... splitPoints) throws IOException{
        TableName tableInfo=tableInfoFactory.getTableInfo(tableName);

        try {
            if (splitPoints != null && splitPoints.length > 0) {
                for (byte[] splitPoint : splitPoints) {
                    retriableSplit(tableInfo, splitPoint);
                }
            } else {
                retriableSplit(tableInfo, null);
            }
        }
        catch (Exception e) {
            throw new IOException(e);
        }
    }

    private void retriableSplit(TableName tableInfo, byte[] splitPoint) throws IOException, SQLException{

        List<HRegionInfo> regions = admin.getTableRegions(tableInfo);
        int size1 = regions.size();
        boolean done = false;
        while (!done) {
            try {
                if (splitPoint != null) {
                    admin.split(tableInfo, splitPoint);
                }
                else {
                    admin.split(tableInfo);
                }
                done = true;
            } catch (NotServingRegionException | ConnectionClosingException e) {
                
                done = false;
                try {
                    timeKeeper.sleep(100, TimeUnit.MILLISECONDS);
                } catch (InterruptedException ie) {
                    throw new InterruptedIOException();
                }
            } catch (IOException e) {
                String errMsg = e.getMessage();
                if (errMsg.compareTo("should not give a splitkey which equals to startkey!") == 0 ||
                        errMsg.contains("NOT splittable")) {
                    SpliceLogUtils.warn(LOG, "%s : %s", errMsg, Bytes.toStringBinary(splitPoint));
                    SQLWarning warning =
                            StandardException.newWarning(SQLState.SPLITKEY_EQUALS_STARTKEY, Bytes.toStringBinary(splitPoint));
                    try {
                        InternalDriver internalDriver = InternalDriver.activeDriver();
                        if (internalDriver != null) {
                            EmbedConnection connection = (EmbedConnection)internalDriver.connect("jdbc:default:connection", null);
                            if (connection != null) {
                                LanguageConnectionContext lcc = connection.getLanguageConnection();
                                if (lcc != null) {
                                    Activation lastActivation = lcc.getLastActivation();
                                    lastActivation.addWarning(warning);
                                }
                            }
                        }
                    } catch (Exception ex) {
                        // ignore any error when adding a warning to activation, because the warning is written
                        // to log.
                    }
                    return;
                }
                else
                    throw e;
            }
        }

        // Wait until split takes effect or timeout.
        done = false;
        int wait = 0;
        while (!done && wait < 100) {
            regions = admin.getTableRegions(tableInfo);
            int size2 = regions.size();<commit_msg>DB-8930: retry split until the region is online (#3200)<commit_after>/*
 * Copyright (c) 2012 - 2020 Splice Machine, Inc.
 *
 * This file is part of Splice Machine.
 * Splice Machine is free software: you can redistribute it and/or modify it under the terms of the
 * GNU Affero General Public License as published by the Free Software Foundation, either
 * version 3, or (at your option) any later version.
 * Splice Machine is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU Affero General Public License for more details.
 * You should have received a copy of the GNU Affero General Public License along with Splice Machine.
 * If not, see <http://www.gnu.org/licenses/>.
 */

package com.splicemachine.access.hbase;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.sql.SQLException;
import java.sql.SQLWarning;
import java.util.*;
import java.util.concurrent.TimeUnit;

import com.google.common.collect.Lists;
import com.google.protobuf.InvalidProtocolBufferException;
import com.splicemachine.access.api.ReplicationPeerDescription;
import com.splicemachine.access.configuration.HBaseConfiguration;
import com.splicemachine.db.iapi.error.StandardException;
import com.splicemachine.db.iapi.sql.Activation;
import com.splicemachine.db.iapi.sql.conn.LanguageConnectionContext;
import com.splicemachine.db.impl.jdbc.EmbedConnection;
import com.splicemachine.db.jdbc.InternalDriver;
import com.splicemachine.db.shared.common.reference.SQLState;
import com.splicemachine.primitives.Bytes;
import com.splicemachine.replication.HBaseReplicationPlatformUtil;
import com.splicemachine.si.constants.SIConstants;
import com.splicemachine.storage.ClientPartition;
import com.splicemachine.utils.SpliceLogUtils;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
import org.apache.hadoop.hbase.ipc.HBaseRpcController;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.security.access.AccessControlClient;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.security.access.UserPermission;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
import org.apache.hadoop.hbase.util.HBaseFsckRepair;
import org.apache.log4j.Logger;
import org.spark_project.guava.base.Function;
import org.apache.hadoop.hbase.*;
import org.spark_project.guava.collect.Collections2;

import com.splicemachine.access.api.PartitionAdmin;
import com.splicemachine.access.api.PartitionCreator;
import com.splicemachine.access.api.TableDescriptor;
import com.splicemachine.concurrent.Clock;
import com.splicemachine.si.impl.driver.SIDriver;
import com.splicemachine.storage.LazyPartitionServer;
import com.splicemachine.storage.Partition;
import com.splicemachine.storage.PartitionInfoCache;
import com.splicemachine.storage.PartitionServer;
import com.splicemachine.storage.RangedClientPartition;

import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor;
/**
 * @author <NAME>
 *         Date: 12/31/15
 */
public class HBasePartitionAdmin implements PartitionAdmin{

    private static final Logger LOG = Logger.getLogger(HBasePartitionAdmin.class);
    private final Admin admin;
    private final Clock timeKeeper;
    private final HBaseTableInfoFactory tableInfoFactory;
    private final PartitionInfoCache<TableName> partitionInfoCache;

    public HBasePartitionAdmin(Admin admin,
                               Clock timeKeeper,
                               HBaseTableInfoFactory tableInfoFactory,
                               PartitionInfoCache<TableName> partitionInfoCache
                             ){
        this.admin=admin;
        this.timeKeeper=timeKeeper;
        this.tableInfoFactory = tableInfoFactory;
        this.partitionInfoCache = partitionInfoCache;
    }

    @Override
    public PartitionCreator newPartition() throws IOException{
        HBaseConnectionFactory instance=HBaseConnectionFactory.getInstance(SIDriver.driver().getConfiguration());
        HColumnDescriptor dataFamily = instance.createDataFamily();
        return new HPartitionCreator(tableInfoFactory,admin.getConnection(),timeKeeper,dataFamily,partitionInfoCache);
    }

    @Override
    public void deleteTable(String tableName) throws IOException{

        TableName tn = tableInfoFactory.getTableInfo(tableName);
        try{
            if (admin.isTableEnabled(tn)) {
                admin.disableTable(tn);
            }
            admin.deleteTable(tn);
        }catch(TableNotFoundException ignored){
            /*
             * If the table isn't there, then we probably have two concurrent Vacuums going on. In that
             * situation, we just ignore the error, since the delete already works.
             */
            SpliceLogUtils.warn(LOG, "Table %s not found. It may be deleted by another running vacuum.", tableName);
        }catch (Exception e) {
            SpliceLogUtils.warn(LOG, "Deleting table %s failed", tableName);
            if (!admin.isTableEnabled(tn)){
                SpliceLogUtils.warn(LOG, "Re-enable table %s", tableName);
                admin.enableTable(tn);
            }
            throw e;
        }
    }

    /**
     * Split a table. Asynchronous operation.
     *
     * @param tableName table to split
     * @param splitPoints the explicit positions to split on. If null, HBase chooses.
     * @throws IOException if a remote or network exception occurs
     */
    @Override
    public void splitTable(String tableName,byte[]... splitPoints) throws IOException{
        TableName tableInfo=tableInfoFactory.getTableInfo(tableName);

        try {
            if (splitPoints != null && splitPoints.length > 0) {
                for (byte[] splitPoint : splitPoints) {
                    retriableSplit(tableInfo, splitPoint);
                }
            } else {
                retriableSplit(tableInfo, null);
            }
        }
        catch (Exception e) {
            throw new IOException(e);
        }
    }

    private void retriableSplit(TableName tableInfo, byte[] splitPoint) throws IOException, SQLException{

        List<HRegionInfo> regions = admin.getTableRegions(tableInfo);
        int size1 = regions.size();
        boolean done = false;
        while (!done) {
            try {
                if (splitPoint != null) {
                    admin.split(tableInfo, splitPoint);
                }
                else {
                    admin.split(tableInfo);
                }
                done = true;
            } catch (NotServingRegionException | ConnectionClosingException e) {
                
                done = false;
                try {
                    timeKeeper.sleep(100, TimeUnit.MILLISECONDS);
                } catch (InterruptedException ie) {
                    throw new InterruptedIOException();
                }
            } catch (IOException e) {
                String errMsg = e.getMessage();
                if (errMsg.compareTo("should not give a splitkey which equals to startkey!") == 0 ||
                        errMsg.contains("NOT splittable")) {
                    SpliceLogUtils.warn(LOG, "%s : %s", errMsg, Bytes.toStringBinary(splitPoint));
                    SQLWarning warning =
                            StandardException.newWarning(SQLState.SPLITKEY_EQUALS_STARTKEY, Bytes.toStringBinary(splitPoint));
                    try {
                        InternalDriver internalDriver = InternalDriver.activeDriver();
                        if (internalDriver != null) {
                            EmbedConnection connection = (EmbedConnection)internalDriver.connect("jdbc:default:connection", null);
                            if (connection != null) {
                                LanguageConnectionContext lcc = connection.getLanguageConnection();
                                if (lcc != null) {
                                    Activation lastActivation = lcc.getLastActivation();
                                    lastActivation.addWarning(warning);
                                }
                            }
                        }
                    } catch (Exception ex) {
                        // ignore any error when adding a warning to activation, because the warning is written
                        // to log.
                    }
                    return;
                }
                else if (errMsg.contains("is not OPEN")) {
                    try {
                        timeKeeper.sleep(100, TimeUnit.MILLISECONDS);
                    } catch (InterruptedException ie) {
                        throw new InterruptedIOException();
                    }
                    SpliceLogUtils.warn(LOG, "Encounter an error: %s ", errMsg);
                }
                else
                    throw e;
            }
        }

        // Wait until split takes effect or timeout.
        done = false;
        int wait = 0;
        while (!done && wait < 100) {
            regions = admin.getTableRegions(tableInfo);
            int size2 = regions.size();
{'fraction_non_alphanumeric': 0.0827319040543409, 'fraction_numerical': 0.003555508384631713, 'mean_word_length': 2.3742166517457477, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4906780', 'n_tokens_mistral': 4933, 'n_tokens_neox': 4702, 'n_words': 1143}