<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * <p/>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p/>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.test;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.List;

import org.apache.commons.cli.CommandLine;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.IntegrationTestingUtility;
import org.apache.hadoop.hbase.IntegrationTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.ScannerCallable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.visibility.Authorizations;
import org.apache.hadoop.hbase.security.visibility.CellVisibility;
import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
import org.apache.hadoop.hbase.security.visibility.VisibilityController;
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.junit.experimental.categories.Category;

/**
 * A large test which loads a lot of data with cell visibility, and verifies the data. Test adds 2
 * users with different sets of visibility labels authenticated for them. Every row (so cells in
 * that) added with visibility expressions. In load step, 200 map tasks are launched, which in turn
 * write loadmapper.num_to_write (default 100K) rows to an hbase table. Rows are written in blocks,
 * for a total of 100 blocks.
 * 
 * Verify step scans the table as both users with Authorizations. This step asserts that user can
 * see only those rows (and so cells) with visibility for which they have label auth.
 * 
 * This class can be run as a unit test, as an integration test, or from the command line.
 * 
 * Originally taken from Apache Bigtop.
 */
@Category(IntegrationTests.class)
public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationTestLoadAndVerify {
  private static final char NOT = '!';
  private static final char OR = '|';
  private static final char AND = '&';
  private static final String TEST_NAME = "IntegrationTestCellVisibilityLoadAndVerify";
  private static final String CONFIDENTIAL = "confidential";
  private static final String TOPSECRET = "topsecret";
  private static final String SECRET = "secret";
  private static final String PUBLIC = "public";
  private static final String PRIVATE = "private";
  private static final String[] LABELS = { CONFIDENTIAL, TOPSECRET, SECRET, PRIVATE, PUBLIC };
  private static final String[] VISIBILITY_EXPS = { CONFIDENTIAL + AND + TOPSECRET + AND + PRIVATE,
      CONFIDENTIAL + OR + TOPSECRET, PUBLIC,
      '(' + SECRET + OR + PRIVATE + ')' + AND + NOT + CONFIDENTIAL };
  private static final int VISIBILITY_EXPS_COUNT = VISIBILITY_EXPS.length;
  private static final byte[] TEST_FAMILY = Bytes.toBytes("f1");
  private static final byte[] TEST_QUALIFIER = Bytes.toBytes("q1");
  private static final String NUM_TO_WRITE_KEY = "loadmapper.num_to_write";
  private static final long NUM_TO_WRITE_DEFAULT = 100 * 1000;
  private static final int SCANNER_CACHING = 500;

  private long numRowsLoadedWithExp1, numRowsLoadedWithExp2, numRowsLoadWithExp3,
      numRowsLoadWithExp4;
  private long numRowsReadWithExp1, numRowsReadWithExp2, numRowsReadWithExp3, numRowsReadWithExp4;

  private static User ADMIN;
  private static User NORMAL_USER;

  private enum Counters {
    ROWS_VIS_EXP_1, ROWS_VIS_EXP_2, ROWS_VIS_EXP_3, ROWS_VIS_EXP_4;
  }

  @Override
  public void setUpCluster() throws Exception {
    util = getTestingUtil(null);
    Configuration conf = util.getConfiguration();
    conf.setInt(HFile.FORMAT_VERSION_KEY, 3);
    conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName());
    conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName());
    conf.set("hbase.superuser", "admin," + User.getCurrent().getName());
    super.setUpCluster();
    ADMIN = User.createUserForTesting(conf, "admin", new String[] { "supergroup" });
    NORMAL_USER = User.createUserForTesting(conf, "user1", new String[] {});
    addLabelsAndAuths();
  }

  private void addLabelsAndAuths() throws Exception {
    try {
      VisibilityClient.addLabels(util.getConfiguration(), LABELS);
      VisibilityClient.setAuths(util.getConfiguration(), LABELS, User.getCurrent().getName());
      VisibilityClient.setAuths(util.getConfiguration(), new String[] { CONFIDENTIAL, TOPSECRET,
          SECRET, PRIVATE }, ADMIN.getName());
      VisibilityClient.setAuths(util.getConfiguration(), new String[] { PUBLIC },
          NORMAL_USER.getName());
    } catch (Throwable t) {
      throw new IOException(t);
    }
  }<commit_msg>HBASE-10700 IntegrationTestWithCellVisibilityLoadAndVerify should allow current user to be the admin<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * <p/>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p/>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.test;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.List;

import org.apache.commons.cli.CommandLine;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.IntegrationTestingUtility;
import org.apache.hadoop.hbase.IntegrationTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.ScannerCallable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.visibility.Authorizations;
import org.apache.hadoop.hbase.security.visibility.CellVisibility;
import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
import org.apache.hadoop.hbase.security.visibility.VisibilityController;
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.junit.experimental.categories.Category;

/**
 * A large test which loads a lot of data with cell visibility, and verifies the data. Test adds 2
 * users with different sets of visibility labels authenticated for them. Every row (so cells in
 * that) added with visibility expressions. In load step, 200 map tasks are launched, which in turn
 * write loadmapper.num_to_write (default 100K) rows to an hbase table. Rows are written in blocks,
 * for a total of 100 blocks.
 * 
 * Verify step scans the table as both users with Authorizations. This step asserts that user can
 * see only those rows (and so cells) with visibility for which they have label auth.
 * 
 * This class can be run as a unit test, as an integration test, or from the command line.
 * 
 * Originally taken from Apache Bigtop.
 */
@Category(IntegrationTests.class)
public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationTestLoadAndVerify {
  private static final char NOT = '!';
  private static final char OR = '|';
  private static final char AND = '&';
  private static final String TEST_NAME = "IntegrationTestCellVisibilityLoadAndVerify";
  private static final String CONFIDENTIAL = "confidential";
  private static final String TOPSECRET = "topsecret";
  private static final String SECRET = "secret";
  private static final String PUBLIC = "public";
  private static final String PRIVATE = "private";
  private static final String[] LABELS = { CONFIDENTIAL, TOPSECRET, SECRET, PRIVATE, PUBLIC };
  private static final String[] VISIBILITY_EXPS = { CONFIDENTIAL + AND + TOPSECRET + AND + PRIVATE,
      CONFIDENTIAL + OR + TOPSECRET, PUBLIC,
      '(' + SECRET + OR + PRIVATE + ')' + AND + NOT + CONFIDENTIAL };
  private static final int VISIBILITY_EXPS_COUNT = VISIBILITY_EXPS.length;
  private static final byte[] TEST_FAMILY = Bytes.toBytes("f1");
  private static final byte[] TEST_QUALIFIER = Bytes.toBytes("q1");
  private static final String NUM_TO_WRITE_KEY = "loadmapper.num_to_write";
  private static final long NUM_TO_WRITE_DEFAULT = 100 * 1000;
  private static final int SCANNER_CACHING = 500;

  private long numRowsLoadedWithExp1, numRowsLoadedWithExp2, numRowsLoadWithExp3,
      numRowsLoadWithExp4;
  private long numRowsReadWithExp1, numRowsReadWithExp2, numRowsReadWithExp3, numRowsReadWithExp4;

  private static User ADMIN;
  private static User NORMAL_USER;

  private enum Counters {
    ROWS_VIS_EXP_1, ROWS_VIS_EXP_2, ROWS_VIS_EXP_3, ROWS_VIS_EXP_4;
  }

  @Override
  public void setUpCluster() throws Exception {
    util = getTestingUtil(null);
    Configuration conf = util.getConfiguration();
    conf.setInt(HFile.FORMAT_VERSION_KEY, 3);
    conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName());
    conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName());
    String adminName = User.getCurrent().getName();
    conf.set("hbase.superuser", adminName);
    super.setUpCluster();
    ADMIN = User.createUserForTesting(conf, adminName, new String[] { "supergroup" });
    NORMAL_USER = User.createUserForTesting(conf, "user1", new String[] {});
    addLabelsAndAuths();
  }

  private void addLabelsAndAuths() throws Exception {
    try {
      VisibilityClient.addLabels(util.getConfiguration(), LABELS);
      VisibilityClient.setAuths(util.getConfiguration(), LABELS, User.getCurrent().getName());
      VisibilityClient.setAuths(util.getConfiguration(), new String[] { CONFIDENTIAL, TOPSECRET,
          SECRET, PRIVATE }, ADMIN.getName());
      VisibilityClient.setAuths(util.getConfiguration(), new String[] { PUBLIC },
          NORMAL_USER.getName());
    } catch (Throwable t) {
      throw new IOException(t);
    }
  }
{'fraction_non_alphanumeric': 0.09390356265356266, 'fraction_numerical': 0.006526412776412777, 'mean_word_length': 4.896333182435491, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6726068', 'n_tokens_mistral': 3872, 'n_tokens_neox': 3731, 'n_words': 1081}