<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.accumulo.test;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.Set;

import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
import org.apache.accumulo.server.security.AuditedSecurityOperation;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.apache.hadoop.io.Text;
import org.junit.Before;
import org.junit.Test;

/**
 * Tests that Accumulo is outputting audit messages as expected. Since this is using MiniAccumuloCluster, it could take a while if we test everything in
 * isolation. We test blocks of related operations, run the whole test in one MiniAccumulo instance, trying to clean up objects between each test. The
 * MiniAccumuloClusterTest sets up the log4j stuff differently to an installed instance, instead piping everything through stdout and writing to a set location
 * so we have to find the logs and grep the bits we need out.
 */
public class AuditMessageIT extends ConfigurableMacIT {

  private static final String AUDIT_USER_1 = "AuditUser1";
  private static final String AUDIT_USER_2 = "AuditUser2";
  private static final String PASSWORD = "password";
  private static final String OLD_TEST_TABLE_NAME = "apples";
  private static final String NEW_TEST_TABLE_NAME = "oranges";
  private static final String THIRD_TEST_TABLE_NAME = "pears";
  private static final Authorizations auths = new Authorizations("private", "public");

  // Must be static to survive Junit re-initialising the class every time.
  private static String lastAuditTimestamp;
  private Connector auditConnector;
  private Connector conn;

  private static ArrayList<String> findAuditMessage(ArrayList<String> input, String pattern) {
    ArrayList<String> result = new ArrayList<String>();
    for (String s : input) {
      if (s.matches(".*" + pattern + ".*"))
        result.add(s);
    }
    return result;
  }

  /**
   * Returns a List of Audit messages that have been grep'd out of the MiniAccumuloCluster output.
   * 
   * @param stepName
   *          A unique name for the test being executed, to identify the System.out messages.
   * @return A List of the Audit messages, sorted (so in chronological order).
   */
  private ArrayList<String> getAuditMessages(String stepName) throws IOException {

    for (MiniAccumuloClusterImpl.LogWriter lw : getCluster().getLogWriters()) {
      lw.flush();
    }

    // Grab the audit messages<commit_msg>ACCUMULO-2840 turn audit logging back on for the IT<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.accumulo.test;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.Set;

import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.accumulo.server.security.AuditedSecurityOperation;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.apache.hadoop.io.Text;
import org.junit.Before;
import org.junit.Test;

/**
 * Tests that Accumulo is outputting audit messages as expected. Since this is using MiniAccumuloCluster, it could take a while if we test everything in
 * isolation. We test blocks of related operations, run the whole test in one MiniAccumulo instance, trying to clean up objects between each test. The
 * MiniAccumuloClusterTest sets up the log4j stuff differently to an installed instance, instead piping everything through stdout and writing to a set location
 * so we have to find the logs and grep the bits we need out.
 */
public class AuditMessageIT extends ConfigurableMacIT {

  private static final String AUDIT_USER_1 = "AuditUser1";
  private static final String AUDIT_USER_2 = "AuditUser2";
  private static final String PASSWORD = "password";
  private static final String OLD_TEST_TABLE_NAME = "apples";
  private static final String NEW_TEST_TABLE_NAME = "oranges";
  private static final String THIRD_TEST_TABLE_NAME = "pears";
  private static final Authorizations auths = new Authorizations("private", "public");
  
  public void beforeClusterStart(MiniAccumuloConfigImpl cfg) throws Exception {
    new File(cfg.getConfDir(), "auditLog.xml").delete();
  }

  // Must be static to survive Junit re-initialising the class every time.
  private static String lastAuditTimestamp;
  private Connector auditConnector;
  private Connector conn;

  private static ArrayList<String> findAuditMessage(ArrayList<String> input, String pattern) {
    ArrayList<String> result = new ArrayList<String>();
    for (String s : input) {
      if (s.matches(".*" + pattern + ".*"))
        result.add(s);
    }
    return result;
  }

  /**
   * Returns a List of Audit messages that have been grep'd out of the MiniAccumuloCluster output.
   * 
   * @param stepName
   *          A unique name for the test being executed, to identify the System.out messages.
   * @return A List of the Audit messages, sorted (so in chronological order).
   */
  private ArrayList<String> getAuditMessages(String stepName) throws IOException {

    for (MiniAccumuloClusterImpl.LogWriter lw : getCluster().getLogWriters()) {
      lw.flush();
    }

    // Grab the audit messages
{'fraction_non_alphanumeric': 0.08311688311688312, 'fraction_numerical': 0.002380952380952381, 'mean_word_length': 4.655446756425949, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6815610', 'n_tokens_mistral': 2598, 'n_tokens_neox': 2535, 'n_words': 856}