<commit_before>// Copyright (C) 2011 Splunk Inc.
//
// Splunk Inc. licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.splunk.shuttl.archiver.endtoend;

import static java.util.Arrays.*;
import static org.testng.Assert.*;

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;

import com.splunk.Service;
import com.splunk.shuttl.ShuttlConstants;
import com.splunk.shuttl.archiver.archive.ArchiveConfiguration;
import com.splunk.shuttl.archiver.filesystem.ArchiveFileSystem;
import com.splunk.shuttl.archiver.filesystem.ArchiveFileSystemFactory;
import com.splunk.shuttl.archiver.filesystem.PathResolver;
import com.splunk.shuttl.archiver.model.Bucket;
import com.splunk.shuttl.archiver.model.LocalBucket;
import com.splunk.shuttl.server.mbeans.util.EndpointUtils;
import com.splunk.shuttl.testutil.TUtilsBucket;
import com.splunk.shuttl.testutil.TUtilsEnvironment;
import com.splunk.shuttl.testutil.TUtilsMBean;
import com.splunk.shuttl.testutil.TUtilsTestNG;

@Test(groups = { "cluster-test" })
public class ClusterReplicatedBucketArchivingTest {

	private String index;

	@Parameters(value = { "cluster.slave1.host", "cluster.slave1.port",
			"cluster.slave2.host", "cluster.slave2.port",
			"cluster.slave2.shuttl.port", "splunk.username", "splunk.password",
			"cluster.slave2.splunk.home" })
	public void test(String slave1Host, String slave1Port, String slave2Host,
			String slave2Port, String slave2ShuttlPort, String splunkUser,
			String splunkPass, final String splunkHome) {
		index = "shuttl";
		Service slave1 = getLoggedInService(slave1Host, slave1Port, splunkUser,
				splunkPass);
		Service slave2 = getLoggedInService(slave2Host, slave2Port, splunkUser,
				splunkPass);

		assertTrue(slave2.getIndexes().containsKey(index));
		String coldPathExpanded = slave2.getIndexes().get(index)
				.getColdPathExpanded();

		String slave1Guid = slave1.getInfo().getGuid();

		final LocalBucket rb = TUtilsBucket.createReplicatedBucket(index, new File(
				coldPathExpanded), slave1Guid);

		try {
			callSlave2ArchiveBucketEndpoint(index, rb.getDirectory()
					.getAbsolutePath(), slave2Host, slave2ShuttlPort);
		} catch (IOException e1) {
			TUtilsTestNG.failForException(
					"failed when calling archive bucket endpoint", e1);
		}

		String slave1SplunkHome = slave1.getSettings().getSplunkHome();
		final File slave1ShuttlConfDir = new File(slave1SplunkHome
				+ "/etc/apps/shuttl/conf");
		TUtilsEnvironment.runInCleanEnvironment(new Runnable() {

			@Override
			public void run() {
				TUtilsEnvironment.setEnvironmentVariable("SPLUNK_HOME", splunkHome);
				TUtilsMBean.runWithRegisteredMBeans(slave1ShuttlConfDir,
						new AssertBucketWasArchived(rb));
			}
		});
	}

	private static class AssertBucketWasArchived implements Runnable {
		private LocalBucket replicatedBucket;

		public AssertBucketWasArchived(LocalBucket rb) {
			this.replicatedBucket = rb;
		}

		@Override
		public void run() {
			ArchiveConfiguration config = ArchiveConfiguration
					.createConfigurationFromMBean();
			PathResolver pathResolver = new PathResolver(config);
			String normalBucketName = replicatedBucket.getName().replaceFirst("rb",
					"db");
			Bucket normalBucket = new Bucket(replicatedBucket.getPath(),
					replicatedBucket.getIndex(), normalBucketName,
					replicatedBucket.getFormat());
			String archivePath = pathResolver.resolveArchivePath(normalBucket);
			ArchiveFileSystem archiveFileSystem = ArchiveFileSystemFactory
					.getWithConfiguration(config);
			try {
				assertTrue(archiveFileSystem.exists(archivePath),
						"archiveFileSystem.exists(archivePath) was false, with archivePath: "
								+ archivePath);
			} catch (IOException e) {
				TUtilsTestNG.failForException("Path did not exist: " + archivePath, e);<commit_msg>Refactored Cluster bucket archiving test, for yet another test case. #102<commit_after>// Copyright (C) 2011 Splunk Inc.
//
// Splunk Inc. licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.splunk.shuttl.archiver.endtoend;

import static java.util.Arrays.*;
import static org.testng.Assert.*;

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;

import com.splunk.Service;
import com.splunk.shuttl.ShuttlConstants;
import com.splunk.shuttl.archiver.archive.ArchiveConfiguration;
import com.splunk.shuttl.archiver.filesystem.ArchiveFileSystem;
import com.splunk.shuttl.archiver.filesystem.ArchiveFileSystemFactory;
import com.splunk.shuttl.archiver.filesystem.PathResolver;
import com.splunk.shuttl.archiver.model.Bucket;
import com.splunk.shuttl.archiver.model.LocalBucket;
import com.splunk.shuttl.server.mbeans.util.EndpointUtils;
import com.splunk.shuttl.testutil.TUtilsBucket;
import com.splunk.shuttl.testutil.TUtilsEnvironment;
import com.splunk.shuttl.testutil.TUtilsMBean;
import com.splunk.shuttl.testutil.TUtilsTestNG;

@Test(groups = { "cluster-test" })
public class ClusterReplicatedBucketArchivingTest {

	private String index = "shuttl";

	public interface ReplicatedBucketProvider {

		LocalBucket create(String coldPathExpanded, String slave1Guid);
	}

	public interface ArchivePathAsserter {
		void assertStateOfArchivePathOnFileSystem(String archivePath,
				ArchiveFileSystem archiveFileSystem) throws IOException;
	}

	@Parameters(value = { "cluster.slave1.host", "cluster.slave1.port",
			"cluster.slave2.host", "cluster.slave2.port",
			"cluster.slave2.shuttl.port", "splunk.username", "splunk.password",
			"cluster.slave2.splunk.home" })
	public void _givenReplicatedBucket_archivesBucketFromSlave2LookingLikeItCameFromSlave1(
			String slave1Host, String slave1Port, String slave2Host,
			String slave2Port, String slave2ShuttlPort, String splunkUser,
			String splunkPass, final String splunkHome) {
		runClusterArchivingTest(slave1Host, slave1Port, slave2Host, slave2Port,
				slave2ShuttlPort, splunkUser, splunkPass, splunkHome,
				new FullReplicatedBucketProvider(index), new AssertsArchivePathExists());
	}

	private void runClusterArchivingTest(String slave1Host, String slave1Port,
			String slave2Host, String slave2Port, String slave2ShuttlPort,
			String splunkUser, String splunkPass, final String splunkHome,
			ReplicatedBucketProvider replicatedBucketProvider,
			final ArchivePathAsserter archivePathAsserter) {
		Service slave1 = getLoggedInService(slave1Host, slave1Port, splunkUser,
				splunkPass);
		Service slave2 = getLoggedInService(slave2Host, slave2Port, splunkUser,
				splunkPass);

		assertTrue(slave2.getIndexes().containsKey(index));
		String coldPathExpanded = slave2.getIndexes().get(index)
				.getColdPathExpanded();

		String slave1Guid = slave1.getInfo().getGuid();

		final LocalBucket rb = replicatedBucketProvider.create(coldPathExpanded,
				slave1Guid);

		try {
			callSlave2ArchiveBucketEndpoint(index, rb.getDirectory()
					.getAbsolutePath(), slave2Host, slave2ShuttlPort);
		} catch (IOException e1) {
			TUtilsTestNG.failForException(
					"failed when calling archive bucket endpoint", e1);
		}

		String slave1SplunkHome = slave1.getSettings().getSplunkHome();
		final File slave1ShuttlConfDir = new File(slave1SplunkHome
				+ "/etc/apps/shuttl/conf");
		TUtilsEnvironment.runInCleanEnvironment(new Runnable() {

			@Override
			public void run() {
				TUtilsEnvironment.setEnvironmentVariable("SPLUNK_HOME", splunkHome);
				TUtilsMBean.runWithRegisteredMBeans(slave1ShuttlConfDir,
						new AssertBucketWasArchived(rb, archivePathAsserter));
			}
		});
	}

	public static class FullReplicatedBucketProvider implements
			ReplicatedBucketProvider {

		private final String index;

		public FullReplicatedBucketProvider(String index) {
			this.index = index;
		}

		@Override
		public LocalBucket create(String coldPathExpanded, String slave1Guid) {
			return TUtilsBucket.createReplicatedBucket(index, new File(
					coldPathExpanded), slave1Guid);
		}
	}

	public static class AssertsArchivePathExists implements ArchivePathAsserter {

		@Override
		public void assertStateOfArchivePathOnFileSystem(String archivePath,
				ArchiveFileSystem archiveFileSystem) throws IOException {
			assertTrue(archiveFileSystem.exists(archivePath),
					"archiveFileSystem.exists(archivePath) was false, with archivePath: "
							+ archivePath);
		}
	}

	private static class AssertBucketWasArchived implements Runnable {
		private final LocalBucket replicatedBucket;
		private final ArchivePathAsserter archivePathAsserter;

		public AssertBucketWasArchived(LocalBucket rb,
				ArchivePathAsserter archivePathAsserter) {
			this.replicatedBucket = rb;
			this.archivePathAsserter = archivePathAsserter;
		}

		@Override
		public void run() {
			ArchiveConfiguration config = ArchiveConfiguration
					.createConfigurationFromMBean();
			PathResolver pathResolver = new PathResolver(config);
			String normalBucketName = replicatedBucket.getName().replaceFirst("rb",
					"db");
			Bucket normalBucket = new Bucket(replicatedBucket.getPath(),
					replicatedBucket.getIndex(), normalBucketName,
					replicatedBucket.getFormat());
			String archivePath = pathResolver.resolveArchivePath(normalBucket);
			ArchiveFileSystem archiveFileSystem = ArchiveFileSystemFactory
					.getWithConfiguration(config);
			try {
				archivePathAsserter.assertStateOfArchivePathOnFileSystem(archivePath,
						archiveFileSystem);
			} catch (IOException e) {
				TUtilsTestNG.failForException("Path did not exist: " + archivePath, e);
{'fraction_non_alphanumeric': 0.09217046580773043, 'fraction_numerical': 0.00882962429047662, 'mean_word_length': 5.2464828362408555, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5400289', 'n_tokens_mistral': 3640, 'n_tokens_neox': 3233, 'n_words': 731}