<commit_before>/*
 * Copyright 2015 Delft University of Technology
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package science.atlarge.graphalytics.openg;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import nl.tudelft.granula.archiver.PlatformArchive;
import nl.tudelft.granula.modeller.job.JobModel;
import nl.tudelft.granula.modeller.platform.Openg;
import org.apache.commons.io.output.TeeOutputStream;
import science.atlarge.graphalytics.domain.graph.FormattedGraph;
import science.atlarge.graphalytics.report.result.BenchmarkMetrics;
import science.atlarge.graphalytics.report.result.BenchmarkResult;
import science.atlarge.graphalytics.domain.benchmark.BenchmarkRun;
import science.atlarge.graphalytics.granula.GranulaAwarePlatform;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import it.unimi.dsi.fastutil.longs.Long2LongMap;
import science.atlarge.graphalytics.execution.PlatformExecutionException;
import science.atlarge.graphalytics.configuration.ConfigurationUtil;
import science.atlarge.graphalytics.configuration.InvalidConfigurationException;
import science.atlarge.graphalytics.domain.graph.PropertyList;
import science.atlarge.graphalytics.domain.graph.PropertyType;
import science.atlarge.graphalytics.openg.algorithms.bfs.BreadthFirstSearchJob;
import science.atlarge.graphalytics.openg.algorithms.cdlp.CommunityDetectionLPJob;
import science.atlarge.graphalytics.openg.algorithms.lcc.LocalClusteringCoefficientJob;
import science.atlarge.graphalytics.openg.algorithms.pr.PageRankJob;
import science.atlarge.graphalytics.openg.algorithms.sssp.SingleSourceShortestPathsJob;
import science.atlarge.graphalytics.openg.algorithms.wcc.WeaklyConnectedComponentsJob;
import science.atlarge.graphalytics.openg.config.JobConfiguration;
import science.atlarge.graphalytics.openg.config.JobConfigurationParser;
import org.json.simple.JSONObject;
import science.atlarge.graphalytics.domain.algorithms.*;

/**
 * OpenG platform integration for the Graphalytics benchmark.
 *
 * @author <NAME>
 * @author <NAME>
 */
public class OpengPlatform implements GranulaAwarePlatform {


	private static PrintStream sysOut;
	private static PrintStream sysErr;

	protected static final Logger LOG = LogManager.getLogger();

	public static final String PLATFORM_NAME = "openg";
	public static final String PLATFORM_PROPERTIES_FILE = PLATFORM_NAME + ".properties";
	private static final String BENCHMARK_PROPERTIES_FILE = "benchmark.properties";
	private static final String GRANULA_PROPERTIES_FILE = "granula.properties";

	public static final String GRANULA_ENABLE_KEY = "benchmark.run.granula.enabled";
	public static final String OPENG_INTERMEDIATE_DIR_KEY = "openg.intermediate-dir";

	public static String OPENG_BINARY_DIRECTORY = "./bin/standard";

	protected JobConfiguration jobConfiguration;
	protected String intermediateDirectory;
	protected String graphOutputDirectory;

	protected String intermediatePath;
	protected String currentGraphPath;
	protected Long2LongMap currentGraphVertexIdTranslation;

	public OpengPlatform() {
		try {
			loadConfiguration();
		} catch (InvalidConfigurationException e) {
			// TODO: Implement cleaner exit procedure
			LOG.fatal(e);
			System.exit(-1);
		}

	}

	protected void loadConfiguration() throws InvalidConfigurationException {
		LOG.info("Parsing OpenG configuration file.");

		Configuration benchmarkConfig;
		Configuration platformConfig;
		Configuration granulaConfig;

		// Load OpenG-specific configuration
		try {
			platformConfig = new PropertiesConfiguration(PLATFORM_PROPERTIES_FILE);
			benchmarkConfig = new PropertiesConfiguration(BENCHMARK_PROPERTIES_FILE);
			granulaConfig = new PropertiesConfiguration(GRANULA_PROPERTIES_FILE);
		} catch (ConfigurationException e) {
			// Fall-back to an empty properties file
			LOG.warn("Could not find or load \"{}\"", PLATFORM_PROPERTIES_FILE);
			LOG.warn("Could not find or load \"{}\"", BENCHMARK_PROPERTIES_FILE);
			LOG.warn("Could not find or load \"{}\"", GRANULA_PROPERTIES_FILE);
			platformConfig = new PropertiesConfiguration();
			benchmarkConfig = new PropertiesConfiguration();
			granulaConfig = new PropertiesConfiguration();
		}

		// Parse generic job configuration from the OpenG properties file
		jobConfiguration = JobConfigurationParser.parseOpenGPropertiesFile(platformConfig);

		intermediateDirectory = ConfigurationUtil.getString(platformConfig, OPENG_INTERMEDIATE_DIR_KEY);
		ensureDirectoryExists(intermediateDirectory, OPENG_INTERMEDIATE_DIR_KEY);

		boolean granulaEnabled = granulaConfig.getBoolean(GRANULA_ENABLE_KEY, false);
		OPENG_BINARY_DIRECTORY = granulaEnabled ? "./bin/granula": OPENG_BINARY_DIRECTORY;
	}<commit_msg>Simply configuration loading.<commit_after>/*
 * Copyright 2015 Delft University of Technology
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package science.atlarge.graphalytics.openg;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import nl.tudelft.granula.archiver.PlatformArchive;
import nl.tudelft.granula.modeller.job.JobModel;
import nl.tudelft.granula.modeller.platform.Openg;
import org.apache.commons.io.output.TeeOutputStream;
import science.atlarge.graphalytics.domain.graph.FormattedGraph;
import science.atlarge.graphalytics.report.result.BenchmarkMetrics;
import science.atlarge.graphalytics.report.result.BenchmarkResult;
import science.atlarge.graphalytics.domain.benchmark.BenchmarkRun;
import science.atlarge.graphalytics.granula.GranulaAwarePlatform;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import it.unimi.dsi.fastutil.longs.Long2LongMap;
import science.atlarge.graphalytics.execution.PlatformExecutionException;
import science.atlarge.graphalytics.configuration.ConfigurationUtil;
import science.atlarge.graphalytics.configuration.InvalidConfigurationException;
import science.atlarge.graphalytics.domain.graph.PropertyList;
import science.atlarge.graphalytics.domain.graph.PropertyType;
import science.atlarge.graphalytics.openg.algorithms.bfs.BreadthFirstSearchJob;
import science.atlarge.graphalytics.openg.algorithms.cdlp.CommunityDetectionLPJob;
import science.atlarge.graphalytics.openg.algorithms.lcc.LocalClusteringCoefficientJob;
import science.atlarge.graphalytics.openg.algorithms.pr.PageRankJob;
import science.atlarge.graphalytics.openg.algorithms.sssp.SingleSourceShortestPathsJob;
import science.atlarge.graphalytics.openg.algorithms.wcc.WeaklyConnectedComponentsJob;
import science.atlarge.graphalytics.openg.config.JobConfiguration;
import science.atlarge.graphalytics.openg.config.JobConfigurationParser;
import org.json.simple.JSONObject;
import science.atlarge.graphalytics.domain.algorithms.*;

/**
 * OpenG platform integration for the Graphalytics benchmark.
 *
 * @author <NAME>
 * @author <NAME>
 */
public class OpengPlatform implements GranulaAwarePlatform {


	private static PrintStream sysOut;
	private static PrintStream sysErr;

	protected static final Logger LOG = LogManager.getLogger();

	public static final String PLATFORM_NAME = "openg";
	public static final String PLATFORM_PROPERTIES_FILE = PLATFORM_NAME + ".properties";
	private static final String BENCHMARK_PROPERTIES_FILE = "benchmark.properties";
	private static final String GRANULA_PROPERTIES_FILE = "granula.properties";

	public static final String GRANULA_ENABLE_KEY = "benchmark.run.granula.enabled";
	public static final String OPENG_INTERMEDIATE_DIR_KEY = "openg.intermediate-dir";

	public static String OPENG_BINARY_DIRECTORY = "./bin/standard";

	protected JobConfiguration jobConfiguration;
	protected String intermediateDirectory;
	protected String graphOutputDirectory;

	protected String intermediatePath;
	protected String currentGraphPath;
	protected Long2LongMap currentGraphVertexIdTranslation;

	public OpengPlatform() {
		try {
			loadConfiguration();
		} catch (InvalidConfigurationException e) {
			// TODO: Implement cleaner exit procedure
			LOG.fatal(e);
			System.exit(-1);
		}

	}

	protected void loadConfiguration() throws InvalidConfigurationException {
		LOG.info("Parsing OpenG configuration file.");

		Configuration benchmarkConfig;
		Configuration platformConfig;
		Configuration granulaConfig;

		// Load OpenG-specific configuration
		try {
			platformConfig = ConfigurationUtil.loadConfiguration(PLATFORM_PROPERTIES_FILE);
			benchmarkConfig = ConfigurationUtil.loadConfiguration(BENCHMARK_PROPERTIES_FILE);
			granulaConfig = ConfigurationUtil.loadConfiguration(GRANULA_PROPERTIES_FILE);
		} catch (InvalidConfigurationException e) {
			// Fall-back to an empty properties file
			LOG.warn("Could not find or load \"{}\"", PLATFORM_PROPERTIES_FILE);
			LOG.warn("Could not find or load \"{}\"", BENCHMARK_PROPERTIES_FILE);
			LOG.warn("Could not find or load \"{}\"", GRANULA_PROPERTIES_FILE);
			platformConfig = new PropertiesConfiguration();
			benchmarkConfig = new PropertiesConfiguration();
			granulaConfig = new PropertiesConfiguration();
		}

		// Parse generic job configuration from the OpenG properties file
		jobConfiguration = JobConfigurationParser.parseOpenGPropertiesFile(platformConfig);

		intermediateDirectory = ConfigurationUtil.getString(platformConfig, OPENG_INTERMEDIATE_DIR_KEY);
		ensureDirectoryExists(intermediateDirectory, OPENG_INTERMEDIATE_DIR_KEY);

		boolean granulaEnabled = granulaConfig.getBoolean(GRANULA_ENABLE_KEY, false);
		OPENG_BINARY_DIRECTORY = granulaEnabled ? "./bin/granula": OPENG_BINARY_DIRECTORY;
	}
{'fraction_non_alphanumeric': 0.08373732921992066, 'fraction_numerical': 0.0022917584839136185, 'mean_word_length': 6.162878787878788, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7488098', 'n_tokens_mistral': 3319, 'n_tokens_neox': 3145, 'n_words': 713}