<commit_before>/**
 * © <NAME> 2015
 */
package io.spacedog.server;

import java.io.IOException;
import java.util.Collections;
import java.util.concurrent.ExecutionException;

import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.common.logging.LogConfigurator;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.Settings.Builder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.reindex.ReindexPlugin;
import org.elasticsearch.node.InternalSettingsPreparer;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.transport.Netty4Plugin;
import org.joda.time.DateTimeZone;

import io.spacedog.utils.ClassResources;
import io.spacedog.utils.DateTimeZones;
import io.spacedog.utils.Exceptions;
import io.spacedog.utils.Json;
import net.codestory.http.AbstractWebServer;
import net.codestory.http.Request;
import net.codestory.http.Response;
import net.codestory.http.internal.Handler;
import net.codestory.http.internal.HttpServerWrapper;
import net.codestory.http.internal.SimpleServerWrapper;
import net.codestory.http.payload.Payload;
import net.codestory.http.routes.Routes;
import net.codestory.http.websockets.WebSocketHandler;

public class Server {

	public static final String CLUSTER_NAME = "spacedog-v1-cluster";

	private ElasticNode elasticNode;
	private ElasticClient elasticClient;
	private FluentServer fluent;
	private ServerConfiguration config;
	private Info info;

	public ElasticClient elasticClient() {
		return elasticClient;
	}

	public ElasticNode elasticNode() {
		return elasticNode;
	}

	public ServerConfiguration configuration() {
		return config;
	}

	public static void main(String[] args) {
		DateTimeZone.setDefault(DateTimeZones.PARIS);
		Server server = new Server();
		server.start();
	}

	public void start() {

		try {
			init();
			startElastic();
			elasticIsStarted();
			startFluent();
			fluentIsStarted();

		} catch (Throwable t) {
			t.printStackTrace();
			if (fluent != null)
				fluent.stop();
			if (elasticClient != null)
				elasticClient.close();
			if (elasticNode != null)
				elasticClient.close();
			System.exit(-1);
		}
	}

	protected void init() {
		this.config = new ServerConfiguration();
		System.setProperty("http.agent", configuration().serverUserAgent());
		String string = ClassResources.loadAsString(this, "info.json");
		info = Json.toPojo(string, Info.class);
	}

	protected void startElastic()
			throws InterruptedException, ExecutionException, IOException, NodeValidationException {

		Builder builder = Settings.builder()//
				.put(Environment.PATH_HOME_SETTING.getKey(), //
						config.homePath().toAbsolutePath().toString());
		// .put(Node.NODE_MASTER_SETTING.getKey(), true)//
		// .put(Node.NODE_DATA_SETTING.getKey(), true)//
		// .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "single-node")//
		// // .put(NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), "netty4") //
		// .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), CLUSTER_NAME)//
		// // disable automatic index creation
		// .put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false)//
		// // disable dynamic indexing
		// // .put("index.mapper.dynamic", false)//
		// // .put("index.max_result_window", 5000)//
		// // disable rebalance to avoid automatic rebalance
		// // when a temporary second node appears
		// .put("cluster.routing.rebalance.enable", "none")//
		// .put(NetworkModule.HTTP_ENABLED.getKey(), //
		// config.elasticIsHttpEnabled())//
		// .put(NetworkService.GLOBAL_NETWORK_HOST_SETTING.getKey(), //
		// config.elasticNetworkHost())//
		// .put(Environment.PATH_LOGS_SETTING.getKey(), //
		// config.elasticLogsPath().toAbsolutePath().toString())
		// .put(Environment.PATH_DATA_SETTING.getKey(), //
		// config.elasticDataPath().toAbsolutePath().toString());
		//
		// if (config.elasticSnapshotsPath().isPresent())
		// builder.put(Environment.PATH_REPO_SETTING.getKey(), //
		// config.elasticSnapshotsPath().get().toAbsolutePath().toString());

		Settings settings = builder.build();

		Environment environment = InternalSettingsPreparer.prepareEnvironment(//
				settings, Terminal.DEFAULT, Collections.emptyMap(), config.elasticConfigPath());

		try {
			LogConfigurator.configure(environment);
		} catch (Exception e) {
			throw Exceptions.runtime(e, "error configuring elastic log");
		}

		elasticNode = new ElasticNode(environment, //
				Netty4Plugin.class, //
				CommonAnalysisPlugin.class, //
				ReindexPlugin.class);

		// S3RepositoryPlugin.class);

		elasticNode.start();
		this.elasticClient = new ElasticClient(elasticNode.client());

		// wait for cluster to fully initialize and turn asynchronously from
		// RED status to GREEN before to initialize anything else
		// wait for 60 seconds maximum by default
		// configurable with spacedog.server.green.check and timeout properties
		elasticClient.ensureAllIndicesAreGreen();
	}

	protected void elasticIsStarted() {
		// init templates
		this.elasticClient.internal().admin().indices()//
				.preparePutTemplate("data")//
				.setSource(//
						ClassResources.loadAsBytes(this, "data-template.json"), //
						XContentType.JSON)//
				.get();

		// init root backend indices
		String backendId = config.apiBackend().backendId();
		AdminService.get().initBackendIndices(backendId);
	}

	protected void startFluent() {
		fluent = new FluentServer();
		fluent.configure(routes -> configure(routes));
		fluent.start(config.serverPort());
	}

	protected void fluentIsStarted() {
	}

	protected void configure(Routes routes) {
		routes.add(AdminService.get())//
				.add(DataService.get())//
				.add(SchemaService.get())//
				.add(CredentialsService.get())//
				.add(LinkedinService.get())//
				.add(BatchService.get())//
				.add(EmailService.get())//
				.add(SmsService.get())//
				.add(LogService.get())//
				.add(PushService.get())//<commit_msg>Refactor Server class to allow sub classing<commit_after>/**
 * © <NAME> 2015
 */
package io.spacedog.server;

import java.io.IOException;
import java.util.Collections;
import java.util.concurrent.ExecutionException;

import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.common.logging.LogConfigurator;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.Settings.Builder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.reindex.ReindexPlugin;
import org.elasticsearch.node.InternalSettingsPreparer;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.transport.Netty4Plugin;
import org.joda.time.DateTimeZone;

import io.spacedog.utils.ClassResources;
import io.spacedog.utils.DateTimeZones;
import io.spacedog.utils.Exceptions;
import io.spacedog.utils.Json;
import net.codestory.http.AbstractWebServer;
import net.codestory.http.Request;
import net.codestory.http.Response;
import net.codestory.http.internal.Handler;
import net.codestory.http.internal.HttpServerWrapper;
import net.codestory.http.internal.SimpleServerWrapper;
import net.codestory.http.payload.Payload;
import net.codestory.http.routes.Routes;
import net.codestory.http.websockets.WebSocketHandler;

public class Server {

	public static final String CLUSTER_NAME = "spacedog-v1-cluster";

	private ElasticNode elasticNode;
	private ElasticClient elasticClient;
	private FluentServer fluent;
	private ServerConfiguration config;
	private Info info;

	public ElasticClient elasticClient() {
		return elasticClient;
	}

	public ElasticNode elasticNode() {
		return elasticNode;
	}

	public ServerConfiguration configuration() {
		return config;
	}

	public static void main(String[] args) {
		DateTimeZone.setDefault(DateTimeZones.PARIS);
		Server server = new Server();
		server.start();
	}

	public void start() {

		try {
			init();
			startElastic();
			elasticIsStarted();
			startFluent();
			fluentIsStarted();

		} catch (Throwable t) {
			t.printStackTrace();
			if (fluent != null)
				fluent.stop();
			if (elasticClient != null)
				elasticClient.close();
			if (elasticNode != null)
				elasticClient.close();
			System.exit(-1);
		}
	}

	protected void init() {
		this.config = new ServerConfiguration();
		System.setProperty("http.agent", configuration().serverUserAgent());
		String string = ClassResources.loadAsString(Server.class, "info.json");
		info = Json.toPojo(string, Info.class);
	}

	protected void startElastic()
			throws InterruptedException, ExecutionException, IOException, NodeValidationException {

		Builder builder = Settings.builder()//
				.put(Environment.PATH_HOME_SETTING.getKey(), //
						config.homePath().toAbsolutePath().toString());
		// .put(Node.NODE_MASTER_SETTING.getKey(), true)//
		// .put(Node.NODE_DATA_SETTING.getKey(), true)//
		// .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "single-node")//
		// // .put(NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), "netty4") //
		// .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), CLUSTER_NAME)//
		// // disable automatic index creation
		// .put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false)//
		// // disable dynamic indexing
		// // .put("index.mapper.dynamic", false)//
		// // .put("index.max_result_window", 5000)//
		// // disable rebalance to avoid automatic rebalance
		// // when a temporary second node appears
		// .put("cluster.routing.rebalance.enable", "none")//
		// .put(NetworkModule.HTTP_ENABLED.getKey(), //
		// config.elasticIsHttpEnabled())//
		// .put(NetworkService.GLOBAL_NETWORK_HOST_SETTING.getKey(), //
		// config.elasticNetworkHost())//
		// .put(Environment.PATH_LOGS_SETTING.getKey(), //
		// config.elasticLogsPath().toAbsolutePath().toString())
		// .put(Environment.PATH_DATA_SETTING.getKey(), //
		// config.elasticDataPath().toAbsolutePath().toString());
		//
		// if (config.elasticSnapshotsPath().isPresent())
		// builder.put(Environment.PATH_REPO_SETTING.getKey(), //
		// config.elasticSnapshotsPath().get().toAbsolutePath().toString());

		Settings settings = builder.build();

		Environment environment = InternalSettingsPreparer.prepareEnvironment(//
				settings, Terminal.DEFAULT, Collections.emptyMap(), config.elasticConfigPath());

		try {
			LogConfigurator.configure(environment);
		} catch (Exception e) {
			throw Exceptions.runtime(e, "error configuring elastic log");
		}

		elasticNode = new ElasticNode(environment, //
				Netty4Plugin.class, //
				CommonAnalysisPlugin.class, //
				ReindexPlugin.class);

		// S3RepositoryPlugin.class);

		elasticNode.start();
		this.elasticClient = new ElasticClient(elasticNode.client());

		// wait for cluster to fully initialize and turn asynchronously from
		// RED status to GREEN before to initialize anything else
		// wait for 60 seconds maximum by default
		// configurable with spacedog.server.green.check and timeout properties
		elasticClient.ensureAllIndicesAreGreen();
	}

	protected void elasticIsStarted() {
		// init templates
		this.elasticClient.internal().admin().indices()//
				.preparePutTemplate("data")//
				.setSource(//
						ClassResources.loadAsBytes(Server.class, "data-template.json"), //
						XContentType.JSON)//
				.get();

		// init root backend indices
		String backendId = config.apiBackend().backendId();
		AdminService.get().initBackendIndices(backendId);
	}

	protected void startFluent() {
		fluent = new FluentServer();
		fluent.configure(routes -> configure(routes));
		fluent.start(config.serverPort());
	}

	protected void fluentIsStarted() {
	}

	protected void configure(Routes routes) {
		routes.add(AdminService.get())//
				.add(DataService.get())//
				.add(SchemaService.get())//
				.add(CredentialsService.get())//
				.add(LinkedinService.get())//
				.add(BatchService.get())//
				.add(EmailService.get())//
				.add(SmsService.get())//
				.add(LogService.get())//
				.add(PushService.get())//
{'fraction_non_alphanumeric': 0.13857615894039735, 'fraction_numerical': 0.0026490066225165563, 'mean_word_length': 4.315002199736032, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7386075', 'n_tokens_mistral': 3995, 'n_tokens_neox': 3634, 'n_words': 657}