<commit_before>/*
 * GRAKN.AI - THE KNOWLEDGE GRAPH
 * Copyright (C) 2018 Grakn Labs Ltd
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

package grakn.core.rule;

import grakn.core.common.config.Config;
import grakn.core.common.config.ConfigKey;
import grakn.core.server.GraknStorage;
import grakn.core.server.Server;
import grakn.core.server.ServerFactory;
import grakn.core.server.deduplicator.AttributeDeduplicatorDaemon;
import grakn.core.server.keyspace.KeyspaceImpl;
import grakn.core.server.keyspace.KeyspaceManager;
import grakn.core.server.rpc.KeyspaceService;
import grakn.core.server.rpc.OpenRequest;
import grakn.core.server.rpc.ServerOpenRequest;
import grakn.core.server.rpc.SessionService;
import grakn.core.server.session.JanusGraphFactory;
import grakn.core.server.session.SessionFactory;
import grakn.core.server.session.SessionImpl;
import grakn.core.server.util.LockManager;
import grakn.core.server.util.ServerID;
import grakn.core.server.util.ServerLockManager;
import io.grpc.ServerBuilder;
import org.apache.commons.io.FileUtils;
import org.junit.rules.ExternalResource;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.ServerSocket;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.UUID;

/**
 * This rule is a test server rule which starts Cassandra and Grakn Core Server on random, unused ports.
 * It allows multiple test server instances to run concurrently.
 * It enables all of the integration tests to run concurrently on the same machine.
 */
public class GraknTestServer extends ExternalResource {
    protected static final Path DEFAULT_SERVER_CONFIG_PATH = Paths.get("server/conf/grakn.properties");
    protected static final Path DEFAULT_CASSANDRA_CONFIG_PATH = Paths.get("test-integration/resources/cassandra-embedded.yaml");

    // Grakn Core Server
    protected final Path serverConfigPath;
    protected Config serverConfig;
    protected Server graknServer;
    protected int grpcPort;
    protected KeyspaceManager keyspaceStore;
    protected SessionFactory sessionFactory;
    protected Path dataDirTmp;

    // Cassandra
    protected final Path originalCassandraConfigPath;
    protected File updatedCassandraConfigPath;
    protected int storagePort;
    protected int rpcPort;
    protected int nativeTransportPort;


    public GraknTestServer() {
        this(DEFAULT_SERVER_CONFIG_PATH, DEFAULT_CASSANDRA_CONFIG_PATH);
    }

    public GraknTestServer(Path serverConfigPath, Path cassandraConfigPath) {
        System.setProperty("java.security.manager", "nottodaypotato");
        this.serverConfigPath = serverConfigPath;
        this.originalCassandraConfigPath = cassandraConfigPath;
    }

    @Override
    protected void before() {
        try {
            // Start Cassandra
            System.out.println("Starting Grakn Storage...");
            generateCassandraRandomPorts();
            updatedCassandraConfigPath = buildCassandraConfigWithRandomPorts();
            System.setProperty("cassandra.config", "file:" + updatedCassandraConfigPath.getAbsolutePath());
            System.setProperty("cassandra-foreground", "true");
            System.out.println("cassandraConfig.getAbsolutePath() = " + updatedCassandraConfigPath.getAbsolutePath());
            GraknStorage.main(new String[]{});
            System.out.println("Grakn Storage started");

            // Start Grakn Core Server
            grpcPort = findUnusedLocalPort();
            dataDirTmp = Files.createTempDirectory("db-for-test");
            serverConfig = createTestConfig(dataDirTmp.toString());
            System.out.println("Starting Grakn Core Server...");
            graknServer = createServer();
            graknServer.start();
            System.out.println("Grakn Core Server started");
        } catch (IOException e) {
            throw new RuntimeException("Cannot start Grakn Core Server", e);
        }
    }

    @Override
    protected void after() {
        try {
            keyspaceStore.closeStore();
            graknServer.close();
            FileUtils.deleteDirectory(dataDirTmp.toFile());
            updatedCassandraConfigPath.delete();
        } catch (Exception e) {
            throw new RuntimeException("Could not shut down ", e);
        }
    }


    // Getters

    public String grpcUri() {
        return serverConfig.getProperty(ConfigKey.SERVER_HOST_NAME) + ":" +serverConfig.getProperty(ConfigKey.GRPC_PORT);
    }

    public int nativeTransportPort() {
        return nativeTransportPort;
    }

    public SessionImpl sessionWithNewKeyspace() {
        KeyspaceImpl randomKeyspace = KeyspaceImpl.of("a" + UUID.randomUUID().toString().replaceAll("-", ""));
        return session(randomKeyspace);
    }

    public SessionImpl session(String keyspace) {
        return session(KeyspaceImpl.of(keyspace));
    }

    public SessionImpl session(KeyspaceImpl keyspace) {
        return sessionFactory.session(keyspace);
    }

    public SessionFactory sessionFactory() {
        return sessionFactory;
    }

    // Cassandra Helpers

    protected void generateCassandraRandomPorts() throws IOException {
        storagePort = findUnusedLocalPort();
        nativeTransportPort = findUnusedLocalPort();
        rpcPort = findUnusedLocalPort();
    }

    protected File buildCassandraConfigWithRandomPorts() throws IOException {
        byte[] bytes = Files.readAllBytes(originalCassandraConfigPath);
        String configString = new String(bytes, StandardCharsets.UTF_8);

        configString = configString + "\nstorage_port: " + storagePort;
        configString = configString + "\nnative_transport_port: " + nativeTransportPort;
        configString = configString + "\nrpc_port: " + rpcPort;
        InputStream configStream = new ByteArrayInputStream(configString.getBytes(StandardCharsets.UTF_8));

        String directory = "target/embeddedCassandra";
        org.apache.cassandra.io.util.FileUtils.createDirectory(directory);
        Path copyName = Paths.get(directory, "cassandra-embedded.yaml");
        // Create file in directory we just created and copy the stream content into it.
        Files.copy(configStream, copyName);
        return copyName.toFile();
    }

    protected static int findUnusedLocalPort() throws IOException {
        try (ServerSocket serverSocket = new ServerSocket(0)) {
            return serverSocket.getLocalPort();
        }
    }

    // Grakn Core Server helpers

    protected Config createTestConfig(String dataDir) throws FileNotFoundException {
        InputStream testConfig = new FileInputStream(serverConfigPath.toFile());

        Config config = Config.read(testConfig);
        config.setConfigProperty(ConfigKey.DATA_DIR, dataDir);
        //Override gRPC port with a random free port
        config.setConfigProperty(ConfigKey.GRPC_PORT, grpcPort);
        //Override the default store.port with the RPC_PORT given that we still use Thrift protocol to talk to Cassandra
        config.setConfigProperty(ConfigKey.STORAGE_PORT, rpcPort);
        //Hadoop cluster uses the Astyanax driver for some operations, so need to override the RPC_PORT (Thrift)<commit_msg>synchronized method in findUnsusedLocalPort in TestServer (#5221)<commit_after>/*
 * GRAKN.AI - THE KNOWLEDGE GRAPH
 * Copyright (C) 2018 Grakn Labs Ltd
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

package grakn.core.rule;

import grakn.core.common.config.Config;
import grakn.core.common.config.ConfigKey;
import grakn.core.server.GraknStorage;
import grakn.core.server.Server;
import grakn.core.server.ServerFactory;
import grakn.core.server.deduplicator.AttributeDeduplicatorDaemon;
import grakn.core.server.keyspace.KeyspaceImpl;
import grakn.core.server.keyspace.KeyspaceManager;
import grakn.core.server.rpc.KeyspaceService;
import grakn.core.server.rpc.OpenRequest;
import grakn.core.server.rpc.ServerOpenRequest;
import grakn.core.server.rpc.SessionService;
import grakn.core.server.session.JanusGraphFactory;
import grakn.core.server.session.SessionFactory;
import grakn.core.server.session.SessionImpl;
import grakn.core.server.util.LockManager;
import grakn.core.server.util.ServerID;
import grakn.core.server.util.ServerLockManager;
import io.grpc.ServerBuilder;
import org.apache.commons.io.FileUtils;
import org.junit.rules.ExternalResource;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.ServerSocket;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.UUID;

/**
 * This rule is a test server rule which starts Cassandra and Grakn Core Server on random, unused ports.
 * It allows multiple test server instances to run concurrently.
 * It enables all of the integration tests to run concurrently on the same machine.
 */
public class GraknTestServer extends ExternalResource {
    protected static final Path DEFAULT_SERVER_CONFIG_PATH = Paths.get("server/conf/grakn.properties");
    protected static final Path DEFAULT_CASSANDRA_CONFIG_PATH = Paths.get("test-integration/resources/cassandra-embedded.yaml");

    // Grakn Core Server
    protected final Path serverConfigPath;
    protected Config serverConfig;
    protected Server graknServer;
    protected int grpcPort;
    protected KeyspaceManager keyspaceStore;
    protected SessionFactory sessionFactory;
    protected Path dataDirTmp;

    // Cassandra
    protected final Path originalCassandraConfigPath;
    protected File updatedCassandraConfigPath;
    protected int storagePort;
    protected int rpcPort;
    protected int nativeTransportPort;


    public GraknTestServer() {
        this(DEFAULT_SERVER_CONFIG_PATH, DEFAULT_CASSANDRA_CONFIG_PATH);
    }

    public GraknTestServer(Path serverConfigPath, Path cassandraConfigPath) {
        System.setProperty("java.security.manager", "nottodaypotato");
        this.serverConfigPath = serverConfigPath;
        this.originalCassandraConfigPath = cassandraConfigPath;
    }

    @Override
    protected void before() {
        try {
            // Start Cassandra
            System.out.println("Starting Grakn Storage...");
            generateCassandraRandomPorts();
            updatedCassandraConfigPath = buildCassandraConfigWithRandomPorts();
            System.setProperty("cassandra.config", "file:" + updatedCassandraConfigPath.getAbsolutePath());
            System.setProperty("cassandra-foreground", "true");
            System.out.println("cassandraConfig.getAbsolutePath() = " + updatedCassandraConfigPath.getAbsolutePath());
            GraknStorage.main(new String[]{});
            System.out.println("Grakn Storage started");

            // Start Grakn Core Server
            grpcPort = findUnusedLocalPort();
            dataDirTmp = Files.createTempDirectory("db-for-test");
            serverConfig = createTestConfig(dataDirTmp.toString());
            System.out.println("Starting Grakn Core Server...");
            graknServer = createServer();
            graknServer.start();
            System.out.println("Grakn Core Server started");
        } catch (IOException e) {
            throw new RuntimeException("Cannot start Grakn Core Server", e);
        }
    }

    @Override
    protected void after() {
        try {
            keyspaceStore.closeStore();
            graknServer.close();
            FileUtils.deleteDirectory(dataDirTmp.toFile());
            updatedCassandraConfigPath.delete();
        } catch (Exception e) {
            throw new RuntimeException("Could not shut down ", e);
        }
    }


    // Getters

    public String grpcUri() {
        return serverConfig.getProperty(ConfigKey.SERVER_HOST_NAME) + ":" +serverConfig.getProperty(ConfigKey.GRPC_PORT);
    }

    public int nativeTransportPort() {
        return nativeTransportPort;
    }

    public SessionImpl sessionWithNewKeyspace() {
        KeyspaceImpl randomKeyspace = KeyspaceImpl.of("a" + UUID.randomUUID().toString().replaceAll("-", ""));
        return session(randomKeyspace);
    }

    public SessionImpl session(String keyspace) {
        return session(KeyspaceImpl.of(keyspace));
    }

    public SessionImpl session(KeyspaceImpl keyspace) {
        return sessionFactory.session(keyspace);
    }

    public SessionFactory sessionFactory() {
        return sessionFactory;
    }

    // Cassandra Helpers

    protected void generateCassandraRandomPorts() throws IOException {
        storagePort = findUnusedLocalPort();
        nativeTransportPort = findUnusedLocalPort();
        rpcPort = findUnusedLocalPort();
    }

    protected File buildCassandraConfigWithRandomPorts() throws IOException {
        byte[] bytes = Files.readAllBytes(originalCassandraConfigPath);
        String configString = new String(bytes, StandardCharsets.UTF_8);

        configString = configString + "\nstorage_port: " + storagePort;
        configString = configString + "\nnative_transport_port: " + nativeTransportPort;
        configString = configString + "\nrpc_port: " + rpcPort;
        InputStream configStream = new ByteArrayInputStream(configString.getBytes(StandardCharsets.UTF_8));

        String directory = "target/embeddedCassandra";
        org.apache.cassandra.io.util.FileUtils.createDirectory(directory);
        Path copyName = Paths.get(directory, "cassandra-embedded.yaml");
        // Create file in directory we just created and copy the stream content into it.
        Files.copy(configStream, copyName);
        return copyName.toFile();
    }

    protected synchronized static int findUnusedLocalPort() throws IOException {
        try (ServerSocket serverSocket = new ServerSocket(0)) {
            return serverSocket.getLocalPort();
        }
    }

    // Grakn Core Server helpers

    protected Config createTestConfig(String dataDir) throws FileNotFoundException {
        InputStream testConfig = new FileInputStream(serverConfigPath.toFile());

        Config config = Config.read(testConfig);
        config.setConfigProperty(ConfigKey.DATA_DIR, dataDir);
        //Override gRPC port with a random free port
        config.setConfigProperty(ConfigKey.GRPC_PORT, grpcPort);
        //Override the default store.port with the RPC_PORT given that we still use Thrift protocol to talk to Cassandra
        config.setConfigProperty(ConfigKey.STORAGE_PORT, rpcPort);
        //Hadoop cluster uses the Astyanax driver for some operations, so need to override the RPC_PORT (Thrift)
{'fraction_non_alphanumeric': 0.08587464920486436, 'fraction_numerical': 0.0012472715933894605, 'mean_word_length': 3.5120990433314576, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6975814', 'n_tokens_mistral': 4382, 'n_tokens_neox': 4169, 'n_words': 1118}