<commit_before>package edu.washington.escience.myria.systemtest;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.ProcessBuilder.Redirect;
import java.lang.management.ManagementFactory;
import java.net.HttpURLConnection;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.mina.util.AvailablePortFinder;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.rules.TestName;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.rules.Timeout;
import org.junit.runner.Description;
import org.slf4j.LoggerFactory;

import com.almworks.sqlite4java.SQLiteException;
import com.fasterxml.jackson.databind.ObjectReader;
import com.google.common.collect.ImmutableList;
import com.google.protobuf.ByteString;

import edu.washington.escience.myria.DbException;
import edu.washington.escience.myria.MyriaConstants;
import edu.washington.escience.myria.MyriaSystemConfigKeys;
import edu.washington.escience.myria.RelationKey;
import edu.washington.escience.myria.Schema;
import edu.washington.escience.myria.Type;
import edu.washington.escience.myria.accessmethod.ConnectionInfo;
import edu.washington.escience.myria.accessmethod.SQLiteAccessMethod;
import edu.washington.escience.myria.accessmethod.SQLiteInfo;
import edu.washington.escience.myria.api.MyriaJsonMapperProvider;
import edu.washington.escience.myria.api.encoding.DatasetStatus;
import edu.washington.escience.myria.api.encoding.QueryStatusEncoding;
import edu.washington.escience.myria.coordinator.catalog.CatalogException;
import edu.washington.escience.myria.coordinator.catalog.CatalogMaker;
import edu.washington.escience.myria.coordinator.catalog.WorkerCatalog;
import edu.washington.escience.myria.daemon.MasterDaemon;
import edu.washington.escience.myria.parallel.Server;
import edu.washington.escience.myria.parallel.SocketInfo;
import edu.washington.escience.myria.parallel.Worker;
import edu.washington.escience.myria.storage.TupleBatch;
import edu.washington.escience.myria.storage.TupleBatchBuffer;
import edu.washington.escience.myria.util.FSUtils;
import edu.washington.escience.myria.util.SQLiteUtils;
import edu.washington.escience.myria.util.TestUtils;
import edu.washington.escience.myria.util.Tuple;

public class SystemTestBase {

  @Rule
  public TestRule watcher = new TestWatcher() {
    @Override
    protected void starting(Description description) {
      LOGGER.warn("*********************************************");
      LOGGER.warn(String.format("Starting test: %s()...", description.getMethodName()));
      LOGGER.warn("*********************************************");
    };
  };

  /** Automatically fail system tests that take longer than this many milliseconds. */
  private final int SYSTEM_TEST_TIMEOUT_MILLIS = 120 * 1000;
  @Rule
  public TestRule globalTimeout = new Timeout(SYSTEM_TEST_TIMEOUT_MILLIS);
  @Rule
  public TestName name = new TestName();

  /** The logger for this class. */
  protected static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(SystemTestBase.class);

  public static final Schema JOIN_INPUT_SCHEMA = new Schema(ImmutableList.of(Type.LONG_TYPE, Type.STRING_TYPE),
      ImmutableList.of("id", "name"));
  public static final RelationKey JOIN_TEST_TABLE_1 = RelationKey.of("test", "test", "testtable1");
  public static final RelationKey JOIN_TEST_TABLE_2 = RelationKey.of("test", "test", "testtable2");

  public static final int MASTER_ID = 0;

  public static final int DEFAULT_MASTER_PORT_ = 8001;

  public static final int DEFAULT_REST_PORT = 8753;

  public int WORKER_BOOTUP_TIMEOUT_IN_SECOND = 10; // wait for 10 seconds for workers to get booted

  public volatile int masterPort;
  public volatile int masterDaemonPort = DEFAULT_REST_PORT;

  public static final int DEFAULT_WORKER_STARTING_PORT = 9001;

  public static Process SERVER_PROCESS;

  public volatile int[] workerIDs;
  public volatile int[] workerPorts;
  public volatile Process[] workerProcess;
  public volatile Thread[] workerStdoutReader;

  public volatile static String workerTestBaseFolder;

  /** Whether to run the worker and master daemons in debug mode. */
  public static final boolean DEBUG = false;
  /** How much memory the system tests might use. */
  public static final String MEMORY = "512M";

  public static void createTable(final int workerID, final RelationKey relationKey, final String sqlSchemaString)
      throws IOException, CatalogException {
    try {
      SQLiteUtils.createTable(getAbsoluteDBFile(workerID).getAbsolutePath(), relationKey, sqlSchemaString, true, true);
    } catch (SQLiteException e) {
      throw new CatalogException(e);
    }<commit_msg>SystemTestBase: up worker bootup timeout<commit_after>package edu.washington.escience.myria.systemtest;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.ProcessBuilder.Redirect;
import java.lang.management.ManagementFactory;
import java.net.HttpURLConnection;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.mina.util.AvailablePortFinder;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.rules.TestName;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.rules.Timeout;
import org.junit.runner.Description;
import org.slf4j.LoggerFactory;

import com.almworks.sqlite4java.SQLiteException;
import com.fasterxml.jackson.databind.ObjectReader;
import com.google.common.collect.ImmutableList;
import com.google.protobuf.ByteString;

import edu.washington.escience.myria.DbException;
import edu.washington.escience.myria.MyriaConstants;
import edu.washington.escience.myria.MyriaSystemConfigKeys;
import edu.washington.escience.myria.RelationKey;
import edu.washington.escience.myria.Schema;
import edu.washington.escience.myria.Type;
import edu.washington.escience.myria.accessmethod.ConnectionInfo;
import edu.washington.escience.myria.accessmethod.SQLiteAccessMethod;
import edu.washington.escience.myria.accessmethod.SQLiteInfo;
import edu.washington.escience.myria.api.MyriaJsonMapperProvider;
import edu.washington.escience.myria.api.encoding.DatasetStatus;
import edu.washington.escience.myria.api.encoding.QueryStatusEncoding;
import edu.washington.escience.myria.coordinator.catalog.CatalogException;
import edu.washington.escience.myria.coordinator.catalog.CatalogMaker;
import edu.washington.escience.myria.coordinator.catalog.WorkerCatalog;
import edu.washington.escience.myria.daemon.MasterDaemon;
import edu.washington.escience.myria.parallel.Server;
import edu.washington.escience.myria.parallel.SocketInfo;
import edu.washington.escience.myria.parallel.Worker;
import edu.washington.escience.myria.storage.TupleBatch;
import edu.washington.escience.myria.storage.TupleBatchBuffer;
import edu.washington.escience.myria.util.FSUtils;
import edu.washington.escience.myria.util.SQLiteUtils;
import edu.washington.escience.myria.util.TestUtils;
import edu.washington.escience.myria.util.Tuple;

public class SystemTestBase {

  @Rule
  public TestRule watcher = new TestWatcher() {
    @Override
    protected void starting(Description description) {
      LOGGER.warn("*********************************************");
      LOGGER.warn(String.format("Starting test: %s()...", description.getMethodName()));
      LOGGER.warn("*********************************************");
    };
  };

  /** Automatically fail system tests that take longer than this many milliseconds. */
  private final int SYSTEM_TEST_TIMEOUT_MILLIS = 120 * 1000;
  @Rule
  public TestRule globalTimeout = new Timeout(SYSTEM_TEST_TIMEOUT_MILLIS);
  @Rule
  public TestName name = new TestName();

  /** The logger for this class. */
  protected static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(SystemTestBase.class);

  public static final Schema JOIN_INPUT_SCHEMA = new Schema(ImmutableList.of(Type.LONG_TYPE, Type.STRING_TYPE),
      ImmutableList.of("id", "name"));
  public static final RelationKey JOIN_TEST_TABLE_1 = RelationKey.of("test", "test", "testtable1");
  public static final RelationKey JOIN_TEST_TABLE_2 = RelationKey.of("test", "test", "testtable2");

  public static final int MASTER_ID = 0;

  public static final int DEFAULT_MASTER_PORT_ = 8001;

  public static final int DEFAULT_REST_PORT = 8753;

  public int WORKER_BOOTUP_TIMEOUT_IN_SECOND = 30; // wait up to 30 seconds for workers to get booted

  public volatile int masterPort;
  public volatile int masterDaemonPort = DEFAULT_REST_PORT;

  public static final int DEFAULT_WORKER_STARTING_PORT = 9001;

  public static Process SERVER_PROCESS;

  public volatile int[] workerIDs;
  public volatile int[] workerPorts;
  public volatile Process[] workerProcess;
  public volatile Thread[] workerStdoutReader;

  public volatile static String workerTestBaseFolder;

  /** Whether to run the worker and master daemons in debug mode. */
  public static final boolean DEBUG = false;
  /** How much memory the system tests might use. */
  public static final String MEMORY = "512M";

  public static void createTable(final int workerID, final RelationKey relationKey, final String sqlSchemaString)
      throws IOException, CatalogException {
    try {
      SQLiteUtils.createTable(getAbsoluteDBFile(workerID).getAbsolutePath(), relationKey, sqlSchemaString, true, true);
    } catch (SQLiteException e) {
      throw new CatalogException(e);
    }
{'fraction_non_alphanumeric': 0.11155040948884495, 'fraction_numerical': 0.006401204932693213, 'mean_word_length': 5.61519302615193, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2371885', 'n_tokens_mistral': 3328, 'n_tokens_neox': 3190, 'n_words': 699}