<commit_before>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include "vast/ids.hpp"
#include "vast/concept/parseable/to.hpp"
#include "vast/concept/parseable/vast/expression.hpp"

#include "vast/system/partition.hpp"
#include "vast/system/task.hpp"

#define SUITE system
#include "test.hpp"
#include "fixtures/actor_system_and_events.hpp"

using namespace caf;
using namespace vast;
using namespace std::chrono;

namespace {

struct partition_fixture : fixtures::actor_system_and_events {
  partition_fixture() {
    directory /= "partition";
    MESSAGE("ingesting conn.log");
    partition = self->spawn(system::partition, directory);
    self->send(partition, bro_conn_log);
    MESSAGE("ingesting http.log");
    self->send(partition, bro_http_log);
    MESSAGE("ingesting bgpdump log");
    self->send(partition, bgpdump_txt);
    MESSAGE("completed ingestion");
  }

  ~partition_fixture() {
    self->send(partition, system::shutdown_atom::value);
    self->wait_for(partition);
  }

  ids query(const std::string& str) {
    MESSAGE("sending query");
    auto expr = to<expression>(str);
    REQUIRE(expr);
    ids result;
    self->request(partition, infinite, *expr).receive(
      [&](ids& hits) {
        result = std::move(hits);
      },
      error_handler()
    );
    MESSAGE("shutting down partition");
    self->send(partition, system::shutdown_atom::value);
    self->wait_for(partition);
    REQUIRE(exists(directory));
    REQUIRE(exists(directory / "547119946" / "data" / "id" / "orig_h"));
    REQUIRE(exists(directory / "547119946" / "meta" / "time"));
    REQUIRE(exists(directory / "547119946" / "meta" / "type"));
    MESSAGE("respawning partition and sending query again");
    partition = self->spawn(system::partition, directory);
    self->request(partition, infinite, *expr).receive(
      [&](const ids& hits) {
        REQUIRE_EQUAL(hits, result);
      },
      error_handler()
    );
    return result;
  }

  actor partition;
};

} // namespace <anonymous>

FIXTURE_SCOPE(partition_tests, partition_fixture)

TEST(partition queries - type extractors) {
  auto hits = query(":string == \"SF\" && :port == 443/?");
  CHECK_EQUAL(rank(hits), 38u);
  hits = query(":subnet in 192.168.127.12/23");
  CHECK_EQUAL(rank(hits), 72u);
}

TEST(partition queries - key extractors) {
  auto hits = query("conn_state == \"SF\" && id.resp_p == 443/?");
  CHECK_EQUAL(rank(hits), 38u);
}

TEST(partition queries - attribute extractors) {
  MESSAGE("&type");
  auto hits = query("&type == \"bro::http\"");
  CHECK_EQUAL(rank(hits), 4896u);
  hits = query("&type == \"bro::conn\"");
  CHECK_EQUAL(rank(hits), 8462u);
  MESSAGE("&time");
  hits = query("&time > 1970-01-01");
  CHECK_EQUAL(rank(hits), 4896u + 8462u);
}

TEST(partition queries - mixed) {
  auto hits = query("service == \"http\" && :addr == 172.16.17.32");
  CHECK_EQUAL(rank(hits), 28u);
}

FIXTURE_SCOPE_END()<commit_msg>Fix broken partition unit test<commit_after>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include "vast/ids.hpp"
#include "vast/concept/parseable/to.hpp"
#include "vast/concept/parseable/vast/expression.hpp"

#include "vast/system/partition.hpp"
#include "vast/system/task.hpp"

#define SUITE system
#include "test.hpp"
#include "fixtures/actor_system_and_events.hpp"

using namespace caf;
using namespace vast;
using namespace std::chrono;

namespace {

struct partition_fixture : fixtures::actor_system_and_events {
  partition_fixture() {
    directory /= "partition";
    MESSAGE("ingesting conn.log");
    partition = self->spawn(system::partition, directory);
    self->send(partition, bro_conn_log);
    MESSAGE("ingesting http.log");
    self->send(partition, bro_http_log);
    MESSAGE("ingesting bgpdump log");
    self->send(partition, bgpdump_txt);
    MESSAGE("completed ingestion");
  }

  ~partition_fixture() {
    self->send(partition, system::shutdown_atom::value);
    self->wait_for(partition);
  }

  ids query(const std::string& str) {
    MESSAGE("sending query");
    auto expr = to<expression>(str);
    REQUIRE(expr);
    ids result;
    self->request(partition, infinite, *expr).receive(
      [&](ids& hits) {
        result = std::move(hits);
      },
      error_handler()
    );
    MESSAGE("shutting down partition");
    self->send(partition, system::shutdown_atom::value);
    self->wait_for(partition);
    REQUIRE(exists(directory));
    REQUIRE(exists(directory / "547119946" / "data" / "id" / "orig_h"));
    REQUIRE(exists(directory / "547119946" / "meta" / "time"));
    REQUIRE(exists(directory / "547119946" / "meta" / "type"));
    MESSAGE("respawning partition and sending query again");
    partition = self->spawn(system::partition, directory);
    self->request(partition, infinite, *expr).receive(
      [&](const ids& hits) {
        REQUIRE_EQUAL(hits, result);
      },
      error_handler()
    );
    return result;
  }

  actor partition;
};

} // namespace <anonymous>

FIXTURE_SCOPE(partition_tests, partition_fixture)

TEST(partition queries - type extractors) {
  auto hits = query(":string == \"SF\" && :port == 443/?");
  CHECK_EQUAL(rank(hits), 38u);
  hits = query(":subnet in 192.168.127.12/23");
  CHECK_EQUAL(rank(hits), 72u);
}

TEST(partition queries - key extractors) {
  auto hits = query("conn_state == \"SF\" && id.resp_p == 443/?");
  CHECK_EQUAL(rank(hits), 38u);
}

TEST(partition queries - attribute extractors) {
  MESSAGE("&type");
  auto hits = query("&type == \"bro::http\"");
  CHECK_EQUAL(rank(hits), bro_http_log.size());
  hits = query("&type == \"bro::conn\"");
  CHECK_EQUAL(rank(hits), bro_conn_log.size());
  MESSAGE("&time");
  hits = query("&time > 1970-01-01");
  auto all = bro_http_log.size() + bro_conn_log.size() + bgpdump_txt.size();
  CHECK_EQUAL(rank(hits), all);
}

TEST(partition queries - mixed) {
  auto hits = query("service == \"http\" && :addr == 172.16.17.32");
  CHECK_EQUAL(rank(hits), 28u);
}

FIXTURE_SCOPE_END()
{'fraction_non_alphanumeric': 0.18495620814013394, 'fraction_numerical': 0.020350334878928386, 'mean_word_length': 2.8063725490196076, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6417910', 'n_tokens_mistral': 2534, 'n_tokens_neox': 2299, 'n_words': 495}