<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <<EMAIL>>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <algorithm>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/Language.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-base/util/SimpleRateLimit.h"
#include "fnord-base/InternMap.h"
#include "fnord-json/json.h"
#include "fnord-mdb/MDB.h"
#include "fnord-mdb/MDBUtil.h"
#include "fnord-sstable/sstablereader.h"
#include "fnord-sstable/sstablewriter.h"
#include "fnord-sstable/SSTableColumnSchema.h"
#include "fnord-sstable/SSTableColumnReader.h"
#include "fnord-sstable/SSTableColumnWriter.h"
#include <fnord-fts/fts.h>
#include <fnord-fts/fts_common.h>
#include "fnord-eventdb/TableReader.h"
#include "common.h"
#include "schemas.h"
#include "CustomerNamespace.h"
#include "FeatureSchema.h"
#include "JoinedQuery.h"
#include "CTRCounter.h"
#include "IndexReader.h"
#include "analytics/ReportBuilder.h"
#include "analytics/AnalyticsTableScanSource.h"
#include "analytics/JoinedQueryTableSource.h"
#include "analytics/CTRByPositionMapper.h"
#include "analytics/CTRByPageMapper.h"
#include "analytics/CTRBySearchQueryMapper.h"
#include "analytics/CTRByQueryAttributeMapper.h"
#include "analytics/CTRBySearchTermCrossCategoryMapper.h"
#include "analytics/CTRStatsMapper.h"
#include "analytics/CTRCounterMergeReducer.h"
#include "analytics/CTRCounterTableSink.h"
#include "analytics/CTRCounterTableSource.h"
#include "analytics/RelatedTermsMapper.h"
#include "analytics/TopCategoriesByTermMapper.h"
#include "analytics/TermInfoMergeReducer.h"

using namespace fnord;
using namespace cm;

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  //flags.defineFlag(
  //    "conf",
  //    cli::FlagParser::T_STRING,
  //    false,
  //    NULL,
  //    "./conf",
  //    "conf directory",
  //    "<path>");

  flags.defineFlag(
      "index",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "index directory",
      "<path>");

  flags.defineFlag(
      "replica",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "replica id",
      "<id>");

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "artifact directory",
      "<path>");

  flags.defineFlag(
      "tempdir",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "artifact directory",
      "<path>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto index_path = flags.getString("index");
  //auto conf_path = flags.getString("conf");
  auto tempdir = flags.getString("tempdir");
  auto datadir = flags.getString("datadir");

  /* open index */
  auto index_reader = cm::IndexReader::openIndex(index_path);
  //auto analyzer = RefPtr<fts::Analyzer>(new fts::Analyzer(conf_path));

  /* set up reportbuilder */
  cm::ReportBuilder report_builder;

  auto buildid = 0;

  auto table = eventdb::TableReader::open(
      "dawanda_joined_sessions",
      flags.getString("replica"),
      flags.getString("datadir"),
      joinedSessionsSchema());

  auto snap = table->getSnapshot();

  Set<String> searchterm_x_e1_tables;
  Set<String> related_terms_tables;

  for (const auto& c : snap->head->chunks) {
    auto input_table = StringUtil::format(
        "$0/$1.$2.$3.cst",
        datadir,
        "dawanda_joined_sessions",
        c.replica_id,
        c.chunk_id);

    /* map related terms */
    auto related_terms_table = StringUtil::format(
        "$0/dawanda_related_terms.$1.$2.sst",
        tempdir,
        c.replica_id,
        c.chunk_id);

    related_terms_tables.emplace(related_terms_table);
    report_builder.addReport(
        new RelatedTermsMapper(
            new AnalyticsTableScanSource(input_table),
            new TermInfoTableSink(related_terms_table)));

    /* map serchterm x e1 */
    auto searchterm_x_e1_table = StringUtil::format(
        "$0/dawanda_ctr_by_searchterm_cross_e1.$1.$2.sst",
        tempdir,
        c.replica_id,
        c.chunk_id);

    searchterm_x_e1_tables.emplace(searchterm_x_e1_table);
    report_builder.addReport(
        new CTRBySearchTermCrossCategoryMapper(
            new AnalyticsTableScanSource(input_table),
            new CTRCounterTableSink(0, 0, searchterm_x_e1_table),
            "category1",
            index_reader));
  }<commit_msg>build autocomplete from new table<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <<EMAIL>>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <algorithm>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/Language.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-base/util/SimpleRateLimit.h"
#include "fnord-base/InternMap.h"
#include "fnord-json/json.h"
#include "fnord-mdb/MDB.h"
#include "fnord-mdb/MDBUtil.h"
#include "fnord-sstable/sstablereader.h"
#include "fnord-sstable/sstablewriter.h"
#include "fnord-sstable/SSTableColumnSchema.h"
#include "fnord-sstable/SSTableColumnReader.h"
#include "fnord-sstable/SSTableColumnWriter.h"
#include <fnord-fts/fts.h>
#include <fnord-fts/fts_common.h>
#include "fnord-eventdb/TableReader.h"
#include "common.h"
#include "schemas.h"
#include "CustomerNamespace.h"
#include "FeatureSchema.h"
#include "JoinedQuery.h"
#include "CTRCounter.h"
#include "IndexReader.h"
#include "analytics/ReportBuilder.h"
#include "analytics/AnalyticsTableScanSource.h"
#include "analytics/JoinedQueryTableSource.h"
#include "analytics/CTRByPositionMapper.h"
#include "analytics/CTRByPageMapper.h"
#include "analytics/CTRBySearchQueryMapper.h"
#include "analytics/CTRByQueryAttributeMapper.h"
#include "analytics/CTRBySearchTermCrossCategoryMapper.h"
#include "analytics/CTRStatsMapper.h"
#include "analytics/CTRCounterMergeReducer.h"
#include "analytics/CTRCounterTableSink.h"
#include "analytics/CTRCounterTableSource.h"
#include "analytics/RelatedTermsMapper.h"
#include "analytics/TopCategoriesByTermMapper.h"
#include "analytics/TermInfoMergeReducer.h"

using namespace fnord;
using namespace cm;

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  //flags.defineFlag(
  //    "conf",
  //    cli::FlagParser::T_STRING,
  //    false,
  //    NULL,
  //    "./conf",
  //    "conf directory",
  //    "<path>");

  //flags.defineFlag(
  //    "index",
  //    cli::FlagParser::T_STRING,
  //    false,
  //    NULL,
  //    NULL,
  //    "index directory",
  //    "<path>");

  flags.defineFlag(
      "replica",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "replica id",
      "<id>");

  flags.defineFlag(
      "datadir",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "artifact directory",
      "<path>");

  flags.defineFlag(
      "tempdir",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "artifact directory",
      "<path>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  //auto index_path = flags.getString("index");
  //auto conf_path = flags.getString("conf");
  auto tempdir = flags.getString("tempdir");
  auto datadir = flags.getString("datadir");

  /* open index */
  //auto index_reader = cm::IndexReader::openIndex(index_path);
  //auto analyzer = RefPtr<fts::Analyzer>(new fts::Analyzer(conf_path));

  /* set up reportbuilder */
  cm::ReportBuilder report_builder;

  auto buildid = 0;

  auto table = eventdb::TableReader::open(
      "joined_sessions-dawanda",
      flags.getString("replica"),
      flags.getString("datadir"),
      joinedSessionsSchema());

  auto snap = table->getSnapshot();

  Set<String> searchterm_x_e1_tables;
  Set<String> related_terms_tables;

  for (const auto& c : snap->head->chunks) {
    auto input_table = StringUtil::format(
        "$0/$1.$2.$3.cst",
        datadir,
        "joined_sessions-dawanda",
        c.replica_id,
        c.chunk_id);

    /* map related terms */
    auto related_terms_table = StringUtil::format(
        "$0/dawanda_related_terms.$1.$2.sst",
        tempdir,
        c.replica_id,
        c.chunk_id);

    related_terms_tables.emplace(related_terms_table);
    report_builder.addReport(
        new RelatedTermsMapper(
            new AnalyticsTableScanSource(input_table),
            new TermInfoTableSink(related_terms_table)));

    /* map serchterm x e1 */
    auto searchterm_x_e1_table = StringUtil::format(
        "$0/dawanda_ctr_by_searchterm_cross_e1.$1.$2.sst",
        tempdir,
        c.replica_id,
        c.chunk_id);

    searchterm_x_e1_tables.emplace(searchterm_x_e1_table);
    report_builder.addReport(
        new CTRBySearchTermCrossCategoryMapper(
            new AnalyticsTableScanSource(input_table),
            new CTRCounterTableSink(0, 0, searchterm_x_e1_table),
            "category1"));
  }
{'fraction_non_alphanumeric': 0.13830855765367617, 'fraction_numerical': 0.0050220972278023305, 'mean_word_length': 3.517695099818512, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1434398', 'n_tokens_mistral': 3472, 'n_tokens_neox': 3391, 'n_words': 589}