<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <<EMAIL>>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <algorithm>
#include <iostream>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-fts/QueryAnalyzer.h"
#include "fnord-fts/GermanStemmer.h"
#include "common.h"

using namespace fnord;

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "lang",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "language",
      "<lang>");

  flags.defineFlag(
      "stopwords",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "stopwords",
      "<path>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto lang = languageFromString(flags.getString("lang"));

  fts::StopwordDictionary stopwords;
  if (flags.isSet("stopwords")) {
    stopwords.loadStopwordFile(flags.getString("stopwords"));
  } else {
    fnord::logWarning("cm-tokenize", "no stopword file provided");
  }

  fnord::fts::SynonymDictionary synonyms;
  synonyms.addSynonym(Language::DE, "mützen", "mütze");
  synonyms.addSynonym(Language::DE, "bänder", "band");

  fnord::fts::GermanStemmer german_stemmer(
      "conf/hunspell_de.aff",
      "conf/hunspell_de.dic",
      "conf/hunspell_de.hyphen",
      &synonyms);

  fts::QueryAnalyzer analyzer(&stopwords, &german_stemmer);
  for (String line; std::getline(std::cin, line); ) {
    Set<String> tokens;
    analyzer.analyze(lang, line, &tokens);
    fnord::iputs("\ninput: $0\noutput: $1", line, tokens);
  }

  return 0;
}
<commit_msg>ignore utf8 dingbats in tokenization<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <<EMAIL>>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include <algorithm>
#include <iostream>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include "fnord-base/io/fileutil.h"
#include "fnord-base/application.h"
#include "fnord-base/logging.h"
#include "fnord-base/cli/flagparser.h"
#include "fnord-fts/QueryAnalyzer.h"
#include "fnord-fts/GermanStemmer.h"
#include "common.h"

using namespace fnord;

int main(int argc, const char** argv) {
  fnord::Application::init();
  fnord::Application::logToStderr();

  fnord::cli::FlagParser flags;

  flags.defineFlag(
      "lang",
      cli::FlagParser::T_STRING,
      true,
      NULL,
      NULL,
      "language",
      "<lang>");

  flags.defineFlag(
      "stopwords",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "stopwords",
      "<path>");

  flags.defineFlag(
      "loglevel",
      fnord::cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  auto lang = languageFromString(flags.getString("lang"));

  fts::StopwordDictionary stopwords;
  if (flags.isSet("stopwords")) {
    stopwords.loadStopwordFile(flags.getString("stopwords"));
  } else {
    fnord::logWarning("cm-tokenize", "no stopword file provided");
  }

  fnord::fts::SynonymDictionary synonyms;
  synonyms.addSynonym(Language::DE, "mützen", "mütze");
  synonyms.addSynonym(Language::DE, "bänder", "band");
  synonyms.addSynonym(Language::DE, "girlanden", "girlande");

  fnord::fts::GermanStemmer german_stemmer(
      "conf/hunspell_de.aff",
      "conf/hunspell_de.dic",
      "conf/hunspell_de.hyphen",
      &synonyms);

  fts::QueryAnalyzer analyzer(&stopwords, &german_stemmer);
  for (String line; std::getline(std::cin, line); ) {
    Set<String> tokens;
    analyzer.analyze(lang, line, &tokens);
    fnord::iputs("\ninput: $0\noutput: $1", line, tokens);
  }

  return 0;
}

{'fraction_non_alphanumeric': 0.15596534103532547, 'fraction_numerical': 0.0033325927571650742, 'mean_word_length': 3.82529474812433, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1025063', 'n_tokens_mistral': 1609, 'n_tokens_neox': 1540, 'n_words': 303}