<commit_before>//
// Command-line tool
//
// Copyright(C) 2009  <NAME> <<EMAIL>>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; version 2 of the License.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
//

#include <ctime>
#include <fstream>
#include <iostream>
#include <map>
#include <utility>
#include "cluster.h"
#include <unistd.h> 

typedef std::map<std::string, double> Feature;

/* global variables */
const std::string DELIMITER = "\t";
const size_t MAX_VECTOR_ITEM = 50;
std::string g_progname;  // program name

/* function prototypes */
int main(int argc, char **argv);
static void usage();
static size_t parse_tsv(std::string &tsv, Feature &feature, size_t max);
static size_t add_documents(std::ifstream &ifs, bayon::Analyzer &analyzer,
                            std::map<int, std::string> &docidmap);
static void parse_options(int argc, char **argv,
                          std::map<std::string, std::string> &option);

/* main function */
int main(int argc, char **argv) {
  g_progname = argv[0];
  std::map<std::string, std::string> option;
  parse_options(argc, argv, option);
  if (option.find("input") == option.end()
      || (option.find("number") == option.end()
          && option.find("limit") == option.end())) {
    usage();
    return 1;
  }

  srand((unsigned int)time(NULL));
  bayon::Analyzer analyzer;
  std::map<int, std::string> docidmap;

  std::ifstream ifs(option["input"].c_str());
  if (!ifs) {
    std::cerr << "[ERROR]File not found: " << option["input"] << std::endl;
    return 1;
  }
  add_documents(ifs, analyzer, docidmap);
  if (option.find("number") != option.end()) {
    analyzer.set_cluster_size_limit(atoi(option["number"].c_str()));
  } else if (option.find("limit") != option.end()) {
    analyzer.set_eval_limit(atof(option["limit"].c_str()));
  }
  std::string method = "rb"; // default
  if (option.find("method") != option.end()) method = option["method"];
  analyzer.do_clustering(method);

  bayon::Cluster cluster;
  size_t cluster_count = 1;
  while (analyzer.get_next_result(cluster)) {
    if (cluster.size() > 0) {
      std::cout << cluster_count++ << "\t";
      for (size_t i = 0; i < cluster.documents().size(); i++) {
        if (i > 0) std::cout << " ";
        std::cout << docidmap[cluster.documents()[i]->id()];
      }
      std::cout << std::endl;
    }
  }
  
  return 0;
}

/* show usage */
static void usage() {
  std::cerr
    << g_progname << ": Clustering Tool" << std::endl
    << "Usage:" << std::endl
    << " " << g_progname << " -n num -i file [-m method]" << std::endl
    << " " << g_progname << " -l limit -i file [-m method]" << std::endl
    << "    -n, --number num    ... number of clusters" << std::endl
    << "    -l, --limit lim     ... limit value of cluster bisection" << std::endl<commit_msg>[trunk] modify commandline tool<commit_after>//
// Command-line tool
//
// Copyright(C) 2009  <NAME> <<EMAIL>>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; version 2 of the License.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
//

#include <ctime>
#include <fstream>
#include <iostream>
#include <map>
#include <utility>
#include "cluster.h"
#include <unistd.h> 

typedef std::map<std::string, double> Feature;

/* global variables */
const std::string DELIMITER("\t");
const size_t MAX_VECTOR_ITEM = 50;

/* function prototypes */
int main(int argc, char **argv);
static void usage(std::string progname);
static size_t parse_tsv(std::string &tsv, Feature &feature, size_t max);
static size_t add_documents(std::ifstream &ifs, bayon::Analyzer &analyzer,
                            std::map<int, std::string> &docidmap);
static void parse_options(int argc, char **argv,
                          std::map<std::string, std::string> &option);

/* main function */
int main(int argc, char **argv) {
  std::string progname(argv[0]);
  std::map<std::string, std::string> option;
  parse_options(argc, argv, option);
  if (option.find("input") == option.end()
      || (option.find("number") == option.end()
          && option.find("limit") == option.end())) {
    usage(progname);
    return 1;
  }

  srand((unsigned int)time(NULL));
  bayon::Analyzer analyzer;
  std::map<int, std::string> docidmap;

  std::ifstream ifs(option["input"].c_str());
  if (!ifs) {
    std::cerr << "[ERROR]File not found: " << option["input"] << std::endl;
    return 1;
  }
  add_documents(ifs, analyzer, docidmap);
  if (option.find("number") != option.end()) {
    analyzer.set_cluster_size_limit(atoi(option["number"].c_str()));
  } else if (option.find("limit") != option.end()) {
    analyzer.set_eval_limit(atof(option["limit"].c_str()));
  }
  std::string method = "rb"; // default
  if (option.find("method") != option.end()) method = option["method"];
  analyzer.do_clustering(method);

  bayon::Cluster cluster;
  size_t cluster_count = 1;
  while (analyzer.get_next_result(cluster)) {
    if (cluster.size() > 0) {
      std::cout << cluster_count++ << "\t";
      for (size_t i = 0; i < cluster.documents().size(); i++) {
        if (i > 0) std::cout << " ";
        std::cout << docidmap[cluster.documents()[i]->id()];
      }
      std::cout << std::endl;
    }
  }
  
  return 0;
}

/* show usage */
static void usage(std::string progname) {
  std::cerr
    << progname << ": Clustering Tool" << std::endl
    << "Usage:" << std::endl
    << " " << progname << " -n num -i file [-m method]" << std::endl
    << " " << progname << " -l limit -i file [-m method]" << std::endl
    << "    -n, --number num    ... number of clusters" << std::endl
    << "    -l, --limit lim     ... limit value of cluster bisection" << std::endl
{'fraction_non_alphanumeric': 0.1704630043174036, 'fraction_numerical': 0.007741551287777281, 'mean_word_length': 3.3994760969220694, 'pattern_counts': {'":': 2, '<': 133, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '428994', 'n_tokens_mistral': 2218, 'n_tokens_neox': 2128, 'n_words': 639}