<commit_before>package org.thunlp.learning.lda;

import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.thunlp.misc.Flags;
import org.thunlp.tool.FolderReader;
import org.thunlp.tool.GenericTool;

import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.logging.Logger;

/**
 * The main entry for the distributed trainer.
 *
 * @author <NAME> (<EMAIL>)
 */
public class LdaTrainer implements GenericTool {
    Logger LOG = Logger.getAnonymousLogger();
    long startTime = 0;

    public void run(String[] args) throws Exception {
        startTime = System.currentTimeMillis();
        GibbsSamplingTool sampler = new GibbsSamplingTool();
        InitModelTool initializer = new InitModelTool();

        Flags flags = new Flags();
        flags.add("input", "input documents, each is space-separated words.");
        flags.add("output", "the final model, a plain text file.");
        flags.add("working_dir", "temporary directory to hold intermediate files.");
        flags.add("num_topics", "number of topics.");
        flags.add("num_iterations", "number of iterations.");
        flags.addWithDefaultValue(
                "alpha", "-1",
                "symmetric hyper-parameter alpha. [default k/50]");
        flags.addWithDefaultValue(
                "beta", "0.01",
                "symmetric hyper-parameter beta. [default 0.01]");
        flags.addWithDefaultValue(
                "iterations_to_keep", "10",
                "number of iterations to keep on disk, " +
                        "and used for final model. [default 10]");
        flags.addWithDefaultValue(<commit_msg>DOC: add some comments to LdaTrainer as I read through it<commit_after>package org.thunlp.learning.lda;

import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.thunlp.misc.Flags;
import org.thunlp.tool.FolderReader;
import org.thunlp.tool.GenericTool;

import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.logging.Logger;

/**
 * The main entry for the distributed trainer.
 *
 * @author <NAME> (<EMAIL>)
 */
public class LdaTrainer implements GenericTool {
    Logger LOG = Logger.getAnonymousLogger();
    long startTime = 0;

    public void run(String[] args) throws Exception {
        // init sampler and model initializer
        startTime = System.currentTimeMillis();
        GibbsSamplingTool sampler = new GibbsSamplingTool();
        InitModelTool initializer = new InitModelTool();

        // parse input
        Flags flags = new Flags();
        flags.add("input", "input documents, each is space-separated words.");
        flags.add("output", "the final model, a plain text file.");
        flags.add("working_dir", "temporary directory to hold intermediate files.");
        flags.add("num_topics", "number of topics.");
        flags.add("num_iterations", "number of iterations.");
        flags.addWithDefaultValue(
                "alpha", "-1",
                "symmetric hyper-parameter alpha. [default k/50]");
        flags.addWithDefaultValue(
                "beta", "0.01",
                "symmetric hyper-parameter beta. [default 0.01]");
        flags.addWithDefaultValue(
                "iterations_to_keep", "10",
                "number of iterations to keep on disk, " +
                        "and used for final model. [default 10]");
        flags.addWithDefaultValue(
{'fraction_non_alphanumeric': 0.11566325256057956, 'fraction_numerical': 0.0069947539345490885, 'mean_word_length': 3.004, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2973094', 'n_tokens_mistral': 1163, 'n_tokens_neox': 1108, 'n_words': 294}