<commit_before>package txtfnnl.uima.collection;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.util.HashMap;

import org.apache.uima.UIMAException;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_component.AnalysisComponent;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.CASException;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.util.Level;
import org.apache.uima.util.Logger;

import org.uimafit.component.CasConsumer_ImplBase;
import org.uimafit.descriptor.ConfigurationParameter;
import org.uimafit.factory.AnalysisEngineFactory;

import txtfnnl.uima.AnalysisComponentBuilder;
import txtfnnl.uima.UIMAUtils;
import txtfnnl.uima.Views;
import txtfnnl.utils.IOUtils;

/**
 * A CAS consumer that writes the document text (only).
 * <p>
 * Note that on <b>Apple OSX</b> the default encoding would be <i>MacRoman</i>; however, this
 * consumer uses either the encoding defined by the <code>LANG</code> environment variable or
 * otherwise defaults to <b>UTF-8</b> as a far more sensible encoding instead. The output can be
 * written to individual files or STDOUT.
 * 
 * @author <NAME>
 */
public class TextWriter extends CasConsumer_ImplBase {
  /**
   * Optional configuration parameter String that defines the path to a directory where the output
   * files will be written. Note that the directory will be created if it does not exist.
   */
  public static final String PARAM_OUTPUT_DIRECTORY = "OutputDirectory";
  @ConfigurationParameter(name = PARAM_OUTPUT_DIRECTORY)
  protected File outputDirectory;
  /**
   * Force a particular output encoding. By default, the system's default encoding is used.
   */
  public static final String PARAM_ENCODING = "Encoding";
  @ConfigurationParameter(name = PARAM_ENCODING)
  protected String encoding;
  /**
   * Optional flag leading to the overwriting any existing files; defaults to <code>false</code>.
   * <p>
   * By inserting ".<b>n</b>" between the file name and its new suffix, the file name is made
   * unique; otherwise the existing file is replaced (where <b>n</b> is the first positive integer
   * that makes the file name "unique").
   */
  public static final String PARAM_OVERWRITE_FILES = "OverwriteFiles";
  @ConfigurationParameter(name = PARAM_OVERWRITE_FILES, defaultValue = "false")
  protected boolean overwriteFiles;
  /**
   * If <code>true</code>, the output will (also) be written to STDOUT. By default, text is not
   * written to STDOUT.
   */
  public static final String PARAM_PRINT_TO_STDOUT = "PrintToStdout";
  @ConfigurationParameter(name = PARAM_PRINT_TO_STDOUT, defaultValue = "false")
  protected Boolean printToStdout;

  public static class Builder extends AnalysisComponentBuilder {
    protected Builder(Class<? extends AnalysisComponent> klass) {
      super(klass);
      setOptionalParameter(PARAM_PRINT_TO_STDOUT, true);
    }

    public Builder() {
      super(TextWriter.class);
      setOptionalParameter(PARAM_PRINT_TO_STDOUT, true);
    }

    public Builder setOutputDirectory(File outputDirectory) {
      setOptionalParameter(PARAM_OUTPUT_DIRECTORY, outputDirectory);
      if (outputDirectory == null) setOptionalParameter(PARAM_PRINT_TO_STDOUT, true);
      else setOptionalParameter(PARAM_PRINT_TO_STDOUT, false);
      return this;
    }

    public Builder setEncoding(String encoding) {
      setOptionalParameter(PARAM_ENCODING, encoding);
      return this;
    }

    public Builder overwriteFiles() {
      setOptionalParameter(PARAM_OVERWRITE_FILES, true);
      return this;
    }
  }

  /**
   * Configure a TextWriter descriptor.
   * <p>
   * Note that if the {@link #outputDirectory} is <code>null</code> and {@link #printToStdout} is
   * <code>false</code>, a {@link ResourceInitializationException} will occur when creating the AE.
   * 
   * @param outputDirectory path to the output directory (or null)
   * @param encoding encoding to use for writing (or null)
   * @param printToStdout whether to print to STDOUT or not
   * @param overwriteFiles whether to overwrite existing files or not
   * @return a configured AE description
   * @throws IOException
   * @throws UIMAException
   */
  @SuppressWarnings("serial")
  public static AnalysisEngineDescription configure(final File outputDirectory,
      final String encoding, final boolean printToStdout, final boolean overwriteFiles)
      throws UIMAException, IOException {
    return AnalysisEngineFactory.createPrimitiveDescription(TextWriter.class,
        UIMAUtils.makeParameterArray(new HashMap<String, Object>() {
          {
            put(PARAM_OUTPUT_DIRECTORY, outputDirectory);
            put(PARAM_ENCODING, encoding);
            put(PARAM_PRINT_TO_STDOUT, printToStdout);
            put(PARAM_OVERWRITE_FILES, overwriteFiles);
          }
        }));
  }

  /**
   * Configure a default TextWriter descriptor for a pipeline. Writes to STDOUT using the system's
   * default encoding.
   * 
   * @return a configured AE description
   * @throws IOException
   * @throws UIMAException
   */
  public static AnalysisEngineDescription configure() throws UIMAException, IOException {
    return TextWriter.configure(null, null, true, false);
  }

  protected int counter; // to create "new" output file names if necessary
  protected Logger logger;
  protected Writer outputWriter;

  @Override
  public void initialize(UimaContext ctx) throws ResourceInitializationException {
    super.initialize(ctx);
    logger = ctx.getLogger();
    if (outputDirectory != null && (!outputDirectory.isDirectory() || !outputDirectory.canWrite()))
      throw new ResourceInitializationException(new IOException("'" +
          outputDirectory.getAbsolutePath() + "' not a writeable directory"));
    if (!printToStdout && outputDirectory == null)
      throw new ResourceInitializationException(new AssertionError(
          "no output defined (neither directory or STDOUT specified)"));
    if (printToStdout) {
      if (encoding == null && IOUtils.isMacOSX()) {
        // fix broken Mac JDK that uses MacRoman instead of the LANG
        // setting as default encoding; if LANG is not set, use UTF-8.
        encoding = IOUtils.getLocaleEncoding();
        if (encoding == null) {
          encoding = "UTF-8";
        }
        try {
          IOUtils.setOutputEncoding(encoding);
        } catch (final UnsupportedEncodingException e) {
          throw new ResourceInitializationException(e);
        }
      } else if (encoding != null) {
        try {
          IOUtils.setOutputEncoding(encoding);
        } catch (final UnsupportedEncodingException e) {
          throw new ResourceInitializationException(e);
        }
      }
      if (encoding != null) {
        logger.log(Level.INFO, "set STDOUT to use '" + encoding + "' encoding");
      }
    }
    logger.log(Level.INFO, "initialized {0}", this.getClass().getName());
    counter = 0;
  }

  @Override
  public void process(CAS cas) throws AnalysisEngineProcessException {
    JCas textJCas;
    CAS rawCas;
    try {
      textJCas = cas.getView(Views.CONTENT_TEXT.toString()).getJCas();
      rawCas = cas.getView(Views.CONTENT_RAW.toString());
    } catch (final CASException e) {
      throw new AnalysisEngineProcessException(e);
    }
    try {
      setStream(rawCas);
      final String text = textJCas.getDocumentText();
      if (outputDirectory != null) {
        try {
          outputWriter.write(text);
        } catch (final IOException e) {
          throw new AnalysisEngineProcessException(e);
        }
      }<commit_msg>logging consumer configuration<commit_after>package txtfnnl.uima.collection;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.util.HashMap;

import org.apache.uima.UIMAException;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_component.AnalysisComponent;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.CASException;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.util.Level;
import org.apache.uima.util.Logger;

import org.uimafit.component.CasConsumer_ImplBase;
import org.uimafit.descriptor.ConfigurationParameter;
import org.uimafit.factory.AnalysisEngineFactory;

import txtfnnl.uima.AnalysisComponentBuilder;
import txtfnnl.uima.UIMAUtils;
import txtfnnl.uima.Views;
import txtfnnl.utils.IOUtils;

/**
 * A CAS consumer that writes the document text (only).
 * <p>
 * Note that on <b>Apple OSX</b> the default encoding would be <i>MacRoman</i>; however, this
 * consumer uses either the encoding defined by the <code>LANG</code> environment variable or
 * otherwise defaults to <b>UTF-8</b> as a far more sensible encoding instead. The output can be
 * written to individual files or STDOUT.
 * 
 * @author <NAME>
 */
public class TextWriter extends CasConsumer_ImplBase {
  /**
   * Optional configuration parameter String that defines the path to a directory where the output
   * files will be written. Note that the directory will be created if it does not exist.
   */
  public static final String PARAM_OUTPUT_DIRECTORY = "OutputDirectory";
  @ConfigurationParameter(name = PARAM_OUTPUT_DIRECTORY)
  protected File outputDirectory;
  /**
   * Force a particular output encoding. By default, the system's default encoding is used.
   */
  public static final String PARAM_ENCODING = "Encoding";
  @ConfigurationParameter(name = PARAM_ENCODING)
  protected String encoding;
  /**
   * Optional flag leading to the overwriting any existing files; defaults to <code>false</code>.
   * <p>
   * By inserting ".<b>n</b>" between the file name and its new suffix, the file name is made
   * unique; otherwise the existing file is replaced (where <b>n</b> is the first positive integer
   * that makes the file name "unique").
   */
  public static final String PARAM_OVERWRITE_FILES = "OverwriteFiles";
  @ConfigurationParameter(name = PARAM_OVERWRITE_FILES, defaultValue = "false")
  protected boolean overwriteFiles;
  /**
   * If <code>true</code>, the output will (also) be written to STDOUT. By default, text is not
   * written to STDOUT.
   */
  public static final String PARAM_PRINT_TO_STDOUT = "PrintToStdout";
  @ConfigurationParameter(name = PARAM_PRINT_TO_STDOUT, defaultValue = "false")
  protected Boolean printToStdout;

  public static class Builder extends AnalysisComponentBuilder {
    protected Builder(Class<? extends AnalysisComponent> klass) {
      super(klass);
      setOptionalParameter(PARAM_PRINT_TO_STDOUT, true);
    }

    public Builder() {
      super(TextWriter.class);
      setOptionalParameter(PARAM_PRINT_TO_STDOUT, true);
    }

    public Builder setOutputDirectory(File outputDirectory) {
      setOptionalParameter(PARAM_OUTPUT_DIRECTORY, outputDirectory);
      if (outputDirectory == null) setOptionalParameter(PARAM_PRINT_TO_STDOUT, true);
      else setOptionalParameter(PARAM_PRINT_TO_STDOUT, false);
      return this;
    }

    public Builder setEncoding(String encoding) {
      setOptionalParameter(PARAM_ENCODING, encoding);
      return this;
    }

    public Builder overwriteFiles() {
      setOptionalParameter(PARAM_OVERWRITE_FILES, true);
      return this;
    }
  }

  /**
   * Configure a TextWriter descriptor.
   * <p>
   * Note that if the {@link #outputDirectory} is <code>null</code> and {@link #printToStdout} is
   * <code>false</code>, a {@link ResourceInitializationException} will occur when creating the AE.
   * 
   * @param outputDirectory path to the output directory (or null)
   * @param encoding encoding to use for writing (or null)
   * @param printToStdout whether to print to STDOUT or not
   * @param overwriteFiles whether to overwrite existing files or not
   * @return a configured AE description
   * @throws IOException
   * @throws UIMAException
   */
  @SuppressWarnings("serial")
  public static AnalysisEngineDescription configure(final File outputDirectory,
      final String encoding, final boolean printToStdout, final boolean overwriteFiles)
      throws UIMAException, IOException {
    return AnalysisEngineFactory.createPrimitiveDescription(TextWriter.class,
        UIMAUtils.makeParameterArray(new HashMap<String, Object>() {
          {
            put(PARAM_OUTPUT_DIRECTORY, outputDirectory);
            put(PARAM_ENCODING, encoding);
            put(PARAM_PRINT_TO_STDOUT, printToStdout);
            put(PARAM_OVERWRITE_FILES, overwriteFiles);
          }
        }));
  }

  /**
   * Configure a default TextWriter descriptor for a pipeline. Writes to STDOUT using the system's
   * default encoding.
   * 
   * @return a configured AE description
   * @throws IOException
   * @throws UIMAException
   */
  public static AnalysisEngineDescription configure() throws UIMAException, IOException {
    return TextWriter.configure(null, null, true, false);
  }

  protected int counter; // to create "new" output file names if necessary
  protected Logger logger;
  protected Writer outputWriter;

  @Override
  public void initialize(UimaContext ctx) throws ResourceInitializationException {
    super.initialize(ctx);
    logger = ctx.getLogger();
    if (outputDirectory != null && (!outputDirectory.isDirectory() || !outputDirectory.canWrite()))
      throw new ResourceInitializationException(new IOException("'" +
          outputDirectory.getAbsolutePath() + "' not a writeable directory"));
    if (!printToStdout && outputDirectory == null)
      throw new ResourceInitializationException(new AssertionError(
          "no output defined (neither directory or STDOUT specified)"));
    if (printToStdout) {
      if (encoding == null && IOUtils.isMacOSX()) {
        // fix broken Mac JDK that uses MacRoman instead of the LANG
        // setting as default encoding; if LANG is not set, use UTF-8.
        encoding = IOUtils.getLocaleEncoding();
        if (encoding == null) {
          encoding = "UTF-8";
        }
        try {
          IOUtils.setOutputEncoding(encoding);
        } catch (final UnsupportedEncodingException e) {
          throw new ResourceInitializationException(e);
        }
      } else if (encoding != null) {
        try {
          IOUtils.setOutputEncoding(encoding);
        } catch (final UnsupportedEncodingException e) {
          throw new ResourceInitializationException(e);
        }
      }
      if (encoding != null) {
        logger.log(Level.INFO, "writing to STDOUT with '" + encoding + "' encoding");
      } else {
        logger.log(Level.INFO, "writing to STDOUT using the default encoding");
      }
    } else {
      logger.log(Level.INFO, "writing to '" + outputDirectory.getAbsolutePath() + "'");
    }
    logger.log(Level.INFO, "initialized {0}", this.getClass().getName());
    counter = 0;
  }

  @Override
  public void process(CAS cas) throws AnalysisEngineProcessException {
    JCas textJCas;
    CAS rawCas;
    try {
      textJCas = cas.getView(Views.CONTENT_TEXT.toString()).getJCas();
      rawCas = cas.getView(Views.CONTENT_RAW.toString());
    } catch (final CASException e) {
      throw new AnalysisEngineProcessException(e);
    }
    try {
      setStream(rawCas);
      final String text = textJCas.getDocumentText();
      if (outputDirectory != null) {
        try {
          outputWriter.write(text);
        } catch (final IOException e) {
          throw new AnalysisEngineProcessException(e);
        }
      }
{'fraction_non_alphanumeric': 0.09092613423322085, 'fraction_numerical': 0.0006249218847644045, 'mean_word_length': 3.6574505238649593, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '678877', 'n_tokens_mistral': 4441, 'n_tokens_neox': 4351, 'n_words': 1286}