<commit_before>/*
 *  Copyright (c) 2012-2013 Malhar, Inc.
 *  All Rights Reserved.
 */
package com.malhartech.demos.wordcount;

import com.malhartech.api.ApplicationFactory;
import com.malhartech.api.Context.OperatorContext;
import com.malhartech.api.DAG;
import com.malhartech.lib.io.ConsoleOutputOperator;
import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;

/**
 * To run the test, you need to generate a data file as sample file.
 * You can use "hadoop jar hadoop-*-examples.jar teragen 10000000000 in-dir" command to generate the data file. The number here is the line number of the file.
 * The "samplefile" provided contains 10000 files
 *
 * @author <NAME> <<EMAIL>>
 */
public class WordCountTopology implements ApplicationFactory
{
  private boolean allInline =  false;

  @Override
  public DAG getApplication(Configuration conf)
  {
    allInline = true;
    DAG dag =new DAG(conf);

    RandomSentenceSpout spout = dag.addOperator("spout", new RandomSentenceSpout());<commit_msg>cleaned unused import<commit_after>/*
 *  Copyright (c) 2012-2013 Malhar, Inc.
 *  All Rights Reserved.
 */
package com.malhartech.demos.wordcount;

import com.malhartech.api.ApplicationFactory;
import com.malhartech.api.DAG;
import com.malhartech.lib.io.ConsoleOutputOperator;
import org.apache.hadoop.conf.Configuration;

/**
 * To run the test, you need to generate a data file as sample file.
 * You can use "hadoop jar hadoop-*-examples.jar teragen 10000000000 in-dir" command to generate the data file. The number here is the line number of the file.
 * The "samplefile" provided contains 10000 files
 *
 * @author <NAME> <<EMAIL>>
 */
public class WordCountTopology implements ApplicationFactory
{
  private boolean allInline =  false;

  @Override
  public DAG getApplication(Configuration conf)
  {
    allInline = true;
    DAG dag =new DAG(conf);

    RandomSentenceSpout spout = dag.addOperator("spout", new RandomSentenceSpout());
{'fraction_non_alphanumeric': 0.09606540623403168, 'fraction_numerical': 0.02452733776188043, 'mean_word_length': 4.423822714681441, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6649325', 'n_tokens_mistral': 619, 'n_tokens_neox': 580, 'n_words': 189}