<commit_before>package com.krux.kafka.demos;

import java.util.Timer;
import java.util.TimerTask;

import joptsimple.OptionParser;
import joptsimple.OptionSet;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.krux.kafka.consumer.KafkaConsumer;
import com.krux.stdlib.KruxStdLib;

/**
 * A simple consumer that will listen to a passed-in topic and log consumed
 * message counts
 */
public class DemoConsumer {

    private static final Logger LOG = LoggerFactory.getLogger( KafkaConsumer.class );

    public static void main( String[] args ) {

        OptionParser parser = new OptionParser();

        // expose all kafka consumer config params to the cli
        KafkaConsumer.addStandardOptionsToParser( parser );

        KruxStdLib.setOptionParser( parser );
        OptionSet options = KruxStdLib.initialize( args );

        // ensure required cl options are present
        if ( !options.has( "topic-threads" ) || !options.has( "group.id" ) || !options.has( "zookeeper.connect" ) ) {
            LOG.error( "'--topic-threads', '--group.id', and '--zookeeper.connect' and all required parameters. Exitting!" );
            System.exit( -1 );<commit_msg>fix grammar, spelling<commit_after>package com.krux.kafka.demos;

import java.util.Timer;
import java.util.TimerTask;

import joptsimple.OptionParser;
import joptsimple.OptionSet;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.krux.kafka.consumer.KafkaConsumer;
import com.krux.stdlib.KruxStdLib;

/**
 * A simple consumer that will listen to a passed-in topic and log consumed
 * message counts
 */
public class DemoConsumer {

    private static final Logger LOG = LoggerFactory.getLogger( KafkaConsumer.class );

    public static void main( String[] args ) {

        OptionParser parser = new OptionParser();

        // expose all kafka consumer config params to the cli
        KafkaConsumer.addStandardOptionsToParser( parser );

        KruxStdLib.setOptionParser( parser );
        OptionSet options = KruxStdLib.initialize( args );

        // ensure required cl options are present
        if ( !options.has( "topic-threads" ) || !options.has( "group.id" ) || !options.has( "zookeeper.connect" ) ) {
            LOG.error( "'--topic-threads', '--group.id', and '--zookeeper.connect' are all required parameters. Exiting!" );
            System.exit( -1 );
{'fraction_non_alphanumeric': 0.11083017277707544, 'fraction_numerical': 0.0025284450063211127, 'mean_word_length': 3.324225865209472, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1456716', 'n_tokens_mistral': 718, 'n_tokens_neox': 695, 'n_words': 192}