<commit_before>##############################################################################
# This file is autogenerated by chef                                         #
#                                                                            #
# Collectd Kafka Producer plugin configuration file                          #
#----------------------------------------------------------------------------#
# By: <NAME> - Webtrends 2012                                         #
#                                                                            #
# Revisions: 06/11/2012 - Initial script creation                            #
#            06/21/2012 - Added JVM stats, and logreplay metrics             #
#                                                                            #
##############################################################################
LoadPlugin java

<Plugin "java">
        JVMArg "-verbose:jni"
        JVMArg "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/generic-jmx.jar"
        LoadPlugin "org.collectd.java.GenericJMX"
        <Plugin "GenericJMX">
        ################
        # MBean blocks #
        ################
		# Memory usage by memory pool.
		<MBean "memory_pool">
			ObjectName "java.lang:type=MemoryPool,*"
			InstancePrefix "memory_pool-"
			InstanceFrom "name"
			<Value>
				Type "memory"
				#InstancePrefix ""
				#InstanceFrom ""
				Table true
				Attribute "Usage"
			</Value>
		</MBean>	
		# Heap memory usage
		<MBean "memory-heap">
			ObjectName "java.lang:type=Memory"
			#InstanceFrom ""
			InstancePrefix "memory-heap"  
			# Creates four values: committed, init, max, used
			<Value>
				Type "memory"  
				#InstancePrefix ""
				#InstanceFrom ""
				Table true
				Attribute "HeapMemoryUsage"
			</Value>
		</MBean>
		# Non-heap memory usage
		<MBean "memory_nonheap">
			ObjectName "java.lang:type=Memory"
			#InstanceFrom ""
			InstancePrefix "memory-nonheap"
			# Creates four values: committed, init, max, used
			<Value>
				Type "memory"
				#InstancePrefix ""
				#InstanceFrom ""
				Table true
				Attribute "NonHeapMemoryUsage"
			</Value>
		</MBean>
		# Non-heap memory usage
		<MBean "java_threads">
			ObjectName "java.lang:type=Threading"
			#InstanceFrom ""
			InstancePrefix "java_threads"
			<Value>
				Type "gauge"
				InstancePrefix "ThreadCount"
				#InstanceFrom ""
				Attribute "ThreadCount"
			</Value>
		</MBean>	
		# Garbage Collector stats
		<MBean "garbage_collector">
			ObjectName "java.lang:type=GarbageCollector,*"
			InstancePrefix "gc-"
			InstanceFrom "name"
			<Value>
				Type "invocations"
				InstancePrefix "collection_count"
				#InstanceFrom ""
				Table false
				Attribute "CollectionCount"
			</Value>
			<Value>
				Type "total_time_in_ms"
				InstancePrefix "collection_time"
				#InstanceFrom ""
				Table false
				Attribute "CollectionTime"
			</Value>
		</MBean>
		# Number of classes being loaded.
		<MBean "classes">
			ObjectName "java.lang:type=ClassLoading"
			InstancePrefix "loaded_classes"
			#InstanceFrom ""
			<Value>
				Type "gauge"
				InstancePrefix "loaded_classes"
				#InstanceFrom ""
				Table false
				Attribute "LoadedClassCount"
			</Value>
		</MBean>
		# java runtime attributes
		<MBean "runtime">
			ObjectName "java.lang:type=Runtime"
			InstancePrefix "runtime"
			#InstanceFrom ""
			<Value>
				Type "gauge"
				InstancePrefix "JVM_uptime"
				#InstanceFrom ""
				Table false
				Attribute "Uptime"
			</Value>
		</MBean>
		#Kafka Metrics
        <MBean "kafka">
                ObjectName "kafka:type=kafka.KafkaProducerStats"
                InstancePrefix "KafkaProducerStats"
                #InstanceFrom ""
                <Value>
                        Type "gauge"
                        InstancePrefix "AvgProduceRequestMs"
                        #InstanceFrom ""
                        Table false
                        Attribute "AvgProduceRequestMs"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "MaxProduceRequestMs"
                        #InstanceFrom ""
                        Table false
                        Attribute "MaxProduceRequestMs"
                </Value>
                <Value>
                        Type "counter"
                        InstancePrefix "NumProduceRequests"
                        #InstanceFrom ""
                        Table false
                        Attribute "NumProduceRequests"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "ProduceRequestsPerSecond"
                        #InstanceFrom ""
                        Table false
                        Attribute "ProduceRequestsPerSecond"
                </Value>
        </MBean>
        <MBean "async">
                ObjectName "kafka.producer.Producer:type=AsyncProducerStats"
                InstancePrefix "AsyncProducerStats"
                #InstanceFrom ""
                <Value>
                        Type "gauge"
                        InstancePrefix "AsyncProducerQueueSize"
                        #InstanceFrom ""
                        Table false
                        Attribute "AsyncProducerQueueSize"
                </Value>
        </MBean>
        #Metrics
        <MBean "active-dcsid">
			ObjectName "logreplayer:type=config,name=active-dcsid"
			InstancePrefix ""
			InstanceFrom "name"
			<Value>
				Type "gauge"
				InstancePrefix "Value"
				Table false
				Attribute "Value"
			</Value>
	    </MBean>
        <MBean "files-in-queue">
			ObjectName "logreplayer:type=filemover,name=files-in-queue"
			InstancePrefix ""
			InstanceFrom "name"
			<Value>
				Type "gauge"
				InstancePrefix "Value"
				Table false
				Attribute "Value"
			</Value>
        </MBean>
        <MBean "hits-accepted">
                ObjectName "logreplayer:type=logconsumer,name=hits-accepted"
                #InstancePrefix ""
                InstanceFrom "name"
                <Value>
                        Type "counter"
                        InstancePrefix "Count"
                        #InstanceFrom ""
                        Table false
                        Attribute "Count"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "MeanRate"
                        #InstanceFrom ""
                        Table false
                        Attribute "MeanRate"
                </Value>
        </MBean>
        <MBean "hits-per-log-file">
                ObjectName "logreplayer:type=logconsumer,name=hits-per-log-file"
                #InstancePrefix ""
                InstanceFrom "name"
                <Value>
                        Type "counter"
                        InstancePrefix "Count"
                        #InstanceFrom ""
                        Table false
                        Attribute "Count"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "Max"
                        #InstanceFrom ""
                        Table false
                        Attribute "Max"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "Mean"
                        #InstanceFrom ""
                        Table false
                        Attribute "Mean"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "Min"
                        #InstanceFrom ""
                        Table false
                        Attribute "Min"
                </Value>
        </MBean>
       <MBean "log-files-replayed">
                ObjectName "logreplayer:type=logconsumer,name=log-files-replayed"
                #InstancePrefix ""
                InstanceFrom "name"
                <Value>
                        Type "counter"
                        InstancePrefix "Count"
                        #InstanceFrom ""
                        Table false
                        Attribute "Count"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "MeanRate"
                        #InstanceFrom ""
                        Table false
                        Attribute "MeanRate"
                </Value>
        </MBean>
        #####################
        # Connection blocks #
        #####################
        #Connection Block
        <Connection>
			Host "<%= @node[:hostname] %>"
			ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:9999/jmxrmi"
			Collect "classes"
			Collect "garbage_collector"
			Collect "java_threads"
			Collect "memory-heap"
			Collect "memory_nonheap"
			Collect "runtime"
			Collect "kafka"
			Collect "async"
			Collect "hits-accepted"
			Collect "hits-per-log-file"
			Collect "log-files-replayed"
        </Connection>
        </Plugin>
</Plugin>
<commit_msg>Use the jmx_port attribute in the collectd producer script as well<commit_after>##############################################################################
# This file is autogenerated by chef                                         #
#                                                                            #
# Collectd Kafka Producer plugin configuration file                          #
#----------------------------------------------------------------------------#
# By: <NAME> - Webtrends 2012                                         #
#                                                                            #
# Revisions: 06/11/2012 - Initial script creation                            #
#            06/21/2012 - Added JVM stats, and logreplay metrics             #
#                                                                            #
##############################################################################
LoadPlugin java

<Plugin "java">
        JVMArg "-verbose:jni"
        JVMArg "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/generic-jmx.jar"
        LoadPlugin "org.collectd.java.GenericJMX"
        <Plugin "GenericJMX">
        ################
        # MBean blocks #
        ################
		# Memory usage by memory pool.
		<MBean "memory_pool">
			ObjectName "java.lang:type=MemoryPool,*"
			InstancePrefix "memory_pool-"
			InstanceFrom "name"
			<Value>
				Type "memory"
				#InstancePrefix ""
				#InstanceFrom ""
				Table true
				Attribute "Usage"
			</Value>
		</MBean>
		# Heap memory usage
		<MBean "memory-heap">
			ObjectName "java.lang:type=Memory"
			#InstanceFrom ""
			InstancePrefix "memory-heap"
			# Creates four values: committed, init, max, used
			<Value>
				Type "memory"
				#InstancePrefix ""
				#InstanceFrom ""
				Table true
				Attribute "HeapMemoryUsage"
			</Value>
		</MBean>
		# Non-heap memory usage
		<MBean "memory_nonheap">
			ObjectName "java.lang:type=Memory"
			#InstanceFrom ""
			InstancePrefix "memory-nonheap"
			# Creates four values: committed, init, max, used
			<Value>
				Type "memory"
				#InstancePrefix ""
				#InstanceFrom ""
				Table true
				Attribute "NonHeapMemoryUsage"
			</Value>
		</MBean>
		# Non-heap memory usage
		<MBean "java_threads">
			ObjectName "java.lang:type=Threading"
			#InstanceFrom ""
			InstancePrefix "java_threads"
			<Value>
				Type "gauge"
				InstancePrefix "ThreadCount"
				#InstanceFrom ""
				Attribute "ThreadCount"
			</Value>
		</MBean>
		# Garbage Collector stats
		<MBean "garbage_collector">
			ObjectName "java.lang:type=GarbageCollector,*"
			InstancePrefix "gc-"
			InstanceFrom "name"
			<Value>
				Type "invocations"
				InstancePrefix "collection_count"
				#InstanceFrom ""
				Table false
				Attribute "CollectionCount"
			</Value>
			<Value>
				Type "total_time_in_ms"
				InstancePrefix "collection_time"
				#InstanceFrom ""
				Table false
				Attribute "CollectionTime"
			</Value>
		</MBean>
		# Number of classes being loaded.
		<MBean "classes">
			ObjectName "java.lang:type=ClassLoading"
			InstancePrefix "loaded_classes"
			#InstanceFrom ""
			<Value>
				Type "gauge"
				InstancePrefix "loaded_classes"
				#InstanceFrom ""
				Table false
				Attribute "LoadedClassCount"
			</Value>
		</MBean>
		# java runtime attributes
		<MBean "runtime">
			ObjectName "java.lang:type=Runtime"
			InstancePrefix "runtime"
			#InstanceFrom ""
			<Value>
				Type "gauge"
				InstancePrefix "JVM_uptime"
				#InstanceFrom ""
				Table false
				Attribute "Uptime"
			</Value>
		</MBean>
		#Kafka Metrics
        <MBean "kafka">
                ObjectName "kafka:type=kafka.KafkaProducerStats"
                InstancePrefix "KafkaProducerStats"
                #InstanceFrom ""
                <Value>
                        Type "gauge"
                        InstancePrefix "AvgProduceRequestMs"
                        #InstanceFrom ""
                        Table false
                        Attribute "AvgProduceRequestMs"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "MaxProduceRequestMs"
                        #InstanceFrom ""
                        Table false
                        Attribute "MaxProduceRequestMs"
                </Value>
                <Value>
                        Type "counter"
                        InstancePrefix "NumProduceRequests"
                        #InstanceFrom ""
                        Table false
                        Attribute "NumProduceRequests"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "ProduceRequestsPerSecond"
                        #InstanceFrom ""
                        Table false
                        Attribute "ProduceRequestsPerSecond"
                </Value>
        </MBean>
        <MBean "async">
                ObjectName "kafka.producer.Producer:type=AsyncProducerStats"
                InstancePrefix "AsyncProducerStats"
                #InstanceFrom ""
                <Value>
                        Type "gauge"
                        InstancePrefix "AsyncProducerQueueSize"
                        #InstanceFrom ""
                        Table false
                        Attribute "AsyncProducerQueueSize"
                </Value>
        </MBean>
        #Metrics
        <MBean "active-dcsid">
			ObjectName "logreplayer:type=config,name=active-dcsid"
			InstancePrefix ""
			InstanceFrom "name"
			<Value>
				Type "gauge"
				InstancePrefix "Value"
				Table false
				Attribute "Value"
			</Value>
	    </MBean>
        <MBean "files-in-queue">
			ObjectName "logreplayer:type=filemover,name=files-in-queue"
			InstancePrefix ""
			InstanceFrom "name"
			<Value>
				Type "gauge"
				InstancePrefix "Value"
				Table false
				Attribute "Value"
			</Value>
        </MBean>
        <MBean "hits-accepted">
                ObjectName "logreplayer:type=logconsumer,name=hits-accepted"
                #InstancePrefix ""
                InstanceFrom "name"
                <Value>
                        Type "counter"
                        InstancePrefix "Count"
                        #InstanceFrom ""
                        Table false
                        Attribute "Count"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "MeanRate"
                        #InstanceFrom ""
                        Table false
                        Attribute "MeanRate"
                </Value>
        </MBean>
        <MBean "hits-per-log-file">
                ObjectName "logreplayer:type=logconsumer,name=hits-per-log-file"
                #InstancePrefix ""
                InstanceFrom "name"
                <Value>
                        Type "counter"
                        InstancePrefix "Count"
                        #InstanceFrom ""
                        Table false
                        Attribute "Count"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "Max"
                        #InstanceFrom ""
                        Table false
                        Attribute "Max"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "Mean"
                        #InstanceFrom ""
                        Table false
                        Attribute "Mean"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "Min"
                        #InstanceFrom ""
                        Table false
                        Attribute "Min"
                </Value>
        </MBean>
       <MBean "log-files-replayed">
                ObjectName "logreplayer:type=logconsumer,name=log-files-replayed"
                #InstancePrefix ""
                InstanceFrom "name"
                <Value>
                        Type "counter"
                        InstancePrefix "Count"
                        #InstanceFrom ""
                        Table false
                        Attribute "Count"
                </Value>
                <Value>
                        Type "gauge"
                        InstancePrefix "MeanRate"
                        #InstanceFrom ""
                        Table false
                        Attribute "MeanRate"
                </Value>
        </MBean>
        #####################
        # Connection blocks #
        #####################
        #Connection Block
        <Connection>
			Host "<%= @node[:hostname] %>"
			ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:<%= @node['kafka']['jmx_port'] %>/jmxrmi"
			Collect "classes"
			Collect "garbage_collector"
			Collect "java_threads"
			Collect "memory-heap"
			Collect "memory_nonheap"
			Collect "runtime"
			Collect "kafka"
			Collect "async"
			Collect "hits-accepted"
			Collect "hits-per-log-file"
			Collect "log-files-replayed"
        </Connection>
        </Plugin>
</Plugin>

{'fraction_non_alphanumeric': 0.11632742394397023, 'fraction_numerical': 0.0024075289997811337, 'mean_word_length': 1.2831980012492192, 'pattern_counts': {'":': 0, '<': 168, '<?xml version=': 0, '>': 168, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4559246', 'n_tokens_mistral': 5190, 'n_tokens_neox': 4299, 'n_words': 970}