<commit_before>########### These all have default values as shown
########### All the config associated with time is in the unit of milli-seconds, unless otherwise specified.
########### All the config associated with data size is in the unit of bytes, unless otherwise specified.

### system level configs
heron.java.home.path: "/usr/lib/jvm/java-1.8.0-twitter/" # JAVA_HOME path on host's environment

### heron.* configs are general configurations over all componenets

heron.logging.directory: "log-files" # The relative path to the logging directory
heron.logging.maximum.size.mb: 100 # The maximum log file size in MB
heron.logging.maximum.files: 5 # The maximum number of log files
heron.check.tmaster.location.interval.sec: 120 # The interval in seconds after which to check if the tmaster location has been fetched or not
heron.logging.prune.interval.sec: 300 # The interval in seconds to prune logging files in C++
heron.logging.flush.interval.sec: 10 # The interval in seconds to flush log files in C++
heron.logging.err.threshold: 3 # The threadhold level to log error

# The interval in seconds for different components to export metrics to metrics manager
heron.metrics.export.interval.sec: 60


### heron.streammgr.* configs are for the stream manager
heron.streammgr.packet.maximum.size.bytes: 102400 # Maximum size in bytes of a packet to be send out from stream manager
# The tuple cache (used for batching) can be drained in two ways: (a) Time based (b) size based
heron.streammgr.cache.drain.frequency.ms: 10 # The frequency in ms to drain the tuple cache in stream manager
heron.streammgr.cache.drain.size.mb: 100 # The sized based threshold in MB for draining the tuple cache
heron.streammgr.xormgr.rotatingmap.nbuckets: 3 # For efficient acknowledgement
heron.streammgr.client.reconnect.interval.sec: 1 # The reconnect interval to other stream managers in second for stream manager client
heron.streammgr.client.reconnect.tmaster.interval.sec: 10 # The reconnect interval to tamster in second for stream manager client
heron.streammgr.network.options.maximum.packet.mb: 100 # The maximum packet size in MB of stream manager's network options
heron.streammgr.tmaster.heartbeat.interval.sec: 10 # The interval in seconds to send heartbeat
heron.streammgr.connection.read.batch.size.mb: 1 # Maximum batch size in MB to read by stream manager from socket
heron.streammgr.connection.write.batch.size.mb: 1 # Maximum batch size in MB to write by stream manager to socket
# Number of times we should wait to see a buffer full while enqueueing data before declaring start of back pressure
heron.streammgr.network.backpressure.threshold: 3
# High water mark on the num in MB that can be left outstanding on a connection
heron.streammgr.network.backpressure.highwatermark.mb: 50
# Low water mark on the num in MB that can be left outstanding on a connection
heron.streammgr.network.backpressure.lowwatermark.mb: 30


### heron.tmaster.* configs are for the tmaster
heron.tmaster.metrics.collector.maximum.interval.min: 180 # The maximum interval in minutes of metrics to be kept in tmaster
heron.tmaster.establish.retry.times: 30 # The maximum time to retry to establish the tmaster
heron.tmaster.establish.retry.interval.sec: 1 # The interval to retry to establish the tmaster
heron.tmaster.network.master.options.maximum.packet.mb: 16 # The maximum packet size in MB of tmaster's network options for stmgrs to connect to
heron.tmaster.network.controller.options.maximum.packet.mb: 1 # The maximum packet size in MB of tmaster's network options for scheduler to connect to
heron.tmaster.network.stats.options.maximum.packet.mb: 1 # The maximum packet size in MB of tmaster's network options for stat queries
heron.tmaster.metrics.collector.purge.interval.sec: 60 # The inteval for tmaster to purge metrics from socket
heron.tmaster.metrics.collector.maximum.exception: 256 # The maximum # of exception to be stored in tmetrics collector, to prevent potential OOM
heron.tmaster.metrics.network.bindallinterfaces: False # Should the metrics reporter bind on all interfaces
heron.tmaster.stmgr.state.timeout.sec: 60 # The timeout in seconds for stream mgr, compared with (current time - last heartbeat time)


### heron.metricsmgr.* configs are for the metrics manager
# The size of packets to read from socket will be determined by the minimal of: (a) time based (b) size based
heron.metricsmgr.network.read.batch.time.ms: 16 # Time based, the maximum batch time in ms for metricsmgr to read from socket
heron.metricsmgr.network.read.batch.size.bytes: 32768 # Size based, the maximum batch size in bytes to read from socket
# The size of packets to write to socket will be determined by the minimal of: (a) time based (b) size based
heron.metricsmgr.network.write.batch.time.ms: 16 # Time based, the maximum batch time in ms for metricsmgr to write to socket
heron.metricsmgr.network.write.batch.size.bytes: 32768 # Size based, the maximum batch size in bytes to write to socket
heron.metricsmgr.network.options.socket.send.buffer.size.bytes: 6553600 # The maximum socket's send buffer size in bytes
heron.metricsmgr.network.options.socket.received.buffer.size.bytes: 8738000 # The maximum socket's received buffer size in bytes of metricsmgr's network options

### heron.instance.* configs are for the instances
heron.instance.internal.bolt.read.queue.capacity: 128 # The queue capacity (num of items) in bolt for buffer packets to read from stream manager
heron.instance.internal.bolt.write.queue.capacity: 128 # The queue capacity (num of items) in bolt for buffer packets to write to stream manager
heron.instance.internal.spout.read.queue.capacity: 1024 # The queue capacity (num of items) in spout for buffer packets to read from stream manager
heron.instance.internal.spout.write.queue.capacity: 128 # The queue capacity (num of items) in spout for buffer packets to write to stream manager
heron.instance.internal.metrics.write.queue.capacity: 128 # The queue capacity (num of items) for metrics packets to write to metrics manager
# The size of packets read from stream manager will be determined by the minimal of: (a) time based (b) size based
heron.instance.network.read.batch.time.ms: 16 # Time based, the maximum batch time in ms for instance to read from stream manager per attempt
heron.instance.network.read.batch.size.bytes: 32768 # Size based, the maximum batch size in bytes to read from stream manager
# The size of packets written to stream manager will be determined by the minimal of: (a) time based (b) size based
heron.instance.network.write.batch.time.ms: 16 # Time based, the maximum batch time in ms for instance to write to stream manager per attempt
heron.instance.network.write.batch.size.bytes: 32768 # Size based, the maximum batch size in bytes to write to stream manager
heron.instance.network.options.socket.send.buffer.size.bytes: 6553600 # The maximum socket's send buffer size in bytes
heron.instance.network.options.socket.received.buffer.size.bytes: 8738000 # The maximum socket's received buffer size in bytes of instance's network options
heron.instance.set.data.tuple.capacity: 1024 # The maximum # of data tuple to batch in a HeronDataTupleSet protobuf
heron.instance.set.control.tuple.capacity: 1024 # The maximum # of control tuple to batch in a HeronControlTupleSet protobuf
heron.instance.ack.batch.time.ms: 128 # The maximum time in ms for an spout to do acknowledgement per attempt, the ack batch could also break if there are no more ack tuples to process
heron.instance.emit.batch.time.ms: 16 # The maximum time in ms for an spout instance to emit tuples per attempt
heron.instance.emit.batch.size.bytes: 32768 # The maximum batch size in bytes for an spout to emit tuples per attempt
heron.instance.execute.batch.time.ms: 16 # The maximum time in ms for an bolt instance to execute tuples per attempt
heron.instance.execute.batch.size.bytes: 32768 # The maximum batch size in bytes for an bolt instance to execute tuples per attempt
heron.instance.state.check.interval.sec: 5 # The time interval for an instance to check the state change, for instance, the interval a spout using to check whether activate/deactivate is invoked
heron.instance.force.exit.timeout.ms: 2000 # The time to wait before the instance exits forcibly when uncaught exception happens

heron.instance.reconnect.streammgr.interval.sec: 5 # Interval in seconds to reconnect to the stream manager, including the request timeout in connecting
heron.instance.reconnect.streammgr.times: 60
heron.instance.reconnect.metricsmgr.interval.sec: 5 # Interval in seconds to reconnect to the metrics manager, including the request timeout in connecting
heron.instance.reconnect.metricsmgr.times: 60

# The interval in second for an instance to sample its system metrics, for instance, cpu load.
heron.instance.metrics.system.sample.interval.sec: 10

heron.instance.slave.fetch.pplan.interval.sec: 1

heron.instance.acknowledgement.nbuckets: 10 # For efficient acknowledgement
## For dynamically tuning the available sizes in the interval read&write queues to keep high performance while avoid gc issues
heron.instance.tuning.expected.bolt.read.queue.size: 8 # The expected size on read queue in bolt
heron.instance.tuning.expected.bolt.write.queue.size: 8 # The expected size on write queue in bolt
heron.instance.tuning.expected.spout.read.queue.size: 512 # The expected size on read queue in spout
heron.instance.tuning.expected.spout.write.queue.size: 8 # The exepected size on write queue in spout
heron.instance.tuning.expected.metrics.write.queue.size: 8 # The expected size on metrics write queue
heron.instance.tuning.current.sample.weight: 0.8
heron.instance.tuning.interval.ms: 100 # Interval in ms to tune the size of in & out data queue in instance<commit_msg>make heron_internals.yaml more readable<commit_after>################################################################################
# Default values for various configs used inside Heron.
################################################################################
# All the config associated with time is in the unit of milli-seconds, 
# unless otherwise specified.
################################################################################
# All the config associated with data size is in the unit of bytes, unless 
# otherwise specified.
################################################################################

################################################################################
# System level configs
################################################################################

# JAVA_HOME path on host's environment
heron.java.home.path: "/usr/lib/jvm/java-1.8.0-twitter/" 

### heron.* configs are general configurations over all componenets

# The relative path to the logging directory
heron.logging.directory: "log-files" 

# The maximum log file size in MB
heron.logging.maximum.size.mb: 100 

# The maximum number of log files
heron.logging.maximum.files: 5 

# The interval in seconds after which to check if the tmaster location has been fetched or not
heron.check.tmaster.location.interval.sec: 120 

# The interval in seconds to prune logging files in C++
heron.logging.prune.interval.sec: 300 

# The interval in seconds to flush log files in C++
heron.logging.flush.interval.sec: 10 

# The threshold level to log error
heron.logging.err.threshold: 3 

# The interval in seconds for different components to export metrics to metrics manager
heron.metrics.export.interval.sec: 60

################################################################################
# Configs related to Stream Manager, starts with heron.streammgr.*
################################################################################

# Maximum size in bytes of a packet to be send out from stream manager
heron.streammgr.packet.maximum.size.bytes: 102400 

# The tuple cache (used for batching) can be drained in two ways: 
# (a) Time based 
# (b) size based

# The frequency in ms to drain the tuple cache in stream manager
heron.streammgr.cache.drain.frequency.ms: 10 

# The sized based threshold in MB for draining the tuple cache
heron.streammgr.cache.drain.size.mb: 100 

# For efficient acknowledgements
heron.streammgr.xormgr.rotatingmap.nbuckets: 3 

# The reconnect interval to other stream managers in secs for stream manager client
heron.streammgr.client.reconnect.interval.sec: 1 

# The reconnect interval to tamster in second for stream manager client
heron.streammgr.client.reconnect.tmaster.interval.sec: 10 

# The maximum packet size in MB of stream manager's network options
heron.streammgr.network.options.maximum.packet.mb: 100 

# The interval in seconds to send heartbeat
heron.streammgr.tmaster.heartbeat.interval.sec: 10 

# Maximum batch size in MB to read by stream manager from socket
heron.streammgr.connection.read.batch.size.mb: 1 

# Maximum batch size in MB to write by stream manager to socket
heron.streammgr.connection.write.batch.size.mb: 1 

# Number of times we should wait to see a buffer full while enqueueing data 
# before declaring start of back pressure
heron.streammgr.network.backpressure.threshold: 3

# High water mark on the num in MB that can be left outstanding on a connection
heron.streammgr.network.backpressure.highwatermark.mb: 50

# Low water mark on the num in MB that can be left outstanding on a connection
heron.streammgr.network.backpressure.lowwatermark.mb: 30

################################################################################
# Configs related to Topology Master, starts with heron.tmaster.*
################################################################################

# The maximum interval in minutes of metrics to be kept in tmaster
heron.tmaster.metrics.collector.maximum.interval.min: 180 

# The maximum time to retry to establish the tmaster
heron.tmaster.establish.retry.times: 30 

# The interval to retry to establish the tmaster
heron.tmaster.establish.retry.interval.sec: 1 

# Maximum packet size in MB of tmaster's network options to connect to stream managers
heron.tmaster.network.master.options.maximum.packet.mb: 16 

# Maximum packet size in MB of tmaster's network options to connect to scheduler
heron.tmaster.network.controller.options.maximum.packet.mb: 1 

# Maximum packet size in MB of tmaster's network options for stat queries
heron.tmaster.network.stats.options.maximum.packet.mb: 1 

# The interval for tmaster to purge metrics from socket
heron.tmaster.metrics.collector.purge.interval.sec: 60 

# The maximum # of exceptions to be stored in tmetrics collector, to prevent potential OOM
heron.tmaster.metrics.collector.maximum.exception: 256 

# Should the metrics reporter bind on all interfaces
heron.tmaster.metrics.network.bindallinterfaces: False 

# The timeout in seconds for stream mgr, compared with (current time - last heartbeat time)
heron.tmaster.stmgr.state.timeout.sec: 60 

################################################################################
# Configs related to Topology Master, starts with heron.metricsmgr.*
################################################################################

# The size of packets to read from socket will be determined by the minimal of: 
# (a) time based 
# (b) size based

# Time based, the maximum batch time in ms for metricsmgr to read from socket
heron.metricsmgr.network.read.batch.time.ms: 16 

# Size based, the maximum batch size in bytes to read from socket
heron.metricsmgr.network.read.batch.size.bytes: 32768 

# The size of packets to write to socket will be determined by the minimum of 
# (a) time based 
# (b) size based

# Time based, the maximum batch time in ms for metricsmgr to write to socket
heron.metricsmgr.network.write.batch.time.ms: 16 

# Size based, the maximum batch size in bytes to write to socket
heron.metricsmgr.network.write.batch.size.bytes: 32768 

# The maximum socket's send buffer size in bytes
heron.metricsmgr.network.options.socket.send.buffer.size.bytes: 6553600 

# The maximum socket's received buffer size in bytes of metricsmgr's network options
heron.metricsmgr.network.options.socket.received.buffer.size.bytes: 8738000 

################################################################################
# Configs related to Heron Instance, starts with heron.instance.*
################################################################################

# The queue capacity (num of items) in bolt for buffer packets to read from stream manager
heron.instance.internal.bolt.read.queue.capacity: 128 

# The queue capacity (num of items) in bolt for buffer packets to write to stream manager
heron.instance.internal.bolt.write.queue.capacity: 128 

# The queue capacity (num of items) in spout for buffer packets to read from stream manager
heron.instance.internal.spout.read.queue.capacity: 1024 

# The queue capacity (num of items) in spout for buffer packets to write to stream manager
heron.instance.internal.spout.write.queue.capacity: 128 

# The queue capacity (num of items) for metrics packets to write to metrics manager
heron.instance.internal.metrics.write.queue.capacity: 128 

# The size of packets read from stream manager will be determined by the minimal of
# (a) time based 
# (b) size based

# Time based, the maximum batch time in ms for instance to read from stream manager per attempt
heron.instance.network.read.batch.time.ms: 16 

# Size based, the maximum batch size in bytes to read from stream manager
heron.instance.network.read.batch.size.bytes: 32768 

# The size of packets written to stream manager will be determined by the minimum of 
# (a) time based 
# (b) size based

# Time based, the maximum batch time in ms for instance to write to stream manager per attempt
heron.instance.network.write.batch.time.ms: 16 

# Size based, the maximum batch size in bytes to write to stream manager
heron.instance.network.write.batch.size.bytes: 32768 

# The maximum socket's send buffer size in bytes
heron.instance.network.options.socket.send.buffer.size.bytes: 6553600 

# The maximum socket's received buffer size in bytes of instance's network options
heron.instance.network.options.socket.received.buffer.size.bytes: 8738000 

# The maximum # of data tuple to batch in a HeronDataTupleSet protobuf
heron.instance.set.data.tuple.capacity: 1024 

# The maximum # of control tuple to batch in a HeronControlTupleSet protobuf
heron.instance.set.control.tuple.capacity: 1024 

# The maximum time in ms for a spout to do acknowledgement per attempt, the ack batch could 
# also break if there are no more ack tuples to process
heron.instance.ack.batch.time.ms: 128 

# The maximum time in ms for an spout instance to emit tuples per attempt
heron.instance.emit.batch.time.ms: 16 

# The maximum batch size in bytes for an spout to emit tuples per attempt
heron.instance.emit.batch.size.bytes: 32768 

# The maximum time in ms for an bolt instance to execute tuples per attempt
heron.instance.execute.batch.time.ms: 16 

# The maximum batch size in bytes for an bolt instance to execute tuples per attempt
heron.instance.execute.batch.size.bytes: 32768 

# The time interval for an instance to check the state change, 
# for example, the interval a spout uses to check whether activate/deactivate is invoked
heron.instance.state.check.interval.sec: 5 

# The time to wait before the instance exits forcibly when uncaught exception happens
heron.instance.force.exit.timeout.ms: 2000 

# Interval in seconds to reconnect to the stream manager, including the request timeout in connecting
heron.instance.reconnect.streammgr.interval.sec: 5 
heron.instance.reconnect.streammgr.times: 60

# Interval in seconds to reconnect to the metrics manager, including the request timeout in connecting
heron.instance.reconnect.metricsmgr.interval.sec: 5 
heron.instance.reconnect.metricsmgr.times: 60

# The interval in second for an instance to sample its system metrics, for instance, cpu load.
heron.instance.metrics.system.sample.interval.sec: 10

heron.instance.slave.fetch.pplan.interval.sec: 1

# For efficient acknowledgement
heron.instance.acknowledgement.nbuckets: 10 

################################################################################
# For dynamically tuning the available sizes in the interval read & write queues 
# to provide high performance while avoiding GC issues
################################################################################

# The expected size on read queue in bolt
heron.instance.tuning.expected.bolt.read.queue.size: 8 

# The expected size on write queue in bolt
heron.instance.tuning.expected.bolt.write.queue.size: 8 

# The expected size on read queue in spout
heron.instance.tuning.expected.spout.read.queue.size: 512 

# The exepected size on write queue in spout
heron.instance.tuning.expected.spout.write.queue.size: 8 

# The expected size on metrics write queue
heron.instance.tuning.expected.metrics.write.queue.size: 8 

heron.instance.tuning.current.sample.weight: 0.8

# Interval in ms to tune the size of in & out data queue in instance
heron.instance.tuning.interval.ms: 100 
{'fraction_non_alphanumeric': 0.12595293337752733, 'fraction_numerical': 0.017709171835787678, 'mean_word_length': 4.929253228523302, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6867051', 'n_tokens_mistral': 5988, 'n_tokens_neox': 5510, 'n_words': 2352}