<commit_before># Cassandra storage config YAML 
# See http://wiki.apache.org/cassandra/StorageConfiguration for
# explanations of configuration directives.

# name of the cluster
cluster_name: 'Test Cluster'

# Set to true to make new [non-seed] nodes automatically migrate data
# to themselves from the pre-existing nodes in the cluster.  Defaults
# to false because you can only bootstrap N machines at a time from
# an existing cluster of N, so if you are bringing up a cluster of
# 10 machines with 3 seeds you would have to do it in stages.  Leaving
# this off for the initial start simplifies that.
auto_bootstrap: false

# See http://wiki.apache.org/cassandra/HintedHandoff
hinted_handoff_enabled: true

# authentication backend, implementing IAuthenticator; used to limit keyspace access
authenticator: org.apache.cassandra.auth.AllowAllAuthenticator

# any IPartitioner may be used, including your own as long as it is on
# the classpath.  Out of the box, Cassandra provides
# org.apache.cassandra.dht.RandomPartitioner
# org.apache.cassandra.dht.OrderPreservingPartitioner, and
# org.apache.cassandra.dht.CollatingOrderPreservingPartitioner.
partitioner: org.apache.cassandra.dht.RandomPartitioner

# directories where Cassandra should store data on disk.
data_file_directories:
    - /var/lib/cassandra/data

# Addresses of hosts that are deemed contact points. 
# Cassandra nodes use this list of hosts to find each other and learn
# the topology of the ring.  You must change this if you are running
# multiple nodes!
seeds:
    - 127.0.0.1

# Access mode.  mmapped i/o is substantially faster, but only practical on
# a 64bit machine (which notably does not include EC2 "small" instances)
# or relatively small datasets.  "auto", the safe choice, will enable
# mmapping on a 64bit JVM.  Other values are "mmap", "mmap_index_only"
# (which may allow you to get part of the benefits of mmap on a 32bit
# machine by mmapping only index files) and "standard".
# (The buffer size settings that follow only apply to standard,
# non-mmapped i/o.)
disk_access_mode: auto

# Unlike most systems, in Cassandra writes are faster than reads, so
# you can afford more of those in parallel.  A good rule of thumb is 2
# concurrent reads per processor core.  Increase ConcurrentWrites to
# the number of clients writing at once if you enable CommitLogSync +
# CommitLogSyncDelay. -->
concurrent_reads: 8
concurrent_writes: 32

# This sets the amount of memtable flush writer threads.  These will
# be blocked by disk io, and each one will hold a memtable in memory
# while blocked. If you have a large heap and many data directories,
# you can increase this value for better flush performance.
# By default this will be set to the amount of data directories defined.
#memtable_flush_writers: 1

# Buffer size to use when performing contiguous column slices. 
# Increase this to the size of the column slices you typically perform
sliced_buffer_size_in_kb: 64

# TCP port, for commands and data
storage_port: 7000

# Address to bind to and tell other nodes to connect to. You _must_
# change this if you want multiple nodes to be able to communicate!
listen_address: localhost

# The address to bind the Thrift RPC service to
rpc_address: localhost
# port for Thrift to listen on
rpc_port: 9160
# Whether or not to use a framed transport for Thrift.
thrift_framed_transport: false
snapshot_before_compaction: false

# The threshold size in megabytes the binary memtable must grow to,
# before it's submitted for flushing to disk.
binary_memtable_throughput_in_mb: 256
# The maximum time to leave a dirty memtable unflushed.
# (While any affected columnfamilies have unflushed data from a
# commit log segment, that segment cannot be deleted.)
# This needs to be large enough that it won't cause a flush storm
# of all your memtables flushing at once because none has hit
# the size or count thresholds yet.
memtable_flush_after_mins: 60
# Size of the memtable in memory before it is flushed
memtable_throughput_in_mb: 64
# Number of objects in millions in the memtable before it is flushed
memtable_operations_in_millions: 0.3

column_index_size_in_kb: 64

in_memory_compaction_limit_in_mb: 64

# commit log
commitlog_directory: /var/lib/cassandra/commitlog

# Size to allow commitlog to grow to before creating a new segment 
commitlog_rotation_threshold_in_mb: 128

# commitlog_sync may be either "periodic" or "batch." 
# When in batch mode, Cassandra won't ack writes until the commit log
# has been fsynced to disk.  It will wait up to
# CommitLogSyncBatchWindowInMS milliseconds for other writes, before
# performing the sync.
commitlog_sync: periodic

# the other option is "timed," where writes may be acked immediately
# and the CommitLog is simply synced every commitlog_sync_period_in_ms
# milliseconds.
commitlog_sync_period_in_ms: 10000

# Time to wait for a reply from other nodes before failing the command 
rpc_timeout_in_ms: 10000

# phi value that must be reached for a host to be marked down.
# most users should never need to adjust this.
# phi_convict_threshold: 8

# time to wait before garbage collecting tombstones (deletion markers)
gc_grace_seconds: 864000

# endpoint_snitch -- Set this to a class that implements
# IEndpointSnitch, which will let Cassandra know enough
# about your network topology to route requests efficiently.
# Out of the box, Cassandra provides
# org.apache.cassandra.locator.SimpleSnitch,
# org.apache.cassandra.locator.RackInferringSnitch, and
# org.apache.cassandra.locator.PropertyFileSnitch.
endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch

# request_scheduler -- Set this to a class that implements
# RequestScheduler, which will schedule incoming client requests
# according to the specific policy. This is useful for multi-tenancy
# with a single Cassandra cluster.
# NOTE: This is specifically for requests from the client and does
# not affect inter node communication.
# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place
# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of
# client requests to a node with a sepearte queue for each
# reques_scheduler_id. The requests are throttled based on the limit set
# in throttle_limit in the requeset_scheduler_options
request_scheduler: org.apache.cassandra.scheduler.NoScheduler

# Scheduler Options vary based on the type of scheduler
# NoScheduler - Has no options
# RoundRobin
#  - throttle_limit -- The throttle_limit is the number of in-flight
#                      requests per client.  Requests beyond 
#                      that limit are queued up until
#                      running requests can complete.
#                      The value of 80 here is twice the number of
#                      concurrent_reads + concurrent_writes.
# request_scheduler_options:
#    throttle_limit: 80

# request_scheduler_id -- An identifer based on which to perform
# the request scheduling. The current supported option is "keyspace"
request_scheduler_id: keyspace

# dynamic_snitch -- This boolean controls whether the above snitch is
# wrapped with a dynamic snitch, which will monitor read latencies
# and avoid reading from hosts that have slowed (due to compaction,
# for instance)
dynamic_snitch: false

# A ColumnFamily is the Cassandra concept closest to a relational table. 
#
# Keyspaces are separate groups of ColumnFamilies.  Except in very
# unusual circumstances you will have one Keyspace per application.
#
# Keyspace required parameters:
# - name: name of the keyspace; "system" and "definitions" are 
#   reserved for Cassandra Internals.
# - replica_placement_strategy: the class that determines how replicas
#   are distributed among nodes.  Must implement IReplicaPlacementStrategy.
#   Out of the box, Cassandra provides 
#     * org.apache.cassandra.locator.RackUnawareStrategy 
#     * org.apache.cassandra.locator.RackAwareStrategy
#     * org.apache.cassandra.locator.DatacenterShardStrategy
#
#   RackUnawareStrategy is the simplest; it simply places the first
#   replica at the node whose token is closest to the key (as determined
#   by the Partitioner), and additional replicas on subsequent nodes
#   along the ring in increasing Token order.
# 
#   RackAwareStrategy is special cased for replication_factor of 3.  It
#   places one replica in each of two datacenters, and the third on a
#   different rack in in the first.<commit_msg>Reorder snitch config.<commit_after># Cassandra storage config YAML 
# See http://wiki.apache.org/cassandra/StorageConfiguration for
# explanations of configuration directives.

# name of the cluster
cluster_name: 'Test Cluster'

# Set to true to make new [non-seed] nodes automatically migrate data
# to themselves from the pre-existing nodes in the cluster.  Defaults
# to false because you can only bootstrap N machines at a time from
# an existing cluster of N, so if you are bringing up a cluster of
# 10 machines with 3 seeds you would have to do it in stages.  Leaving
# this off for the initial start simplifies that.
auto_bootstrap: false

# See http://wiki.apache.org/cassandra/HintedHandoff
hinted_handoff_enabled: true

# authentication backend, implementing IAuthenticator; used to limit keyspace access
authenticator: org.apache.cassandra.auth.AllowAllAuthenticator

# any IPartitioner may be used, including your own as long as it is on
# the classpath.  Out of the box, Cassandra provides
# org.apache.cassandra.dht.RandomPartitioner
# org.apache.cassandra.dht.OrderPreservingPartitioner, and
# org.apache.cassandra.dht.CollatingOrderPreservingPartitioner.
partitioner: org.apache.cassandra.dht.RandomPartitioner

# directories where Cassandra should store data on disk.
data_file_directories:
    - /var/lib/cassandra/data

# Addresses of hosts that are deemed contact points. 
# Cassandra nodes use this list of hosts to find each other and learn
# the topology of the ring.  You must change this if you are running
# multiple nodes!
seeds:
    - 127.0.0.1

# Access mode.  mmapped i/o is substantially faster, but only practical on
# a 64bit machine (which notably does not include EC2 "small" instances)
# or relatively small datasets.  "auto", the safe choice, will enable
# mmapping on a 64bit JVM.  Other values are "mmap", "mmap_index_only"
# (which may allow you to get part of the benefits of mmap on a 32bit
# machine by mmapping only index files) and "standard".
# (The buffer size settings that follow only apply to standard,
# non-mmapped i/o.)
disk_access_mode: auto

# Unlike most systems, in Cassandra writes are faster than reads, so
# you can afford more of those in parallel.  A good rule of thumb is 2
# concurrent reads per processor core.  Increase ConcurrentWrites to
# the number of clients writing at once if you enable CommitLogSync +
# CommitLogSyncDelay. -->
concurrent_reads: 8
concurrent_writes: 32

# This sets the amount of memtable flush writer threads.  These will
# be blocked by disk io, and each one will hold a memtable in memory
# while blocked. If you have a large heap and many data directories,
# you can increase this value for better flush performance.
# By default this will be set to the amount of data directories defined.
#memtable_flush_writers: 1

# Buffer size to use when performing contiguous column slices. 
# Increase this to the size of the column slices you typically perform
sliced_buffer_size_in_kb: 64

# TCP port, for commands and data
storage_port: 7000

# Address to bind to and tell other nodes to connect to. You _must_
# change this if you want multiple nodes to be able to communicate!
listen_address: localhost

# The address to bind the Thrift RPC service to
rpc_address: localhost
# port for Thrift to listen on
rpc_port: 9160
# Whether or not to use a framed transport for Thrift.
thrift_framed_transport: false
snapshot_before_compaction: false

# The threshold size in megabytes the binary memtable must grow to,
# before it's submitted for flushing to disk.
binary_memtable_throughput_in_mb: 256
# The maximum time to leave a dirty memtable unflushed.
# (While any affected columnfamilies have unflushed data from a
# commit log segment, that segment cannot be deleted.)
# This needs to be large enough that it won't cause a flush storm
# of all your memtables flushing at once because none has hit
# the size or count thresholds yet.
memtable_flush_after_mins: 60
# Size of the memtable in memory before it is flushed
memtable_throughput_in_mb: 64
# Number of objects in millions in the memtable before it is flushed
memtable_operations_in_millions: 0.3

column_index_size_in_kb: 64

in_memory_compaction_limit_in_mb: 64

# commit log
commitlog_directory: /var/lib/cassandra/commitlog

# Size to allow commitlog to grow to before creating a new segment 
commitlog_rotation_threshold_in_mb: 128

# commitlog_sync may be either "periodic" or "batch." 
# When in batch mode, Cassandra won't ack writes until the commit log
# has been fsynced to disk.  It will wait up to
# CommitLogSyncBatchWindowInMS milliseconds for other writes, before
# performing the sync.
commitlog_sync: periodic

# the other option is "timed," where writes may be acked immediately
# and the CommitLog is simply synced every commitlog_sync_period_in_ms
# milliseconds.
commitlog_sync_period_in_ms: 10000

# Time to wait for a reply from other nodes before failing the command 
rpc_timeout_in_ms: 10000

# phi value that must be reached for a host to be marked down.
# most users should never need to adjust this.
# phi_convict_threshold: 8

# time to wait before garbage collecting tombstones (deletion markers)
gc_grace_seconds: 864000

# endpoint_snitch -- Set this to a class that implements
# IEndpointSnitch, which will let Cassandra know enough
# about your network topology to route requests efficiently.
# Out of the box, Cassandra provides
# org.apache.cassandra.locator.SimpleSnitch,
# org.apache.cassandra.locator.RackInferringSnitch, and
# org.apache.cassandra.locator.PropertyFileSnitch.
endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch

# dynamic_snitch -- This boolean controls whether the above snitch is
# wrapped with a dynamic snitch, which will monitor read latencies
# and avoid reading from hosts that have slowed (due to compaction,
# for instance)
dynamic_snitch: false

# request_scheduler -- Set this to a class that implements
# RequestScheduler, which will schedule incoming client requests
# according to the specific policy. This is useful for multi-tenancy
# with a single Cassandra cluster.
# NOTE: This is specifically for requests from the client and does
# not affect inter node communication.
# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place
# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of
# client requests to a node with a sepearte queue for each
# reques_scheduler_id. The requests are throttled based on the limit set
# in throttle_limit in the requeset_scheduler_options
request_scheduler: org.apache.cassandra.scheduler.NoScheduler

# Scheduler Options vary based on the type of scheduler
# NoScheduler - Has no options
# RoundRobin
#  - throttle_limit -- The throttle_limit is the number of in-flight
#                      requests per client.  Requests beyond 
#                      that limit are queued up until
#                      running requests can complete.
#                      The value of 80 here is twice the number of
#                      concurrent_reads + concurrent_writes.
# request_scheduler_options:
#    throttle_limit: 80

# request_scheduler_id -- An identifer based on which to perform
# the request scheduling. The current supported option is "keyspace"
request_scheduler_id: keyspace

# A ColumnFamily is the Cassandra concept closest to a relational table. 
#
# Keyspaces are separate groups of ColumnFamilies.  Except in very
# unusual circumstances you will have one Keyspace per application.
#
# Keyspace required parameters:
# - name: name of the keyspace; "system" and "definitions" are 
#   reserved for Cassandra Internals.
# - replica_placement_strategy: the class that determines how replicas
#   are distributed among nodes.  Must implement IReplicaPlacementStrategy.
#   Out of the box, Cassandra provides 
#     * org.apache.cassandra.locator.RackUnawareStrategy 
#     * org.apache.cassandra.locator.RackAwareStrategy
#     * org.apache.cassandra.locator.DatacenterShardStrategy
#
#   RackUnawareStrategy is the simplest; it simply places the first
#   replica at the node whose token is closest to the key (as determined
#   by the Partitioner), and additional replicas on subsequent nodes
#   along the ring in increasing Token order.
# 
#   RackAwareStrategy is special cased for replication_factor of 3.  It
#   places one replica in each of two datacenters, and the third on a
#   different rack in in the first.
{'fraction_non_alphanumeric': 0.05062614991987655, 'fraction_numerical': 0.008190397056205116, 'mean_word_length': 4.342422320862397, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5067966', 'n_tokens_mistral': 4846, 'n_tokens_neox': 4587, 'n_words': 2117}