<commit_before>
sewer.source=pixel
sewer.sink=null

# when using the 'reliable' sink, this path will be written to.
# the user running sewer must have read/write permissions to this path.
sewer.wal.path=/opt/sewer/wal

# sample reliable sink writing to HDFS
# sewer.sink=roll(30) > reliable > seqfile('hdfs://localhost:9000/test/collect/%Y-%m-%d/%H00/data-%{host}-%Y%m%d-%k%M%S')



# type of compression to use for writing to file/hdfs streams
# valid options: deflate (zlib), gzip, bzip2, snappy
# note: snappy is only available in newer versions of hadoop
# default: gzip (if hadoop native is available), else deflate
#sewer.sink.compression=gzip


# good default buffer size for hadoop read/write buffers
# haven't seen much/any improvement w/ larger buffers
#io.file.buffer.size=65536


# source options:
#
# pixel(port)
# syslog(port)
# pipe('filename')
# tcpwrite(port)


# sink options:
#
# seqfile('hdfs://localhost:9000/test/collect/%{yyyy-MM-dd/HH00}/data-%{host}-%{yyyyMMdd-HHmmss}')
# seqfile('file:///opt/sewer/collect/%{yyyy-MM-dd/HH00}/data-%{host}-%{yyyyMMdd-HHmmss}')
# reliableseq('hdfs://localhost:9000/test/collect/%{yyyy-MM-dd/HH00}/data-%{host}-%{yyyyMMdd-HHmmss}')
# dfs("path")
# tcpwrite("host", port)
# buffer
# buffer_pool(size)
# local_buffer(size)
# roll(sec)
# reliable
#
# primarily for testing:
#
# null
# delayed_open


# file bucketing options:
#
# patterns must be enclosed with the special marker %{} and
# must match the following regex: [\w-./]+
# e.g.: A-Z a-z 0-9 _ - . /
#
# patterns consist primarily of standard SimpleDateFormat tokens:
#
#   http://www.docjar.com/docs/api/java/text/SimpleDateFormat.html
#
# common examples:
#
#   %{yyyy-MM-dd/HH00} => 2012-01-20/1300
#   %{yyyyMMdd-HHmmss} => 20120120-133254
#
# the following special variables are also available:
#
# host      - hostname
# rand      - random integer
# nanos     - system time in nanosec
# timestmap - system time in millisec
# thread    - thread id
#
# NOTE: special variables must *not* be combined with standard date/time
#       patterns.




# advanced pixel source options:

# graceful shutdown timeout
# default: 1000
#sewer.source.pixel.graceful=1000

# jetty connetor acceptQueueSize = Number of connection requests that can be
# queued up before the operating system starts to send rejections.
# default: 100
#sewer.source.pixel.accept_queue=100

# reliableseq sink options

# how long to wait for remote stream (to HDFS) to open.
# if there is an outage, opening HDFS can block for up to 20 seconds by default.
# default: 5
#sewer.sink.secure.remote.timeout=5<commit_msg>updated config docs<commit_after>
sewer.source=pixel
sewer.sink=null

# when using the 'reliable' sink, this path will be written to.
# the user running sewer must have read/write permissions to this path.
sewer.wal.path=/opt/sewer/wal

# sample reliable sink writing to HDFS
# sewer.sink=roll(30) > reliable > seqfile('hdfs://localhost:9000/test/collect/%Y-%m-%d/%H00/data-%{host}-%Y%m%d-%k%M%S')



# type of compression to use for writing to file/hdfs streams
# valid options: deflate (zlib), gzip, bzip2, snappy
# note: snappy is only available in newer versions of hadoop
# default: gzip (if hadoop native is available), else deflate
#sewer.sink.compression=gzip


# good default buffer size for hadoop read/write buffers
# haven't seen much/any improvement w/ larger buffers
#io.file.buffer.size=65536


# source options:
#
# pixel(port)
# syslog(port)
# pipe('filename')
# tcpwrite(port)


# sink options:
#
# seqfile('hdfs://localhost:9000/test/collect/%{yyyy-MM-dd/HH00}/data-%{host}-%{yyyyMMdd-HHmmss}')
# seqfile('file:///opt/sewer/collect/%{yyyy-MM-dd/HH00}/data-%{host}-%{yyyyMMdd-HHmmss}')
# reliableseq('hdfs://localhost:9000/test/collect/%{yyyy-MM-dd/HH00}/data-%{host}-%{yyyyMMdd-HHmmss}')
# dfs("path")
# tcpwrite("host", port)
# buffer
# buffer_pool(size)
# local_buffer(size)
# roll(sec)
# reliable
#
# primarily for testing:
#
# null
# delayed_open


# file bucketing options:
#
# patterns must be enclosed with the special marker %{} and
# must match the following regex: [\w-./]+
# e.g.: A-Z a-z 0-9 _ - . /
#
# patterns consist primarily of standard SimpleDateFormat tokens:
#
#   http://www.docjar.com/docs/api/java/text/SimpleDateFormat.html
#
# common examples:
#
#   %{yyyy-MM-dd/HH00} => 2012-01-20/1300
#   %{yyyyMMdd-HHmmss} => 20120120-133254
#
# the following special variables are also available:
#
# host      - hostname
# rand      - random integer
# nanos     - system time in nanosec
# timestmap - system time in millisec
# thread    - thread id
#
# NOTE: special variables must *not* be combined with standard date/time
#       patterns.




# advanced pixel source options:

# graceful shutdown timeout
# default: 1000
#sewer.source.pixel.graceful=1000

# jetty connetor acceptQueueSize = Number of connection requests that can be
# queued up before the operating system starts to send rejections.
# default: 100
#sewer.source.pixel.accept_queue=100

# reliableseq sink options

# how long to wait for remote stream (to HDFS) to open.
# if there is an outage, opening HDFS can block for up to 20 seconds by default.
# default: 5
#sewer.sink.secure.remote.timeout=5

# rotate at even time boundaries (HH:00 and/or HH:30).
# for example, roll(30) with this flag enabled will rotate at HH:00 and HH:30.
# default: false
#sewer.sink.roll.even.boundaries=true
{'fraction_non_alphanumeric': 0.13316023738872404, 'fraction_numerical': 0.030044510385756677, 'mean_word_length': 4.185576923076923, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4295309', 'n_tokens_mistral': 1982, 'n_tokens_neox': 1816, 'n_words': 525}