<commit_before># Name the components on this agent
a.sources=src

#a.sinks=elasticSink cassandraSink
#a.sinks=streamingSink

a.channels=elasticChannel cassandraChannel
#a.channels=streamingChannel

#############################################
#                   SOURCES                 #
#############################################
# Mapping for multiplexing selector
a.sources.src.type=spoolDir
a.sources.src.spoolDir=data/spooldir
a.sources.src.fileHeader=true
a.sources.src.batchSize=100
a.sources.src.deserializer=LINE
a.sources.src.deserializer.decodeErrorPolicy=REPLACE

# Define the interceptors
a.sources.src.interceptors = morphlineinterceptor
a.sources.src.interceptors.morphlineinterceptor.type = org.apache.flume.sink.solr.morphline.MorphlineInterceptor$Builder
a.sources.src.interceptors.morphlineinterceptor.morphlineFile = conf/interceptor.conf
a.sources.src.interceptors.morphlineinterceptor.morphlineId = morphline1

#############################################
#                   CHANNELS                #
#############################################
a.channels.elasticChannel.type=file
a.channels.elasticChannel.checkpointDir=data/chan1/check/
a.channels.elasticChannel.dataDirs=data/chan1/data/
a.channels.elasticChannel.capacity=1000000
a.channels.elasticChannel.transactionCapacity=10000
a.channels.elasticChannel.checkpointInterval=300000

a.channels.cassandraChannel.type=file
a.channels.cassandraChannel.checkpointDir=data/chan2/check/
a.channels.cassandraChannel.dataDirs=data/chan2/data/
a.channels.cassandraChannel.capacity=1000000
a.channels.cassandraChannel.transactionCapacity=10000
a.channels.cassandraChannel.checkpointInterval=300000

a.channels.streamingChannel.type=file
a.channels.streamingChannel.checkpointDir=data/chan3/check/
a.channels.streamingChannel.dataDirs=data/chan3/data/
a.channels.streamingChannel.capacity=1000000
a.channels.streamingChannel.transactionCapacity=10000
a.channels.streamingChannel.checkpointInterval=300000

#############################################
#                   SINKS                   #
#############################################
a.sinks.elasticSink.type=org.apache.flume.sink.elasticsearch.ElasticSearchSink
a.sinks.elasticSink.batchSize=1000
a.sinks.elasticSink.indexName=apache_logs
a.sinks.elasticSink.hostNames=127.0.0.1:9300
a.sinks.elasticSink.clusterName=elasticsearch
a.sinks.elasticSink.serializer=com.stratio.ingestion.serializer.elasticsearch.ElasticSearchSerializerWithMapping
a.sinks.elasticSink.serializer.mappingFile=conf/elasticsearch_mapping.json

a.sinks.cassandraSink.type = com.stratio.ingestion.sink.cassandra.CassandraSink
a.sinks.cassandraSink.cluster-name = Stratio Cluster
a.sinks.cassandraSink.host = localhost
a.sinks.cassandraSink.tables = test.access_logs
a.sinks.cassandraSink.batchSize = 200
a.sinks.cassandraSink.port = 9042
a.sinks.cassandraSink.cqlFile = /opt/sds/ingestion/examples/apache-logs/conf/init_cassandra.cql


a.sinks.streamingSink.type=com.stratio.ingestion.sink.stratiostreaming.StratioStreamingSink
a.sinks.streamingSink.kafkaHost=localhost
a.sinks.streamingSink.kafkaPort=9092
a.sinks.streamingSink.zookeeperHost=localhost
a.sinks.streamingSink.zookeeperPort=2181
a.sinks.streamingSink.streamDefinitionFile=/opt/flume/conf/stream.conf
#a.sinks.streamingSink.streamFields=log_id,log_host,log_user,log_date,log_http_method,log_url_path,log_http_version,log_http_code,log_bytes_returned

#############################################
#                  BINDING                  #
#############################################
# Bind the source and sinks to the channel
a.sources.src.channels=elasticChannel cassandraChannel
#a.sources.src.channels=streamingChannel
a.sinks.elasticSink.channel=elasticChannel
a.sinks.cassandraSink.channel=cassandraChannel
#a.sinks.streamingSink.channel=streamingChannel

<commit_msg>Changes in apache-logs example and sandbox<commit_after># Name the components on this agent
a.sources=src

a.sinks=elasticSink cassandraSink
#a.sinks=streamingSink

a.channels=elasticChannel cassandraChannel
#a.channels=streamingChannel

#############################################
#                   SOURCES                 #
#############################################
# Mapping for multiplexing selector
a.sources.src.type=spoolDir
a.sources.src.spoolDir=data/spooldir
a.sources.src.fileHeader=true
a.sources.src.batchSize=100
a.sources.src.deserializer=LINE
a.sources.src.deserializer.decodeErrorPolicy=REPLACE

# Define the interceptors
a.sources.src.interceptors = morphlineinterceptor
a.sources.src.interceptors.morphlineinterceptor.type = org.apache.flume.sink.solr.morphline.MorphlineInterceptor$Builder
a.sources.src.interceptors.morphlineinterceptor.morphlineFile = conf/interceptor.conf
a.sources.src.interceptors.morphlineinterceptor.morphlineId = morphline1

#############################################
#                   CHANNELS                #
#############################################
a.channels.elasticChannel.type=file
a.channels.elasticChannel.checkpointDir=data/chan1/check/
a.channels.elasticChannel.dataDirs=data/chan1/data/
a.channels.elasticChannel.capacity=1000000
a.channels.elasticChannel.transactionCapacity=10000
a.channels.elasticChannel.checkpointInterval=300000

a.channels.cassandraChannel.type=file
a.channels.cassandraChannel.checkpointDir=data/chan2/check/
a.channels.cassandraChannel.dataDirs=data/chan2/data/
a.channels.cassandraChannel.capacity=1000000
a.channels.cassandraChannel.transactionCapacity=10000
a.channels.cassandraChannel.checkpointInterval=300000

a.channels.streamingChannel.type=file
a.channels.streamingChannel.checkpointDir=data/chan3/check/
a.channels.streamingChannel.dataDirs=data/chan3/data/
a.channels.streamingChannel.capacity=1000000
a.channels.streamingChannel.transactionCapacity=10000
a.channels.streamingChannel.checkpointInterval=300000

#############################################
#                   SINKS                   #
#############################################
a.sinks.elasticSink.type=org.apache.flume.sink.elasticsearch.ElasticSearchSink
a.sinks.elasticSink.batchSize=1000
a.sinks.elasticSink.indexName=apache_logs
a.sinks.elasticSink.hostNames=127.0.0.1:9300
a.sinks.elasticSink.clusterName=elasticsearch
a.sinks.elasticSink.serializer=com.stratio.ingestion.serializer.elasticsearch.ElasticSearchSerializerWithMapping
a.sinks.elasticSink.serializer.mappingFile=conf/elasticsearch_mapping.json

a.sinks.cassandraSink.type = com.stratio.ingestion.sink.cassandra.CassandraSink
a.sinks.cassandraSink.cluster-name = Stratio Cluster
a.sinks.cassandraSink.host = localhost
a.sinks.cassandraSink.tables = test.access_logs
a.sinks.cassandraSink.batchSize = 200
a.sinks.cassandraSink.port = 9042
a.sinks.cassandraSink.cqlFile = /opt/sds/ingestion/examples/apache-logs/conf/init_cassandra.cql


a.sinks.streamingSink.type=com.stratio.ingestion.sink.stratiostreaming.StratioStreamingSink
a.sinks.streamingSink.kafkaHost=localhost
a.sinks.streamingSink.kafkaPort=9092
a.sinks.streamingSink.zookeeperHost=localhost
a.sinks.streamingSink.zookeeperPort=2181
a.sinks.streamingSink.streamDefinitionFile=/opt/flume/conf/stream.conf
#a.sinks.streamingSink.streamFields=log_id,log_host,log_user,log_date,log_http_method,log_url_path,log_http_version,log_http_code,log_bytes_returned

#############################################
#                  BINDING                  #
#############################################
# Bind the source and sinks to the channel
a.sources.src.channels=elasticChannel cassandraChannel
#a.sources.src.channels=streamingChannel
a.sinks.elasticSink.channel=elasticChannel
a.sinks.cassandraSink.channel=cassandraChannel
#a.sinks.streamingSink.channel=streamingChannel


{'fraction_non_alphanumeric': 0.1792086889061288, 'fraction_numerical': 0.02404965089216447, 'mean_word_length': 6.345679012345679, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5209152', 'n_tokens_mistral': 2609, 'n_tokens_neox': 2428, 'n_words': 205}