<commit_before>thinkbig.provenance.datasource.url=jdbc:h2:file:/opt/nifi/h2/thinkbig-nifi
thinkbig.provenance.datasource.username=sa
thinkbig.provenance.datasource.password=
thinkbig.provenance.datasource.driverClassName=org.h2.Driver
thinkbig.provenance.h2.databaseName=/opt/nifi/h2/thinkbig-nifi
thinkbig.provenance.h2.showWebConsole=true
thinkbig.provenance.h2.webConsolePort=8082

### How to determine if it is a stream.  Default config will queue events for 5 seconds.
### Queue events for {processDelay} milliseconds.   A stream is determined where any processor(partitioned by feed) that receives ${numberOfEventsToConsiderAStream} or more events where the time between each event < ${maxTimeBetweenEventsMillis}
thinkbig.provenance.streamconfig.processDelay=5000
thinkbig.provenance.streamconfig.maxTimeBetweenEventsMillis=1000
thinkbig.provenance.streamconfig.numberOfEventsToConsiderAStream=3
thinkbig.provenance.aggregation.interval.seconds=10


### number of days to keep the cache for any running flow files in memory(@see FlowFileMapDbCache)
thinkbig.provenance.flowfile.cache.persistence.days=3

## how often it should check and expire finished flow files
thinkbig.provenance.flowfile.cache.expire.seconds=10

spring.activemq.broker-url=tcp://localhost:61616
spring.activemq.pooled=true<commit_msg>update config properties for streaming settings<commit_after>thinkbig.provenance.datasource.url=jdbc:h2:file:/opt/nifi/h2/thinkbig-nifi
thinkbig.provenance.datasource.username=sa
thinkbig.provenance.datasource.password=
thinkbig.provenance.datasource.driverClassName=org.h2.Driver
thinkbig.provenance.h2.databaseName=/opt/nifi/h2/thinkbig-nifi
thinkbig.provenance.h2.showWebConsole=true
thinkbig.provenance.h2.webConsolePort=8082

spring.activemq.broker-url=tcp://localhost:61616
spring.activemq.pooled=true

### How to determine if it is a stream.  Default config will queue events for 5 seconds.
### Queue events for {processDelay} milliseconds.   A stream is determined where any processor(partitioned by feed) that receives ${numberOfEventsToConsiderAStream} or more events where the time between each event < ${maxTimeBetweenEventsMillis}
thinkbig.provenance.streamconfig.processDelay=5000
thinkbig.provenance.streamconfig.maxTimeBetweenEventsMillis=1000
thinkbig.provenance.streamconfig.numberOfEventsToConsiderAStream=3
thinkbig.provenance.aggregation.interval.seconds=10


### number of days to keep the cache for any running flow files in memory(@see FlowFileMapDbCache)
thinkbig.provenance.flowfile.cache.persistence.days=3

## how often it should check and expire finished flow files
thinkbig.provenance.flowfile.cache.expire.seconds=10

## the time to wait in millis before sending full events to ops manager
thinkbig.provenance.jmsBatchDelay=7000

## Nifi Rest Client connection info
thinkbig.nifi.rest.host=http://localhost:8079
thinkbig.nifi.rest.username=
thinkbig.nifi.rest.password=

{'fraction_non_alphanumeric': 0.08613589567604667, 'fraction_numerical': 0.024021962937542895, 'mean_word_length': 6.921195652173913, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '24093', 'n_tokens_mistral': 937, 'n_tokens_neox': 882, 'n_words': 192}