<commit_before>import time
from datetime import datetime

import pymongo

db = pymongo.Connection().mongolog

time_key = 'created'
message_key = 'msg'

last_time = list(db.log.find({}, [time_key]).sort(time_key, pymongo.DESCENDING).limit(1))[0][time_key]

while True:
    for row in db.log.find({time_key: {'$gt': last_time}}).sort(time_key, pymongo.ASCENDING):
        message = row.get(message_key, None)
        if message:<commit_msg>add filter capability<commit_after>import sys
import time
from datetime import datetime

import pymongo

db = pymongo.Connection().mongolog

time_key = 'created'
message_key = 'msg'

if len(sys.argv) > 1:
    filter_query = eval(sys.argv[1])
else:
    filter_query = {}

last_time = list(db.log.find(filter_query, [time_key]).sort(time_key, pymongo.DESCENDING).limit(1))[0][time_key]

while True:
    query = {time_key: {'$gt': last_time}}
    query.update(filter_query)
    
    for row in db.log.find(query).sort(time_key, pymongo.ASCENDING):
        message = row.get(message_key, None)
        if message:
{'fraction_non_alphanumeric': 0.1384909264565425, 'fraction_numerical': 0.0057306590257879654, 'mean_word_length': 4.486910994764398, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4504709', 'n_tokens_mistral': 378, 'n_tokens_neox': 376, 'n_words': 77}