<commit_before>import os
import sys
import time
import datetime
import syslog
import boto.sqs
import boto.sqs.message
from boto.exception import SQSError
import boto.utils

#####
##### Subroutines ######
#####
def myLog(message, toSyslog=False):
	procName = __file__
	print '%s' % message
		
	if toSyslog:
		syslogMsg = procName + ": " + message
		syslog.syslog(syslogMsg)

def findQueues(sqsConn,pattern):
	myLog("Looking for queues containing pattern %s" % pattern)
	
	matchingQueues = []
	
	queues = sqsConn.get_all_queues()
	
	for queue in queues:
		if verbose: myLog("Found queue %s" % queue.name)
		if pattern in queue.name:
			if verbose: myLog("Found matching queue %s" %queue.name)
			matchingQueues.append(queue.name)
	
	return matchingQueues

def replaceMessage(queueName,sourceStr):
	# get a message from the queue, delete it and add a new one

	if verbose: myLog("replaceMessage: queueName %s" % queueName)
<commit_msg>flush stdout so we get docker logs<commit_after>import os
import sys
import time
import datetime
import syslog
import boto.sqs
import boto.sqs.message
from boto.exception import SQSError
import boto.utils

#####
##### Subroutines ######
#####
def myLog(message, toSyslog=False):
	procName = __file__
	print '%s' % message
	sys.stdout.flush()
		
	if toSyslog:
		syslogMsg = procName + ": " + message
		syslog.syslog(syslogMsg)

def findQueues(sqsConn,pattern):
	myLog("Looking for queues containing pattern %s" % pattern)
	
	matchingQueues = []
	
	queues = sqsConn.get_all_queues()
	
	for queue in queues:
		if verbose: myLog("Found queue %s" % queue.name)
		if pattern in queue.name:
			if verbose: myLog("Found matching queue %s" %queue.name)
			matchingQueues.append(queue.name)
	
	return matchingQueues

def replaceMessage(queueName,sourceStr):
	# get a message from the queue, delete it and add a new one

	if verbose: myLog("replaceMessage: queueName %s" % queueName)

{'fraction_non_alphanumeric': 0.11023622047244094, 'fraction_numerical': 0.0, 'mean_word_length': 4.723723723723723, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3049991', 'n_tokens_mistral': 685, 'n_tokens_neox': 625, 'n_words': 191}