<commit_before>import os
import argparse
import json
import logging
import datetime
from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor
from pymongo import MongoClient

logger = logging.getLogger(__name__)

class Echo(DatagramProtocol):

    def __init__(self, log_file=None, *args, **kwargs):
        self.log_file = log_file
        print self.log_file
        # DatagramProtocol.__init__(self, *args, **kwargs)
        # super(Echo, self).__init__(*args, **kwargs)

    def datagramReceived(self, data, (host, port)):
        try:
            json_ = json.loads(data)
            client = MongoClient()
            logging.info('MongoDB client: {}'.format(client))
            logging.info('Creating/getting collection: tfl_data')
            db = client.tfl_data
            logging.info('db: {}'.format(db))
            logging.info('Creating/getting my_collection')
            collection = db.my_collection
            logging.info('my_collection: {}'.format(collection))
            logging.info('inserting entry into collection:')
            json_['datetime'] = datetime.datetime.now()
            result = collection.insert_one(json_)
            logging.info('returned output: {}'.format(result))
            logging.info('collection contents:')
            cursor = collection.find()
            for document in cursor:
                logging.info(document)
            
        except ValueError as e:
            print "received %r from %s:%d" % (data, host, port)

        if self.log_file:
            self.log_file.write('{datetime}: {data}\n'.format(datetime=datetime.datetime.now(), data=data))
        
        # self.transport.write(data, (host, port))

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)

    parser = argparse.ArgumentParser(description="Starts a basic UDP server.")
    parser.add_argument('--port', type=int, default=9999)
    parser.add_argument('--log-file', default='log.log')
    args = parser.parse_args()

    print args

    with open(args.log_file, 'wb', 0) as f:

        reactor.listenUDP(9999, Echo(f))
        reactor.run()<commit_msg>Modify server to save data into a rotating collection and rotating log file<commit_after>import os
import argparse
import json
import logging
import datetime
import pymongo
import logging.handlers
from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor
from pymongo import MongoClient

logger = logging.getLogger(__name__)

class Echo(DatagramProtocol):

    def __init__(self, *args, **kwargs):
        self.client = MongoClient()
        self.db = self.client.database
        try:
            self.collection = self.db.create_collection('collection', capped=True, size=100)
            print 'creating new collection'
        except pymongo.errors.CollectionInvalid:
            self.collection = self.db.collection

    def datagramReceived(self, data, (host, port)):
        document = {}
        document['datetime'] = datetime.datetime.now()
        document['msg'] = data
        result = self.collection.insert_one(document)
        logger.info(data)
        
        # self.transport.write(data, (host, port))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Starts a basic UDP server.")
    parser.add_argument('--port', type=int, default=9999)
    parser.add_argument('--log-file', default='log.log')
    args = parser.parse_args()

    print 'args:', args

    handler = logging.handlers.RotatingFileHandler(args.log_file, maxBytes=1e6, backupCount=3)
    formatter = logging.Formatter("%(asctime)s: %(message)s", '%Y-%m-%d %H:%M:%S')
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    logger.setLevel(logging.INFO)

    reactor.listenUDP(9999, Echo())
    reactor.run()
{'fraction_non_alphanumeric': 0.11995809324253535, 'fraction_numerical': 0.006024096385542169, 'mean_word_length': 3.02, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5489283', 'n_tokens_mistral': 1117, 'n_tokens_neox': 1068, 'n_words': 247}