<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Provides a service to store ROS message objects in a mongodb database in JSON.
"""

import rospy
import mongodb_store_msgs.srv as dc_srv
import mongodb_store.util as dc_util
import pymongo
from pymongo import GEO2D
import json
from bson import json_util
from mongodb_store_msgs.msg import  StringPair, StringPairList
from bson.objectid import ObjectId
from datetime import *

MongoClient = dc_util.import_MongoClient()

class MessageStore(object):
    def __init__(self, replicate_on_write=False):

        self.replicate_on_write = replicate_on_write

	self.checker = False

        use_daemon = rospy.get_param('mongodb_use_daemon', False)
        if use_daemon:            
            db_host = rospy.get_param('mongodb_host')
            db_port = rospy.get_param('mongodb_port')
            is_daemon_alive = dc_util.check_connection_to_mongod(db_host, db_port)
            if not is_daemon_alive:
                raise Exception("No Daemon?")
        else:
            have_dc = dc_util.wait_for_mongo()
            if not have_dc:
                raise Exception("No Datacentre?")
            # move these to after the wait_for_mongo check as they may not be set before the db is available
            db_host = rospy.get_param('mongodb_host')
            db_port = rospy.get_param('mongodb_port')


        self._mongo_client=MongoClient(db_host, db_port)


        extras = rospy.get_param('mongodb_store_extras', [])
        self.extra_clients = []
        for extra in extras:
            try:
                self.extra_clients.append(MongoClient(extra[0], extra[1]))
            except pymongo.errors.ConnectionFailure, e:
                rospy.logwarn('Could not connect to extra datacentre at %s:%s' % (extra[0], extra[1]))

        if self.replicate_on_write:            
            rospy.loginfo('Replicating content to a futher %s datacentres',len(self.extra_clients))
        else:
            rospy.loginfo('Querying content in a futher %s datacentres',len(self.extra_clients))

        # advertise ros services
        for attr in dir(self):
            if attr.endswith("_ros_srv"):
                service=getattr(self, attr)
                rospy.Service("/message_store/"+attr[:-8], service.type, service)



    def insert_ros_srv(self, req):
        """
        Receives a 
        """


        # deserialize data into object
        obj = dc_util.deserialise_message(req.message)        
        # convert input tuple to dict
        meta = dc_util.string_pair_list_to_dictionary(req.meta)
        # get requested collection from the db, creating if necessary
        collection = self._mongo_client[req.database][req.collection]
	
	#check if the object has the location attribute
	if hasattr(obj, 'pose'):
	  # print "I have attribute"
	   # if it does create a location index
    	   collection.create_index([("loc", pymongo.GEO2D)])
        #check if the object has the location attribute
	if hasattr(obj, 'geotype'):
	 #  print "I have attribute"
	   # if it does create a location index
    	   collection.create_index([("geoloc", pymongo.GEOSPHERE)])
	#else:
	 #  print "I don't have attribute"
	
	#check if the object has the timestamp attribute TODO ?? really necessary
	#if hasattr(obj, 'logtimestamp'):
	   # if it does create a location index
    	 #  collection.create_index([("datetime", pymongo.GEO2D)])

	#if self.checker == False:
	#   self.checker = True
	
	#   rospy.loginfo("Hello")

        # try:
        meta['inserted_at'] = datetime.utcfromtimestamp(rospy.get_rostime().to_sec())
        meta['inserted_by'] = req._connection_header['callerid']
        #meta['foo'] = 'bar'
	#obj['loc2'] = [5,5]
        obj_id = dc_util.store_message(collection, obj, meta)
      

        if self.replicate_on_write:            
            # also do insert to extra datacentres, making sure object ids are consistent
            for extra_client in self.extra_clients:
                extra_collection = extra_client[req.database][req.collection]            
                dc_util.store_message(extra_collection, obj, meta, obj_id)
<commit_msg>Remove unnecessary print statements<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Provides a service to store ROS message objects in a mongodb database in JSON.
"""

import rospy
import mongodb_store_msgs.srv as dc_srv
import mongodb_store.util as dc_util
import pymongo
from pymongo import GEO2D
import json
from bson import json_util
from mongodb_store_msgs.msg import  StringPair, StringPairList
from bson.objectid import ObjectId
from datetime import *

MongoClient = dc_util.import_MongoClient()

class MessageStore(object):
    def __init__(self, replicate_on_write=False):

        self.replicate_on_write = replicate_on_write

	self.checker = False

        use_daemon = rospy.get_param('mongodb_use_daemon', False)
        if use_daemon:            
            db_host = rospy.get_param('mongodb_host')
            db_port = rospy.get_param('mongodb_port')
            is_daemon_alive = dc_util.check_connection_to_mongod(db_host, db_port)
            if not is_daemon_alive:
                raise Exception("No Daemon?")
        else:
            have_dc = dc_util.wait_for_mongo()
            if not have_dc:
                raise Exception("No Datacentre?")
            # move these to after the wait_for_mongo check as they may not be set before the db is available
            db_host = rospy.get_param('mongodb_host')
            db_port = rospy.get_param('mongodb_port')


        self._mongo_client=MongoClient(db_host, db_port)


        extras = rospy.get_param('mongodb_store_extras', [])
        self.extra_clients = []
        for extra in extras:
            try:
                self.extra_clients.append(MongoClient(extra[0], extra[1]))
            except pymongo.errors.ConnectionFailure, e:
                rospy.logwarn('Could not connect to extra datacentre at %s:%s' % (extra[0], extra[1]))

        if self.replicate_on_write:            
            rospy.loginfo('Replicating content to a futher %s datacentres',len(self.extra_clients))
        else:
            rospy.loginfo('Querying content in a futher %s datacentres',len(self.extra_clients))

        # advertise ros services
        for attr in dir(self):
            if attr.endswith("_ros_srv"):
                service=getattr(self, attr)
                rospy.Service("/message_store/"+attr[:-8], service.type, service)



    def insert_ros_srv(self, req):
        """
        Receives a 
        """


        # deserialize data into object
        obj = dc_util.deserialise_message(req.message)        
        # convert input tuple to dict
        meta = dc_util.string_pair_list_to_dictionary(req.meta)
        # get requested collection from the db, creating if necessary
        collection = self._mongo_client[req.database][req.collection]
	
	#check if the object has the location attribute
	if hasattr(obj, 'pose'):
	   # if it does create a location index
    	   collection.create_index([("loc", pymongo.GEO2D)])
        #check if the object has the location attribute
	if hasattr(obj, 'geotype'):
	   # if it does create a location index
    	   collection.create_index([("geoloc", pymongo.GEOSPHERE)])
	
	#check if the object has the timestamp attribute TODO ?? really necessary
	#if hasattr(obj, 'logtimestamp'):
	   # if it does create a location index
    	 #  collection.create_index([("datetime", pymongo.GEO2D)])

	#if self.checker == False:
	#   self.checker = True
	

        # try:
        meta['inserted_at'] = datetime.utcfromtimestamp(rospy.get_rostime().to_sec())
        meta['inserted_by'] = req._connection_header['callerid']
        obj_id = dc_util.store_message(collection, obj, meta)
      

        if self.replicate_on_write:            
            # also do insert to extra datacentres, making sure object ids are consistent
            for extra_client in self.extra_clients:
                extra_collection = extra_client[req.database][req.collection]            
                dc_util.store_message(extra_collection, obj, meta, obj_id)

{'fraction_non_alphanumeric': 0.09171270718232044, 'fraction_numerical': 0.002578268876611418, 'mean_word_length': 2.6463742166517457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '922813', 'n_tokens_mistral': 2568, 'n_tokens_neox': 2460, 'n_words': 618}