<commit_before># -*- coding: UTF-8 -*-

import pymongo
import __main__
from bson import ObjectId
import time

def _profile(method_to_decorate):
	def wrapper(*args, **kwargs):
		t_start = time.time()
		result = method_to_decorate(*args, **kwargs)
		t_all = time.time() - t_start
		print t_all
		return result

	return wrapper

class mongoAdapter():

	con = {}
	db = {}
	tweenk_conf = __main__.tweenk_core

	db_name = tweenk_conf.loaded_data['db_name']

	col = {}

	PROFILE = False

	def	__init__(self):
		self.con = pymongo.Connection(
			self.tweenk_conf.loaded_data['db_host'],
			self.tweenk_conf.loaded_data['db_port']
		)
		self.db = self.con[self.db_name]
		self.db.authenticate(
			self.tweenk_conf.loaded_data['db_user'],
			self.tweenk_conf.loaded_data['db_passw']
		)

	def profile(self, name, params):
		if self.PROFILE:
			print name,'\t',params

	def _getSorted(self, sort):
		sort_buff = []

		if isinstance(sort, dict):
			for item in sort:
				sort_buff.append([item, sort[item]])

		return sort_buff

	def setCollection(self,collection_name):
		self.collection = self.db[collection_name]

	def collectionExist(self, collection_name):
		return collection_name in self.db.collection_names()

	def createCollection(self, collection_name):
		try:
			self.db.create_collection(collection_name)
			return True
		except Exception:
			return False

	def cleanUp(self, collection_name):
		if self.collectionExist(collection_name):
			items = self.db[collection_name]
			items.remove()
		else:
			self.createCollection(collection_name)

	def addIndex(self, collection_name, field):
		collection = self.db[collection_name]
		collection.ensure_index(field,unique=True)
		print 'INDEXES FOR '+collection_name+' IS: ', collection.index_information()

	def insert(self,collection_name,item):
		items = self.db[collection_name]
		return items.insert(item)

	def remove(self, collection_name, search = {}):
		self.profile('remove',collection_name)
		items = self.db[collection_name]
		items.remove(search)

	def updateWOSet(self,collection_name, search, update, insert = False, multi = False):
		self.profile('update',collection_name)
		items = self.db[collection_name]
		return items.update(search, update, insert, multi)

	def update(self,collection_name, search, update, insert = False, multi = False):
		self.profile('update',collection_name)
		items = self.db[collection_name]
		return items.update(search, {"$set": update}, insert, multi)

	def deleteKeys(self,collection_name, search, keys):
		self.profile('delete key',collection_name)
		items = self.db[collection_name]

		records = {}
		for key in keys:
			records.update({key:1})

		return items.update(search, {"$unset": record})

	def raw_update(self,collection_name, search, update, insert = False, multi = False):
		self.profile('raw update',collection_name)
		items = self.db[collection_name]
		return items.update(search, update, insert, multi)

	def raw_many_update(self,collection_name, search, update):
		self.profile('raw update',collection_name)
		items = self.db[collection_name]
		return items.update(search, update, multi=True)

	def updateInc(self,collection_name, search, update, insert = False, multi = False):
		self.profile('update',collection_name)
		items = self.db[collection_name]
		return items.update(search, {"$inc": update}, insert, multi)

	def findSomeIDs(self,collection_name, ids, fields={}, sort = {}):<commit_msg>XSS fix<commit_after># -*- coding: UTF-8 -*-

import pymongo
import re
import __main__
from bson import ObjectId
import time

def _profile(method_to_decorate):
	def wrapper(*args, **kwargs):
		t_start = time.time()
		result = method_to_decorate(*args, **kwargs)
		t_all = time.time() - t_start
		print t_all
		return result

	return wrapper

def cleanhtml(raw_html):
	cleanr = re.compile('<.*?>')
	cleantext = re.sub(cleanr, '', raw_html)
	return cleantext

def sanitize(d):
	new_dict = {}
	for key, value in d.iteritems():
		if isinstance(value, dict):
			new_dict[key] = sanitize(value)
		elif type(value) == str or type(value) == unicode:
			new_dict[key] = cleanhtml(value)
		else:
			new_dict[key] = value

	return new_dict

class mongoAdapter():

	con = {}
	db = {}
	tweenk_conf = __main__.tweenk_core

	db_name = tweenk_conf.loaded_data['db_name']

	col = {}

	PROFILE = False

	def	__init__(self):
		self.con = pymongo.Connection(
			self.tweenk_conf.loaded_data['db_host'],
			self.tweenk_conf.loaded_data['db_port']
		)
		self.db = self.con[self.db_name]
		self.db.authenticate(
			self.tweenk_conf.loaded_data['db_user'],
			self.tweenk_conf.loaded_data['db_passw']
		)

	def profile(self, name, params):
		if self.PROFILE:
			print name,'\t',params

	def _getSorted(self, sort):
		sort_buff = []

		if isinstance(sort, dict):
			for item in sort:
				sort_buff.append([item, sort[item]])

		return sort_buff

	def setCollection(self,collection_name):
		self.collection = self.db[collection_name]

	def collectionExist(self, collection_name):
		return collection_name in self.db.collection_names()

	def createCollection(self, collection_name):
		try:
			self.db.create_collection(collection_name)
			return True
		except Exception:
			return False

	def cleanUp(self, collection_name):
		if self.collectionExist(collection_name):
			items = self.db[collection_name]
			items.remove()
		else:
			self.createCollection(collection_name)

	def addIndex(self, collection_name, field):
		collection = self.db[collection_name]
		collection.ensure_index(field,unique=True)
		print 'INDEXES FOR '+collection_name+' IS: ', collection.index_information()

	def insert(self,collection_name,item):
		items = self.db[collection_name]
		return items.insert(sanitize(item))

	def remove(self, collection_name, search = {}):
		self.profile('remove',collection_name)
		items = self.db[collection_name]
		items.remove(search)

	def updateWOSet(self,collection_name, search, update, insert = False, multi = False):
		self.profile('update',collection_name)
		items = self.db[collection_name]
		return items.update(search, sanitize(update), insert, multi)

	def update(self,collection_name, search, update, insert = False, multi = False):
		self.profile('update',collection_name)
		items = self.db[collection_name]
		return items.update(search, {"$set": sanitize(update)}, insert, multi)

	def deleteKeys(self,collection_name, search, keys):
		self.profile('delete key',collection_name)
		items = self.db[collection_name]

		records = {}
		for key in keys:
			records.update({key:1})

		return items.update(search, {"$unset": record})

	def raw_update(self,collection_name, search, update, insert = False, multi = False):
		self.profile('raw update',collection_name)
		items = self.db[collection_name]
		return items.update(search, sanitize(update), insert, multi)

	def raw_many_update(self,collection_name, search, update):
		self.profile('raw update',collection_name)
		items = self.db[collection_name]
		return items.update(search, sanitize(update), multi=True)

	def updateInc(self,collection_name, search, update, insert = False, multi = False):
		self.profile('update',collection_name)
		items = self.db[collection_name]
		return items.update(search, {"$inc": sanitize(update)}, insert, multi)

	def findSomeIDs(self,collection_name, ids, fields={}, sort = {}):
{'fraction_non_alphanumeric': 0.12720264317180616, 'fraction_numerical': 0.0005506607929515419, 'mean_word_length': 5.257536606373816, 'pattern_counts': {'":': 6, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1719557', 'n_tokens_mistral': 2604, 'n_tokens_neox': 2387, 'n_words': 514}