<commit_before>
import os
import time
from pymongo import MongoClient
from spojrec.src.crawler.crawler.dataExtractor.signedlistParser import parseSignedlist

class ProblemsDatabase(object):
	def __init__(self, loadMetricsOnly=False):
		PRODUCTION = True
		
		if PRODUCTION:
			host = os.environ['OPENSHIFT_MONGODB_DB_HOST']
			port = os.environ['OPENSHIFT_MONGODB_DB_PORT']
			self.client = MongoClient('mongodb://' + host + ':' + port + '/')
			self.client.index.authenticate('admin', 'vhZcQNxhPHwe', mechanism='MONGODB-CR')
			self.db = self.client.index
		else:
			self.client = MongoClient()
			self.db = self.client.spojrec
		
		if loadMetricsOnly:
			self._metrics = self._load_metrics()
		else:
			self.parsedProblemsByUser = self.load_problems()
			print 'estou vivo'
			#self._split(self.parsedProblemsByUser)
	
	
	def load_problems(self):
		parsedProblemsByUser = dict()
		
		cnt = 0
		for submission in self.db.submissionData.find().batch_size(100):
			cnt += 1
			if cnt %100 == 0:
				print 'parsing', cnt
				
			problems = [{"PROBLEM" : pp['PROBLEM'], 'AC':pp['RESULT']} for pp in parseSignedlist(submission['data'])]
			parsedProblemsByUser[submission['spojId']] = problems
		
		return parsedProblemsByUser
	
	
	def get_problems_of_user_from_db(self, spojId):
		submission = self.db.submissionData.find_one({"spojId": spojId})
		ret = dict()
		ret[spojId] = parseSignedlist(submission['data'])
		return ret
	
	def save_metrics(self, metrics):<commit_msg>solving memory issues. 	modified:   spojrec/src/recommender/database.py<commit_after>
import os
import time
from pymongo import MongoClient
from spojrec.src.crawler.crawler.dataExtractor.signedlistParser import parseSignedlist

class ProblemsDatabase(object):
	def __init__(self, loadMetricsOnly=False):
		PRODUCTION = True
		
		if PRODUCTION:
			host = os.environ['OPENSHIFT_MONGODB_DB_HOST']
			port = os.environ['OPENSHIFT_MONGODB_DB_PORT']
			self.client = MongoClient('mongodb://' + host + ':' + port + '/')
			self.client.index.authenticate('admin', 'vhZcQNxhPHwe', mechanism='MONGODB-CR')
			self.db = self.client.index
		else:
			self.client = MongoClient()
			self.db = self.client.spojrec
		
		if loadMetricsOnly:
			self._metrics = self._load_metrics()
		else:
			self.parsedProblemsByUser = self.load_problems()
			print 'estou vivo'
			#self._split(self.parsedProblemsByUser)
	
	
	def load_problems(self):
		parsedProblemsByUser = dict()
		
		cnt = 0
		for submission in self.db.submissionData.find().batch_size(100):
			cnt += 1
			if cnt %100 == 0:
				print 'parsing', cnt
				
			problems = [{"PROBLEM" : pp['PROBLEM'], 'RESULT':pp['RESULT']} for pp in parseSignedlist(submission['data'])]
			parsedProblemsByUser[submission['spojId']] = problems
		
		return parsedProblemsByUser
	
	
	def get_problems_of_user_from_db(self, spojId):
		submission = self.db.submissionData.find_one({"spojId": spojId})
		ret = dict()
		ret[spojId] = parseSignedlist(submission['data'])
		return ret
	
	def save_metrics(self, metrics):
{'fraction_non_alphanumeric': 0.1252498334443704, 'fraction_numerical': 0.005996002664890073, 'mean_word_length': 4.571428571428571, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '911914', 'n_tokens_mistral': 1231, 'n_tokens_neox': 1052, 'n_words': 197}