<commit_before>import psycopg2
import csv
import os.path

class Data(object):
	def __init__(self):
		self.pgsql = None
		self.csv = dict()
		# TODO: Support this
		self.shelves = dict()
	
	def __enter__(self):
		""" Object scope was started. Make it
			available :-)
		"""
		return self		
	
	def __exit__(self, type, value, traceback):
		""" Data object was deleted. Say goodbye
		    to the database. Bye bye!
		"""
		# Explicit delete CSV dictionary
		del self.csv
		
		if self.pgsql is not None:
			# If a postgresql connection
			# was established, close it
			self.pgsql.close()	

		for key in self.shelves.keys():
			# Close all open shelves
			self.shelves[key].close()
			
	def pgsqlConnect(self, database, user, password, host = 'localhost', port = '5432', connection_factory=None, cursor_factory=None, async=False, autocommit=True):
		""" Sets the class variable pgsql as the connection
		    to a postgresql database.
		"""
		self.pgsql = psycopg2.connect(
			database = database,
			user = user,
			password = password,
			host = host,
			port = str(port)
		)
		self.pgsql.autocommit = autocommit

	def select(self, cols, table, where = None, params = None):
		""" NOTICE: THE PARAMETERS "cols", "table"
			        AND "where" IS _NOT_ ESCAPED.
			        THEY MUST NEVER BE END-USER INPUT.
			        THE PARAMETER "param" _IS_ ESCAPED
			
			Selects the "cols" (or all if cols = None)
			from "table". If "where" is not None, then
			it must be a string of form
			"col1 = %s AND col2 = %s", and params must
			be an iterable of the length corresponding
			to the number of "%s" in "where".
			
			Yields each row returned (None if empty)
			from a select query on the postgresql
			connection.
		"""
		stm = "SELECT "
		if cols is None:
			stm += "* "
		else:
			stm += "{:s} ".format(cols)
		stm += "FROM {:s} ".format(table)
		
		with self.pgsql.cursor() as cur:
			if where is not None:
				stm += "WHERE {:s};"
				stm = cur.mogrify(stm, args)
			cur.execute(stm)
			for row in cur.fetchall():
				yield row
	
	def insert(self, cols, table, values):
		""" NOTICE: THE PARAMETERS "cols" AND "table"
			        IS _NOT_ ESCAPED. THEY MUST NEVER
			        BE END-USER INPUT. THE PARAMETER
			        "values" _IS_ ESCAPED
			Inserts "values" into "table" in the corresponding
			"cols"
			
			Invokes an insert on the postgresql connection.
			
			Returns: Number of elements inserted
		"""
		values = tuple(values)
		if len(values) == 0:
			# If we should not insert anythin, just return 0
			return 0
			
		stm = "INSERT INTO {:s} ({:s}) VALUES ".format(table, cols)
		with self.pgsql.cursor() as cur:
			s = '(' + ','.join(['%s']*len(values[0])) + ')'
			for i in xrange(0, len(values), self.chunkSize):
				# Some stm might become to long to transmit,
				# thus sending in smaller chunks might help
				args_str = ','.join(cur.mogrify(s, x) for x in values[i:i+chunkSize])
				cur.execute(stm + args_str)		
		
		return len(values)
	
	def update(self, table, where, what, values):
		""" NOTICE: THE PARAMETERS "table", "where",
			        AND "what" IS _NOT_ ESCAPED. THEY
			        MUST NEVER BE END-USER INPUT.
			        THE PARAMETER "values" _IS_ ESCAPED
			Finds the columns "where" the "values" matchs
			in "table" and sets "what" according to "values".
			
			Invokes an update on the postgresql connection.
			
			Returns: Number of elements updated
		"""		
		if len(values) == 0:
			return 0
		stm = "UPDATE {:s} SET {:s} WHERE {:s};".format(table, what, where)
		with self.pgsql.cursor() as cur:
			cur.executemany(stm, values)
		return len(values)
		
	def addCSV(self, alias, path):
		""" Adds a "path" to a file under
		    the "alias" given, only if the
		    path is a file.<commit_msg>pgsql is not 'private' and a getter for the cursor object created<commit_after>import psycopg2
import csv
import os.path

class Data(object):
	def __init__(self):
		self._pgsql = None
		self.csv = dict()
		# TODO: Support this
		self.shelves = dict()
	
	def __enter__(self):
		""" Object scope was started. Make it
			available :-)
		"""
		return self		
	
	def __exit__(self, type, value, traceback):
		""" Data object was deleted. Say goodbye
		    to the database. Bye bye!
		"""
		# Explicit delete CSV dictionary
		del self.csv
		
		if self._pgsql is not None:
			# If a postgresql connection
			# was established, close it
			self._pgsql.close()	

		for key in self.shelves.keys():
			# Close all open shelves
			self.shelves[key].close()
			
	def pgsqlConnect(self, database, user, password, host = 'localhost', port = '5432', connection_factory=None, cursor_factory=None, async=False, autocommit=True):
		""" Sets the class variable pgsql as the connection
		    to a postgresql database.
		"""
		self._pgsql = psycopg2.connect(
			database = database,
			user = user,
			password = password,
			host = host,
			port = str(port)
		)
		self._pgsql.autocommit = autocommit
	
	def pgCursor(self):
		""" Returns a cursor object for
		    the postgresql connection
		    or None if connection not
		    open.
		"""
		return self._pgsql.cursor() if self._pgsql is not None else None

	def select(self, cols, table, where = None, params = None):
		""" NOTICE: THE PARAMETERS "cols", "table"
			        AND "where" IS _NOT_ ESCAPED.
			        THEY MUST NEVER BE END-USER INPUT.
			        THE PARAMETER "param" _IS_ ESCAPED
			
			Selects the "cols" (or all if cols = None)
			from "table". If "where" is not None, then
			it must be a string of form
			"col1 = %s AND col2 = %s", and params must
			be an iterable of the length corresponding
			to the number of "%s" in "where".
			
			Yields each row returned (None if empty)
			from a select query on the postgresql
			connection.
		"""
		stm = "SELECT "
		if cols is None:
			stm += "* "
		else:
			stm += "{:s} ".format(cols)
		stm += "FROM {:s} ".format(table)
		
		with self._pgsql.cursor() as cur:
			if where is not None:
				stm += "WHERE {:s};"
				stm = cur.mogrify(stm, args)
			cur.execute(stm)
			for row in cur.fetchall():
				yield row
	
	def insert(self, cols, table, values):
		""" NOTICE: THE PARAMETERS "cols" AND "table"
			        IS _NOT_ ESCAPED. THEY MUST NEVER
			        BE END-USER INPUT. THE PARAMETER
			        "values" _IS_ ESCAPED
			Inserts "values" into "table" in the corresponding
			"cols"
			
			Invokes an insert on the postgresql connection.
			
			Returns: Number of elements inserted
		"""
		values = tuple(values)
		if len(values) == 0:
			# If we should not insert anythin, just return 0
			return 0
			
		stm = "INSERT INTO {:s} ({:s}) VALUES ".format(table, cols)
		with self._pgsql.cursor() as cur:
			s = '(' + ','.join(['%s']*len(values[0])) + ')'
			for i in xrange(0, len(values), self.chunkSize):
				# Some stm might become to long to transmit,
				# thus sending in smaller chunks might help
				args_str = ','.join(cur.mogrify(s, x) for x in values[i:i+chunkSize])
				cur.execute(stm + args_str)		
		
		return len(values)
	
	def update(self, table, where, what, values):
		""" NOTICE: THE PARAMETERS "table", "where",
			        AND "what" IS _NOT_ ESCAPED. THEY
			        MUST NEVER BE END-USER INPUT.
			        THE PARAMETER "values" _IS_ ESCAPED
			Finds the columns "where" the "values" matchs
			in "table" and sets "what" according to "values".
			
			Invokes an update on the postgresql connection.
			
			Returns: Number of elements updated
		"""		
		if len(values) == 0:
			return 0
		stm = "UPDATE {:s} SET {:s} WHERE {:s};".format(table, what, where)
		with self._pgsql.cursor() as cur:
			cur.executemany(stm, values)
		return len(values)
		
	def addCSV(self, alias, path):
		""" Adds a "path" to a file under
		    the "alias" given, only if the
		    path is a file.
{'fraction_non_alphanumeric': 0.11550427793621985, 'fraction_numerical': 0.00388903292714545, 'mean_word_length': 2.815529179030663, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5786899', 'n_tokens_mistral': 3094, 'n_tokens_neox': 2572, 'n_words': 923}