<commit_before>"""
Datasource: MySQL: Querying
"""


import threading
import logging

import mysql.connector
from mysql.connector import errorcode

from utility.log import Log


# Default connection pool size.  Override with connection_data
DEFAULT_CONNECTION_POOL_SIZE = 20


# Connection Pool of Pools (servers on first level (list), pool of connections on second (Connection objects)): dict of lists of Connection objects
CONNECTION_POOL_POOL = {}
CONNECTION_POOL_POOL_LOCK = threading.Lock()


#TODO(g): Make this a base class, that each Handler type sub-classes.  Useful in this case, as it's an interface, and some methods are more virtual than others.  It's good to have a base class for the interface, otherwise every handler implements its own base, and they seem more disconnected...
class Connection:
  """This wraps MySQL connection and cursor objects, as well as tracks the progress of any requests, and if it is available for use by a new request."""
  
  def __init__(self, connection_data, server_id, request):
    Log('Creating new connection: MySQL: %s' % connection_data['datasource']['database'])
    
    # We need these to actually connect
    self.connection_data = connection_data
    self.server_id = server_id
    
    # This tells us who is connection.  Release() to make this connection available for other requests.
    self.request = request
    
    # Generate the server key, since this specifies which CONNECTION_POOL_POOL we are in
    self.server_key = '%s.%s' % (connection_data['alias'], server_id)
    
    self.connection = None
    self.cursor = None
    
    # Connect
    self.Connect()
  
  
  def __del__(self):
    try:
      self.Close()
    
    # Ignore this failure, this happens when the context is lost, because the program is closing, and we are working with NoneTypes, instead of expected types
    except AttributeError, e:
      pass


  def Close(self):
    """Close the cursor and connection, if they are open, and set them to None"""
    if self.cursor:
      try:
        self.cursor.close()
      finally:
        self.cursor = None
      
    if self.connection:
      try:
        self.connection.close()
      finally:      
        self.connection = None


  def Release(self):
    """Release this connection."""
    self.request = None


  def Acquire(self, request):
    """Acquire this Connection for this Request."""
    if self.request != None:
      raise Exception('Attempting to Acquire a Connection when it is already owned: %s' % request)
    
    self.request = request

  
  def IsAvailable(self):
    """Returns boolean, True if not currently being used by a request and has a non-None connection.
    
    NOTE(g): This does not verify the connection, just ensures that we think we have a valid connection.
    """
    if self.request_number and self.connection:
      return False
    
    else:
      return True
  

  def IsInUse(self):
    """Returns boolean, True if not currently being used by a request."""
    if self.request:
      return True
    
    else:
      return False


  def IsUsedByRequest(self, request):
    """Returns boolean, True if currently being used by the same request that is being passed in as request."""
    if self.request == request:
      return True
    
    else:
      return False
  
  
  def GetServerData(self):
    """Returns a dict for server data, which is layered from the server configs"""
    server_data = {}
    
    # Find the master host, which we will assume we are connecting to for now
    found_server = None
    for server_data in self.connection_data['datasource']['servers']:
      if server_data['id'] == self.server_id:
        found_server = server_data
        break
    
    if not found_server:
      raise Exception('Couldnt find ourselves in our server data, something is wrong: %s: %s' % (self.connection_data['alias'], self.server_id))
    
    # Set specific datasource fields, these fields may not be present, and are overridden by the server specific fields
    server_data['database'] = self.connection_data['datasource'].get('database', None)
    server_data['user'] = self.connection_data['datasource'].get('user', None)
    server_data['password_path'] = self.connection_data['datasource'].get('password_path', None)
    
    # Override all previous fields with server specific fields, which gives us the final data (most specific)
    server_data.update(found_server)
    
    return server_data
  
  
  def Connect(self):
    """Connect to the database"""
    server = self.GetServerData()
    
    Log('Connecting to MySQL server: %s: %s' % (server['host'], server['database']))
    
    # Read the password from the first line of the password file
    try:
      password = open(server['password_path']).read().split('\n', 1)[0]
      Log('Loaded password: %s' % server['password_path'])
      
    except Exception, e:
      Log('ERROR: Failed to read from password file: %s' % server['password_path'], logging.ERROR)
      password = None
    
    self.connection = mysql.connector.connect(user=server['user'], password=password, host=server['host'], port=server['port'], database=server['database'], use_unicode=True, charset='latin1')
    self.cursor = self.connection.cursor(dictionary=True)


  def Query(self, sql, params=None, commit=True):
    """Query the database via our connection."""
    Log('Query: %s' % sql)
    
    result = Query(self.connection, self.cursor, sql, params=params, commit=commit)
    
    return result
  
  
  def Commit(self):<commit_msg>Fixed connection pool problem, always created new connections<commit_after>"""
Datasource: MySQL: Querying
"""


import threading
import logging

import mysql.connector
from mysql.connector import errorcode

from utility.log import Log


# Default connection pool size.  Override with connection_data
DEFAULT_CONNECTION_POOL_SIZE = 20


# Connection Pool of Pools (servers on first level (list), pool of connections on second (Connection objects)): dict of lists of Connection objects
CONNECTION_POOL_POOL = {}
CONNECTION_POOL_POOL_LOCK = threading.Lock()


# Default MySQL connection character set
# DEFAULT_CHARSET = 'utf8'
DEFAULT_CHARSET = 'latin1'


#TODO(g): Make this a base class, that each Handler type sub-classes.  Useful in this case, as it's an interface, and some methods are more virtual than others.  It's good to have a base class for the interface, otherwise every handler implements its own base, and they seem more disconnected...
class Connection:
  """This wraps MySQL connection and cursor objects, as well as tracks the progress of any requests, and if it is available for use by a new request."""
  
  def __init__(self, connection_data, server_id, request):
    Log('Creating new connection: MySQL: %s' % connection_data['datasource']['database'])
    
    # We need these to actually connect
    self.connection_data = connection_data
    self.server_id = server_id
    
    # This tells us who is connection.  Release() to make this connection available for other requests.
    self.request = request
    
    # Generate the server key, since this specifies which CONNECTION_POOL_POOL we are in
    self.server_key = '%s.%s' % (connection_data['alias'], server_id)
    
    self.connection = None
    self.cursor = None
    
    # Connect
    self.Connect()
  
  
  def __del__(self):
    try:
      self.Close()
    
    # Ignore this failure, this happens when the context is lost, because the program is closing, and we are working with NoneTypes, instead of expected types
    except AttributeError, e:
      pass


  def Close(self):
    """Close the cursor and connection, if they are open, and set them to None"""
    if self.cursor:
      try:
        self.cursor.close()
      finally:
        self.cursor = None
      
    if self.connection:
      try:
        self.connection.close()
      finally:      
        self.connection = None


  def Release(self):
    """Release this connection."""
    self.request = None


  def Acquire(self, request):
    """Acquire this Connection for this Request."""
    if self.request != None:
      raise Exception('Attempting to Acquire a Connection when it is already owned: %s' % request)
    
    self.request = request

  
  def IsAvailable(self):
    """Returns boolean, True if not currently being used by a request and has a non-None connection.
    
    NOTE(g): This does not verify the connection, just ensures that we think we have a valid connection.
    """
    if self.request_number and self.connection:
      return False
    
    else:
      return True
  

  def IsInUse(self):
    """Returns boolean, True if not currently being used by a request."""
    if self.request:
      return True
    
    else:
      return False


  def IsUsedByRequest(self, request):
    """Returns boolean, True if currently being used by the same request that is being passed in as request."""
    if self.request == request:
      return True
    
    else:
      return False
  
  
  def GetServerData(self):
    """Returns a dict for server data, which is layered from the server configs"""
    server_data = {}
    
    # Find the master host, which we will assume we are connecting to for now
    found_server = None
    for server_data in self.connection_data['datasource']['servers']:
      if server_data['id'] == self.server_id:
        found_server = server_data
        break
    
    if not found_server:
      raise Exception('Couldnt find ourselves in our server data, something is wrong: %s: %s' % (self.connection_data['alias'], self.server_id))
    
    # Set specific datasource fields, these fields may not be present, and are overridden by the server specific fields
    server_data['database'] = self.connection_data['datasource'].get('database', None)
    server_data['user'] = self.connection_data['datasource'].get('user', None)
    server_data['password_path'] = self.connection_data['datasource'].get('password_path', None)
    
    # Override all previous fields with server specific fields, which gives us the final data (most specific)
    server_data.update(found_server)
    
    return server_data
  
  
  def Connect(self):
    """Connect to the database"""
    server = self.GetServerData()
    
    Log('Connecting to MySQL server: %s: %s' % (server['host'], server['database']))
    
    # Read the password from the first line of the password file
    try:
      password = open(server['password_path']).read().split('\n', 1)[0]
      Log('Loaded password: %s' % server['password_path'])
      
    except Exception, e:
      Log('ERROR: Failed to read from password file: %s' % server['password_path'], logging.ERROR)
      password = None
    
    self.connection = mysql.connector.connect(user=server['user'], password=password, host=server['host'], port=server['port'], database=server['database'], use_unicode=True, charset=DEFAULT_CHARSET)
    self.cursor = self.connection.cursor(dictionary=True)


  def Query(self, sql, params=None, commit=True):
    """Query the database via our connection."""
    Log('Query: %s' % sql)
    
    result = Query(self.connection, self.cursor, sql, params=params, commit=commit)
    
    return result
  
  
  def Commit(self):
{'fraction_non_alphanumeric': 0.0945632798573975, 'fraction_numerical': 0.000980392156862745, 'mean_word_length': 3.2844597174494083, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2940795', 'n_tokens_mistral': 3108, 'n_tokens_neox': 3023, 'n_words': 1155}