<commit_before>import itertools
import logging
import phovea_server.config
from .sql_filter import filter_logic
from phovea_server.ns import abort
from .dbmanager import DBManager
from .utils import clean_query, secure_replacements
from werkzeug.datastructures import MultiDict

__author__ = '<NAME>'
_log = logging.getLogger(__name__)
c = phovea_server.config.view('tdp_core')
configs = DBManager()


def _supports_sql_parameters(dialect):
  return dialect.lower() != 'sqlite' and dialect.lower() != 'oracle'  # sqlite doesn't support array parameters, postgres does


def _differentiates_empty_string_and_null(dialect):
  return dialect.lower() != 'oracle'  # for Oracle, an empty string is the same as a null string


def resolve(database):
  """
  finds and return the connector and engine for the given database
  :param database: database key to lookup
  :return: (connector, engine)
  """
  if database not in configs:
    abort(404, u'Database with id "{}" cannot be found'.format(database))
  r = configs[database]
  # derive needed columns
  connector, engine = r
  for view in connector.views.values():
    if view.needs_to_fill_up_columns() and view.table is not None:
      _fill_up_columns(view, engine)
  return r


def resolve_engine(database):
  """
  finds and return the engine for the given database
  :param database: database key to lookup
  :return: engine
  """
  if database not in configs:
    abort(404, u'Database with id "{}" cannot be found'.format(database))
  return configs.engine(database)


def resolve_view(database, view_name, check_default_security=False):
  """
  finds and return the connector, engine, and view for the given database and view_name
  :param database: database key to lookup
  :param view_name: view name to lookup
  :param check_default_security: bool; usually view.can_access returns True when no security is defined on the view. This parameter can be used to tell the method that it should check the security anyway, e.g. that the user is at least logged in
  :return: (connector, engine, view)
  """
  connector, engine = resolve(database)
  if view_name not in connector.views:
    abort(404, u'view with id "{}" cannot be found in database "{}"'.format(view_name, database))
  view = connector.views[view_name]
  # TODO: improve the logic of the view.can_access function, because even for unauthorized can_access returns True, i.e. that the user can access the resource. Somewhere else the server checks whether the user is authenticated or not
  if not view.can_access(check_default_security):
    abort(403)
  return connector, engine, view


def assign_ids(rows, idtype):
  """
  assigns unique ids (stored in '_id') based on the 'id' column and given idtype
  :param rows: the rows having an 'id' column each
  :param idtype: the idtype to resolve the id
  :return: the extended rows
  """
  import phovea_server.plugin

  manager = phovea_server.plugin.lookup('idmanager')
  for _id, row in itertools.izip(manager((r['id'] for r in rows), idtype), rows):
    row['_id'] = _id
  return rows


def to_query(q, supports_array_parameter, parameters):
  """
  converts to the native SQL query using sqlalchemy + handling of array parameters
  :param q: the SQL query
  :param supports_array_parameter: whether array parameters are supported
  :param parameters: dictionary of parameters that are going to be applied
  :return: the transformed query and call by reference updated parameters
  """
  import sqlalchemy

  q = q.replace('\n', ' ').replace('\r', ' ')
  if supports_array_parameter:
    return sqlalchemy.sql.text(q)

  # need to suffix all array parameter and wrap with ()
  for k, v in parameters.items():
    if not isinstance(v, list) and not isinstance(v, tuple):
      continue
    # sounds like an array
    # convert from :ids to (:ids0, :ids1, :ids2)
    subparameters = {(k + str(i)): vi for i, vi in enumerate(v)}
    q = q.replace(':' + k, '({ids})'.format(ids=', '.join(':' + p for p in subparameters.keys())))
    del parameters[k]  # delete single
    parameters.update(subparameters)  # add sub

  return sqlalchemy.sql.text(q)


class WrappedSession(object):
  def __init__(self, engine):
    """
    session wrapper of sql alchemy with auto cleanup
    :param engine:
    """
    _log.info('creating session')
    self._session = configs.create_session(engine)
    self._supports_array_parameter = _supports_sql_parameters(engine.name)

  def execute(self, sql, **kwargs):
    """
    execute the given query with the given args
    :param sql: query
    :param kwargs: additional args to replace
    :return: the session result
    """
    parsed = to_query(sql, self._supports_array_parameter, kwargs)
    _log.info('%s (%s)', parsed, kwargs)
    return self._session.execute(parsed, kwargs)

  def run(self, sql, **kwargs):
    """
    runs the given sql statement, in contrast to execute the result will be converted to a list of dicts
    :param sql: the sql query to execute
    :param kwargs: args for this query
    :return: list of dicts
    """
    parsed = to_query(sql, self._supports_array_parameter, kwargs)
    _log.info('%s (%s)', parsed, kwargs)
    result = self._session.execute(parsed, kwargs)
    columns = result.keys()
    return [{c: r[c] for c in columns} for r in result]

  def __call__(self, sql, **kwargs):
    return self.run(sql, **kwargs)

  def __enter__(self):
    return self

  def commit(self):
    self._session.commit()

  def flush(self):
    self._session.flush()

  def rollback(self):
    self._session.rollback()<commit_msg>refactor `run()` and reuse code in `execute()`<commit_after>import itertools
import logging
import phovea_server.config
from .sql_filter import filter_logic
from phovea_server.ns import abort
from .dbmanager import DBManager
from .utils import clean_query, secure_replacements
from werkzeug.datastructures import MultiDict

__author__ = '<NAME>'
_log = logging.getLogger(__name__)
c = phovea_server.config.view('tdp_core')
configs = DBManager()


def _supports_sql_parameters(dialect):
  return dialect.lower() != 'sqlite' and dialect.lower() != 'oracle'  # sqlite doesn't support array parameters, postgres does


def _differentiates_empty_string_and_null(dialect):
  return dialect.lower() != 'oracle'  # for Oracle, an empty string is the same as a null string


def resolve(database):
  """
  finds and return the connector and engine for the given database
  :param database: database key to lookup
  :return: (connector, engine)
  """
  if database not in configs:
    abort(404, u'Database with id "{}" cannot be found'.format(database))
  r = configs[database]
  # derive needed columns
  connector, engine = r
  for view in connector.views.values():
    if view.needs_to_fill_up_columns() and view.table is not None:
      _fill_up_columns(view, engine)
  return r


def resolve_engine(database):
  """
  finds and return the engine for the given database
  :param database: database key to lookup
  :return: engine
  """
  if database not in configs:
    abort(404, u'Database with id "{}" cannot be found'.format(database))
  return configs.engine(database)


def resolve_view(database, view_name, check_default_security=False):
  """
  finds and return the connector, engine, and view for the given database and view_name
  :param database: database key to lookup
  :param view_name: view name to lookup
  :param check_default_security: bool; usually view.can_access returns True when no security is defined on the view. This parameter can be used to tell the method that it should check the security anyway, e.g. that the user is at least logged in
  :return: (connector, engine, view)
  """
  connector, engine = resolve(database)
  if view_name not in connector.views:
    abort(404, u'view with id "{}" cannot be found in database "{}"'.format(view_name, database))
  view = connector.views[view_name]
  # TODO: improve the logic of the view.can_access function, because even for unauthorized can_access returns True, i.e. that the user can access the resource. Somewhere else the server checks whether the user is authenticated or not
  if not view.can_access(check_default_security):
    abort(403)
  return connector, engine, view


def assign_ids(rows, idtype):
  """
  assigns unique ids (stored in '_id') based on the 'id' column and given idtype
  :param rows: the rows having an 'id' column each
  :param idtype: the idtype to resolve the id
  :return: the extended rows
  """
  import phovea_server.plugin

  manager = phovea_server.plugin.lookup('idmanager')
  for _id, row in itertools.izip(manager((r['id'] for r in rows), idtype), rows):
    row['_id'] = _id
  return rows


def to_query(q, supports_array_parameter, parameters):
  """
  converts to the native SQL query using sqlalchemy + handling of array parameters
  :param q: the SQL query
  :param supports_array_parameter: whether array parameters are supported
  :param parameters: dictionary of parameters that are going to be applied
  :return: the transformed query and call by reference updated parameters
  """
  import sqlalchemy

  q = q.replace('\n', ' ').replace('\r', ' ')
  if supports_array_parameter:
    return sqlalchemy.sql.text(q)

  # need to suffix all array parameter and wrap with ()
  for k, v in parameters.items():
    if not isinstance(v, list) and not isinstance(v, tuple):
      continue
    # sounds like an array
    # convert from :ids to (:ids0, :ids1, :ids2)
    subparameters = {(k + str(i)): vi for i, vi in enumerate(v)}
    q = q.replace(':' + k, '({ids})'.format(ids=', '.join(':' + p for p in subparameters.keys())))
    del parameters[k]  # delete single
    parameters.update(subparameters)  # add sub

  return sqlalchemy.sql.text(q)


class WrappedSession(object):
  def __init__(self, engine):
    """
    session wrapper of sql alchemy with auto cleanup
    :param engine:
    """
    _log.info('creating session')
    self._session = configs.create_session(engine)
    self._supports_array_parameter = _supports_sql_parameters(engine.name)

  def execute(self, sql, **kwargs):
    """
    execute the given query with the given args
    :param sql: query
    :param kwargs: additional args to replace
    :return: the session result
    """
    parsed = to_query(sql, self._supports_array_parameter, kwargs)
    _log.info('%s (%s)', parsed, kwargs)
    return self._session.execute(parsed, kwargs)

  def run(self, sql, **kwargs):
    """
    runs the given sql statement, in contrast to execute the result will be converted to a list of dicts
    :param sql: the sql query to execute
    :param kwargs: args for this query
    :return: list of dicts
    """
    result = self.execute(sql, **kwargs)
    columns = result.keys()
    return [{c: r[c] for c in columns} for r in result]

  def __call__(self, sql, **kwargs):
    return self.run(sql, **kwargs)

  def __enter__(self):
    return self

  def commit(self):
    self._session.commit()

  def flush(self):
    self._session.flush()

  def rollback(self):
    self._session.rollback()
{'fraction_non_alphanumeric': 0.0991570742318499, 'fraction_numerical': 0.0027191153811293393, 'mean_word_length': 4.06379072969252, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2795683', 'n_tokens_mistral': 3331, 'n_tokens_neox': 3252, 'n_words': 1229}