<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    This module copyright (C) 2011-2013 Therp BV (<http://therp.nl>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import os
import inspect
import logging
from openerp import release, tools, SUPERUSER_ID
from openerp.tools.yaml_import import yaml_import
from openerp.osv import orm
from openerp.tools.mail import plaintext2html
from openerp.modules.registry import RegistryManager
import openupgrade_tools
import openerp.osv.fields
import openerp.fields

# The server log level has not been set at this point
# so to log at loglevel debug we need to set it
# manually here. As a consequence, DEBUG messages from
# this file are always logged
logger = logging.getLogger('OpenUpgrade')
logger.setLevel(logging.DEBUG)

__all__ = [
    'migrate',
    'load_data',
    'copy_columns',
    'rename_columns',
    'rename_tables',
    'rename_models',
    'rename_xmlids',
    'add_xmlid',
    'drop_columns',
    'delete_model_workflow',
    'update_workflow_workitems',
    'warn_possible_dataloss',
    'set_defaults',
    'logged_query',
    'column_exists',
    'table_exists',
    'update_module_names',
    'add_ir_model_fields',
    'get_legacy_name',
    'm2o_to_x2m',
    'float_to_integer',
    'message',
    'check_values_selection_field',
    'move_field_m2o',
    'convert_field_to_html',
    'map_values',
    'deactivate_workflow_transitions',
    'reactivate_workflow_transitions',
    'date_to_datetime_tz',
]


def check_values_selection_field(cr, table_name, field_name, allowed_values):
    """
        check if the field selection 'field_name' of the table 'table_name'
        has only the values 'allowed_values'.
        If not return False and log an error.
        If yes, return True.

    .. versionadded:: 8.0
    """
    res = True
    cr.execute("SELECT %s, count(*) FROM %s GROUP BY %s;" %
               (field_name, table_name, field_name))
    for row in cr.fetchall():
        if row[0] not in allowed_values:
            logger.error(
                "Invalid value '%s' in the table '%s' "
                "for the field '%s'. (%s rows).",
                row[0], table_name, field_name, row[1])
            res = False
    return res


def load_data(cr, module_name, filename, idref=None, mode='init'):
    """
    Load an xml, csv or yml data file from your post script. The usual case for
    this is the
    occurrence of newly added essential or useful data in the module that is
    marked with "noupdate='1'" and without "forcecreate='1'" so that it will
    not be loaded by the usual upgrade mechanism. Leaving the 'mode' argument
    to its default 'init' will load the data from your migration script.

    Theoretically, you could simply load a stock file from the module, but be
    careful not to reinitialize any data that could have been customized.
    Preferably, select only the newly added items. Copy these to a file
    in your migrations directory and load that file.
    Leave it to the user to actually delete existing resources that are
    marked with 'noupdate' (other named items will be deleted
    automatically).


    :param module_name: the name of the module
    :param filename: the path to the filename, relative to the module \
    directory.
    :param idref: optional hash with ?id mapping cache?
    :param mode: one of 'init', 'update', 'demo'. Always use 'init' \
    for adding new items from files that are marked with 'noupdate'. Defaults \
    to 'init'.
    """

    if idref is None:
        idref = {}
    logger.info('%s: loading %s' % (module_name, filename))
    _, ext = os.path.splitext(filename)
    pathname = os.path.join(module_name, filename)
    fp = tools.file_open(pathname)
    try:
        if ext == '.csv':
            noupdate = True
            tools.convert_csv_import(
                cr, module_name, pathname, fp.read(), idref, mode, noupdate)
        elif ext == '.yml':
            yaml_import(cr, module_name, fp, None, idref=idref, mode=mode)
        else:
            tools.convert_xml_import(cr, module_name, fp, idref, mode=mode)
    finally:
        fp.close()

# for backwards compatibility
load_xml = load_data
table_exists = openupgrade_tools.table_exists


def copy_columns(cr, column_spec):
    """
    Copy table columns. Typically called in the pre script.

    :param column_spec: a hash with table keys, with lists of tuples as \
    values. Tuples consist of (old_name, new_name, type). Use None for \
    new_name to trigger a conversion of old_name using get_legacy_name() \
    Use None for type to use type of old field

    .. versionadded:: 8.0
    """
    for table_name in column_spec.keys():
        for (old, new, field_type) in column_spec[table_name]:
            if new is None:
                new = get_legacy_name(old)
            if field_type is None:
                cr.execute("""
                    SELECT data_type
                    FROM information_schema.columns
                    WHERE table_name=%s
                        AND column_name = %s;
                    """, (table_name, old))
                field_type = cr.fetchone()[0]
            logged_query(cr, """
                ALTER TABLE %(table_name)s
                ADD COLUMN %(new)s %(field_type)s;
                UPDATE %(table_name)s SET %(new)s=%(old)s;
                """ % {
                'table_name': table_name,
                'old': old,
                'field_type': field_type,
                'new': new,
            })


def rename_columns(cr, column_spec):
    """
    Rename table columns. Typically called in the pre script.

    :param column_spec: a hash with table keys, with lists of tuples as \
    values. Tuples consist of (old_name, new_name). Use None for new_name \
    to trigger a conversion of old_name using get_legacy_name()
    """
    for table in column_spec.keys():
        for (old, new) in column_spec[table]:
            if new is None:
                new = get_legacy_name(old)
            logger.info("table %s, column %s: renaming to %s",
                        table, old, new)
            cr.execute(
                'ALTER TABLE "%s" RENAME "%s" TO "%s"' % (table, old, new,))
            cr.execute('DROP INDEX IF EXISTS "%s_%s_index"' % (table, old))


def rename_tables(cr, table_spec):
    """
    Rename tables. Typically called in the pre script.
    This function also renames the id sequence if it exists and if it is
    not modified in the same run.

    :param table_spec: a list of tuples (old table name, new table name). Use \
    None for new_name to trigger a conversion of old_name to the result of \
    get_legacy_name()
    """
    # Append id sequences
    to_rename = [x[0] for x in table_spec]
    for old, new in list(table_spec):
        if new is None:
            new = get_legacy_name(old)
        if (table_exists(cr, old + '_id_seq') and
                old + '_id_seq' not in to_rename):
            table_spec.append((old + '_id_seq', new + '_id_seq'))
    for (old, new) in table_spec:
        if new is None:
            new = get_legacy_name(old)
        logger.info("table %s: renaming to %s",
                    old, new)
        cr.execute('ALTER TABLE "%s" RENAME TO "%s"' % (old, new,))


def rename_models(cr, model_spec):
    """
    Rename models. Typically called in the pre script.
    :param model_spec: a list of tuples (old model name, new model name).

    Use case: if a model changes name, but still implements equivalent
    functionality you will want to update references in for instance
    relation fields.

    """
    for (old, new) in model_spec:
        logger.info("model %s: renaming to %s",
                    old, new)
        cr.execute('UPDATE ir_model SET model = %s '
                   'WHERE model = %s', (new, old,))
        cr.execute('UPDATE ir_model_fields SET relation = %s '
                   'WHERE relation = %s', (new, old,))
        cr.execute('UPDATE ir_model_data SET model = %s '
                   'WHERE model = %s', (new, old,))
    # TODO: signal where the model occurs in references to ir_model


def rename_xmlids(cr, xmlids_spec):
    """
    Rename XML IDs. Typically called in the pre script.
    One usage example is when an ID changes module. In OpenERP 6 for example,
    a number of res_groups IDs moved to module base from other modules (
    although they were still being defined in their respective module).

    :param xmlids_spec: a list of tuples (old module.xmlid, new module.xmlid).
    """
    for (old, new) in xmlids_spec:
        if '.' not in old or '.' not in new:
            logger.error(
                'Cannot rename XMLID %s to %s: need the module '
                'reference to be specified in the IDs' % (old, new))
        else:
            query = ("UPDATE ir_model_data SET module = %s, name = %s "
                     "WHERE module = %s and name = %s")
            logged_query(cr, query, tuple(new.split('.') + old.split('.')))


def add_xmlid(cr, module, xmlid, model, res_id, noupdate=False):
    """
    Adds an entry in ir_model_data. Typically called in the pre script.
    One usage example is when an entry has been add in the XML and there is
    a high probability that the user has already created the entry manually.
    For example, a currency was added in the XML data of the base module
    in OpenERP 6 but the user had already created this missing currency
    by hand in it's 5.0 database. In order to avoid having 2 identical
    currencies (which is in fact blocked by an sql_constraint), you have to
    add the entry in ir_model_data before the upgrade.
    """
    # Check if the XMLID doesn't already exists
    cr.execute(
        "SELECT id FROM ir_model_data WHERE module=%s AND name=%s "
        "AND model=%s",
        (module, xmlid, model))
    already_exists = cr.fetchone()
    if already_exists:
        return False
    else:
        logged_query(
            cr,
            "INSERT INTO ir_model_data (create_uid, create_date, "
            "write_uid, write_date, date_init, date_update, noupdate, "
            "name, module, model, res_id) "
            "VALUES (%s, (now() at time zone 'UTC'), %s, "
            "(now() at time zone 'UTC'), (now() at time zone 'UTC'), "
            "(now() at time zone 'UTC'), %s, %s, %s, %s, %s)", (
                SUPERUSER_ID, SUPERUSER_ID, noupdate,
                xmlid, module, model, res_id))
        return True


def drop_columns(cr, column_spec):
    """
    Drop columns but perform an additional check if a column exists.
    This covers the case of function fields that may or may not be stored.
    Consider that this may not be obvious: an additional module can govern
    a function fields' store properties.

    :param column_spec: a list of (table, column) tuples
    """
    for (table, column) in column_spec:
        logger.info("table %s: drop column %s",
                    table, column)
        if column_exists(cr, table, column):
            cr.execute('ALTER TABLE "%s" DROP COLUMN "%s"' %
                       (table, column))
        else:
            logger.warn("table %s: column %s did not exist",
                        table, column)


def update_workflow_workitems(cr, pool, ref_spec_actions):
    """Find all the workflow items from the target state to set them to
    the wanted state.

    When a workflow action is removed, from model, the objects whose states
    are in these actions need to be set to another to be able to continue the
    workflow properly.

    Run in pre-migration

    :param ref_spec_actions: list of tuples with couple of workflow.action's
        external ids. The first id is replaced with the second.
    :return: None

    .. versionadded:: 7.0
    """
    workflow_workitems = pool['workflow.workitem']
    ir_model_data_model = pool['ir.model.data']

    for (target_external_id, fallback_external_id) in ref_spec_actions:
        target_activity = ir_model_data_model.get_object(
            cr, SUPERUSER_ID,
            target_external_id.split(".")[0],
            target_external_id.split(".")[1],
        )
        fallback_activity = ir_model_data_model.get_object(
            cr, SUPERUSER_ID,
            fallback_external_id.split(".")[0],
            fallback_external_id.split(".")[1],
        )
        ids = workflow_workitems.search(
            cr, SUPERUSER_ID, [('act_id', '=', target_activity.id)]
        )
        if ids:
            logger.info(
                "Moving %d items in the removed workflow action (%s) to a "
                "fallback action (%s): %s",
                len(ids), target_activity.name, fallback_activity.name, ids
            )
            workflow_workitems.write(
                cr, SUPERUSER_ID, ids, {'act_id': fallback_activity.id}
            )


def delete_model_workflow(cr, model):
    """
    Forcefully remove active workflows for obsolete models,
    to prevent foreign key issues when the orm deletes the model.
    """
    logged_query(
        cr,
        "DELETE FROM wkf_workitem WHERE act_id in "
        "( SELECT wkf_activity.id "
        "  FROM wkf_activity, wkf "
        "  WHERE wkf_id = wkf.id AND "
        "  wkf.osv = %s"
        ")", (model,))
    logged_query(
        cr,
        "DELETE FROM wkf WHERE osv = %s", (model,))


def warn_possible_dataloss(cr, pool, old_module, fields):
    """
    Use that function in the following case:
    if a field of a model was moved from a 'A' module to a 'B' module.
    ('B' depend on 'A'),
    This function will test if 'B' is installed.
    If not, count the number of different value and possibly warn the user.
    Use orm, so call from the post script.

    :param old_module: name of the old module
    :param fields: list of dictionary with the following keys:
        'table' : name of the table where the field is.
        'field' : name of the field that are moving.
        'new_module' : name of the new module

    .. versionadded:: 7.0
    """
    module_obj = pool.get('ir.module.module')
    for field in fields:
        module_ids = module_obj.search(
            cr, SUPERUSER_ID, [
                ('name', '=', field['new_module']),
                ('state', 'in', ['installed', 'to upgrade', 'to install'])
            ])
        if not module_ids:
            cr.execute(
                "SELECT count(*) FROM (SELECT %s from %s group by %s) "
                "as tmp" % (
                    field['field'], field['table'], field['field']))
            row = cr.fetchone()
            if row[0] == 1:
                # not a problem, that field wasn't used.
                # Just a loss of functionality
                logger.info(
                    "Field '%s' from module '%s' was moved to module "
                    "'%s' which is not installed: "
                    "No dataloss detected, only loss of functionality"
                    % (field['field'], old_module, field['new_module']))
            else:
                # there is data loss after the migration.
                message(
                    cr, old_module, None, None,
                    "Field '%s' was moved to module "
                    "'%s' which is not installed: "
                    "There were %s distinct values in this field.",
                    field['field'], field['new_module'], row[0])


def set_defaults(cr, pool, default_spec, force=False):
    """
    Set default value. Useful for fields that are newly required. Uses orm, so
    call from the post script.

    :param default_spec: a hash with model names as keys. Values are lists \
    of tuples (field, value). None as a value has a special meaning: it \
    assigns the default value. If this value is provided by a function, the \
    function is called as the user that created the resource.
    :param force: overwrite existing values. To be used for assigning a non- \
    default value (presumably in the case of a new column). The ORM assigns \
    the default value as declared in the model in an earlier stage of the \
    process. Beware of issues with resources loaded from new data that \
    actually do require the model's default, in combination with the post \
    script possible being run multiple times.
    """

    def write_value(ids, field, value):
        logger.debug(
            "model %s, field %s: setting default value of resources %s to %s",
            model, field, ids, unicode(value))
        for res_id in ids:
            # Iterating over ids here as a workaround for lp:1131653
            obj.write(cr, SUPERUSER_ID, [res_id], {field: value})

    for model in default_spec.keys():
        obj = pool.get(model)
        if not obj:
            raise orm.except_orm(
                "Error",
                "Migration: error setting default, no such model: %s" % model)

        for field, value in default_spec[model]:
            domain = not force and [(field, '=', False)] or []
            ids = obj.search(cr, SUPERUSER_ID, domain)
            if not ids:
                continue
            if value is None:
                # Set the value by calling the _defaults of the object.
                # Typically used for company_id on various models, and in that
                # case the result depends on the user associated with the
                # object. We retrieve create_uid for this purpose and need to
                # call the defaults function per resource. Otherwise, write
                # all resources at once.
                if field in obj._defaults:
                    if not callable(obj._defaults[field]):
                        write_value(ids, field, obj._defaults[field])
                    else:
                        cr.execute(
                            "SELECT id, COALESCE(create_uid, 1) FROM %s " %
                            obj._table + "WHERE id in %s", (tuple(ids),))
                        # Execute the function once per user_id
                        user_id_map = {}
                        for row in cr.fetchall():
                            user_id_map.setdefault(row[1], []).append(row[0])
                        for user_id in user_id_map:
                            write_value(
                                user_id_map[user_id], field,
                                obj._defaults[field](obj, cr, user_id, None))
                else:
                    error = (
                        "OpenUpgrade: error setting default, field %s with "
                        "None default value not in %s' _defaults" % (
                            field, model))
                    logger.error(error)
                    # this exeption seems to get lost in a higher up try block
                    orm.except_orm("OpenUpgrade", error)
            else:
                write_value(ids, field, value)


def logged_query(cr, query, args=None):
    """
    Logs query and affected rows at level DEBUG.

    :param query: a query string suitable to pass to cursor.execute()
    :param args: a list, tuple or dictionary passed as substitution values \
to cursor.execute().
    """
    if args is None:
        args = ()
    args = tuple(args) if type(args) == list else args
    cr.execute(query, args)
    logger.debug('Running %s', query % args)
    logger.debug('%s rows affected', cr.rowcount)
    return cr.rowcount


def column_exists(cr, table, column):
    """ Check whether a certain column exists """
    cr.execute(
        'SELECT count(attname) FROM pg_attribute '
        'WHERE attrelid = '
        '( SELECT oid FROM pg_class WHERE relname = %s ) '
        'AND attname = %s',
        (table, column))
    return cr.fetchone()[0] == 1


def update_module_names(cr, namespec):
    """
    Deal with changed module names of certified modules
    in order to prevent  'certificate not unique' error,
    as well as updating the module reference in the
    XML id.

    :param namespec: tuple of (old name, new name)
    """
    for (old_name, new_name) in namespec:
        query = ("UPDATE ir_module_module SET name = %s "
                 "WHERE name = %s")
        logged_query(cr, query, (new_name, old_name))
        query = ("UPDATE ir_model_data SET module = %s "
                 "WHERE module = %s ")
        logged_query(cr, query, (new_name, old_name))
        query = ("UPDATE ir_module_module_dependency SET name = %s "
                 "WHERE name = %s")
        logged_query(cr, query, (new_name, old_name))


def add_ir_model_fields(cr, columnspec):
    """
    Typically, new columns on ir_model_fields need to be added in a very
    early stage in the upgrade process of the base module, in raw sql
    as they need to be in place before any model gets initialized.
    Do not use for fields with additional SQL constraints, such as a
    reference to another table or the cascade constraint, but craft your
    own statement taking them into account.

    :param columnspec: tuple of (column name, column type)
    """
    for column in columnspec:
        query = 'ALTER TABLE ir_model_fields ADD COLUMN %s %s' % (
            column)
        logged_query(cr, query, [])


def get_legacy_name(original_name):
    """
    Returns a versioned name for legacy tables/columns/etc
    Use this function instead of some custom name to avoid
    collisions with future or past legacy tables/columns/etc

    :param original_name: the original name of the column
    :param version: current version as passed to migrate()
    """
    return 'openupgrade_legacy_'+('_').join(
        map(str, release.version_info[0:2]))+'_'+original_name


def m2o_to_x2m(cr, model, table, field, source_field):
    """
    Transform many2one relations into one2many or many2many.
    Use rename_columns in your pre-migrate script to retain the column's old
    value, then call m2o_to_x2m in your post-migrate script.

    WARNING: If converting to one2many, there can be data loss, because only
    one inverse record can be mapped in a one2many, but you can have multiple
    many2one pointing to the same target. Use it when the use case allows this
    conversion.

    :param model: The target model registry object
    :param table: The source table
    :param field: The new field name on the target model
    :param source_field: the (renamed) many2one column on the source table.

    .. versionadded:: 8.0
    """
    if not model._columns.get(field):
        raise orm.except_orm(
            "Error", "m2o_to_x2m: field %s doesn't exist in model %s" % (
                field, model._name))
    if isinstance(model._columns[field], (openerp.osv.fields.many2many,
                                          openerp.fields.Many2many)):
        rel, id1, id2 = openerp.osv.fields.many2many._sql_names(
            model._columns[field], model)
        logged_query(
            cr,
            """
            INSERT INTO %s (%s, %s)
            SELECT id, %s
            FROM %s
            WHERE %s is not null
            """ %
            (rel, id1, id2, source_field, table, source_field))
    elif isinstance(model._columns[field], (openerp.fields.one2many,
                                            openerp.osv.fields.one2many)):
        if isinstance(model._columns[field], openerp.fields.one2many):
            target_table = (
                model.pool[model._columns[field].comodel_name]._table)
            target_field = model._columns[field].inverse_name
        else:
            target_table = model.pool[model._columns[field]._obj]._table
            target_field = model._columns[field]._fields_id
        logged_query(
            cr,
            """<commit_msg>[FIX] use uppercase where necessary<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    This module copyright (C) 2011-2013 Therp BV (<http://therp.nl>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import os
import inspect
import logging
from openerp import release, tools, SUPERUSER_ID
from openerp.tools.yaml_import import yaml_import
from openerp.osv import orm
from openerp.tools.mail import plaintext2html
from openerp.modules.registry import RegistryManager
import openupgrade_tools
import openerp.osv.fields
import openerp.fields

# The server log level has not been set at this point
# so to log at loglevel debug we need to set it
# manually here. As a consequence, DEBUG messages from
# this file are always logged
logger = logging.getLogger('OpenUpgrade')
logger.setLevel(logging.DEBUG)

__all__ = [
    'migrate',
    'load_data',
    'copy_columns',
    'rename_columns',
    'rename_tables',
    'rename_models',
    'rename_xmlids',
    'add_xmlid',
    'drop_columns',
    'delete_model_workflow',
    'update_workflow_workitems',
    'warn_possible_dataloss',
    'set_defaults',
    'logged_query',
    'column_exists',
    'table_exists',
    'update_module_names',
    'add_ir_model_fields',
    'get_legacy_name',
    'm2o_to_x2m',
    'float_to_integer',
    'message',
    'check_values_selection_field',
    'move_field_m2o',
    'convert_field_to_html',
    'map_values',
    'deactivate_workflow_transitions',
    'reactivate_workflow_transitions',
    'date_to_datetime_tz',
]


def check_values_selection_field(cr, table_name, field_name, allowed_values):
    """
        check if the field selection 'field_name' of the table 'table_name'
        has only the values 'allowed_values'.
        If not return False and log an error.
        If yes, return True.

    .. versionadded:: 8.0
    """
    res = True
    cr.execute("SELECT %s, count(*) FROM %s GROUP BY %s;" %
               (field_name, table_name, field_name))
    for row in cr.fetchall():
        if row[0] not in allowed_values:
            logger.error(
                "Invalid value '%s' in the table '%s' "
                "for the field '%s'. (%s rows).",
                row[0], table_name, field_name, row[1])
            res = False
    return res


def load_data(cr, module_name, filename, idref=None, mode='init'):
    """
    Load an xml, csv or yml data file from your post script. The usual case for
    this is the
    occurrence of newly added essential or useful data in the module that is
    marked with "noupdate='1'" and without "forcecreate='1'" so that it will
    not be loaded by the usual upgrade mechanism. Leaving the 'mode' argument
    to its default 'init' will load the data from your migration script.

    Theoretically, you could simply load a stock file from the module, but be
    careful not to reinitialize any data that could have been customized.
    Preferably, select only the newly added items. Copy these to a file
    in your migrations directory and load that file.
    Leave it to the user to actually delete existing resources that are
    marked with 'noupdate' (other named items will be deleted
    automatically).


    :param module_name: the name of the module
    :param filename: the path to the filename, relative to the module \
    directory.
    :param idref: optional hash with ?id mapping cache?
    :param mode: one of 'init', 'update', 'demo'. Always use 'init' \
    for adding new items from files that are marked with 'noupdate'. Defaults \
    to 'init'.
    """

    if idref is None:
        idref = {}
    logger.info('%s: loading %s' % (module_name, filename))
    _, ext = os.path.splitext(filename)
    pathname = os.path.join(module_name, filename)
    fp = tools.file_open(pathname)
    try:
        if ext == '.csv':
            noupdate = True
            tools.convert_csv_import(
                cr, module_name, pathname, fp.read(), idref, mode, noupdate)
        elif ext == '.yml':
            yaml_import(cr, module_name, fp, None, idref=idref, mode=mode)
        else:
            tools.convert_xml_import(cr, module_name, fp, idref, mode=mode)
    finally:
        fp.close()

# for backwards compatibility
load_xml = load_data
table_exists = openupgrade_tools.table_exists


def copy_columns(cr, column_spec):
    """
    Copy table columns. Typically called in the pre script.

    :param column_spec: a hash with table keys, with lists of tuples as \
    values. Tuples consist of (old_name, new_name, type). Use None for \
    new_name to trigger a conversion of old_name using get_legacy_name() \
    Use None for type to use type of old field

    .. versionadded:: 8.0
    """
    for table_name in column_spec.keys():
        for (old, new, field_type) in column_spec[table_name]:
            if new is None:
                new = get_legacy_name(old)
            if field_type is None:
                cr.execute("""
                    SELECT data_type
                    FROM information_schema.columns
                    WHERE table_name=%s
                        AND column_name = %s;
                    """, (table_name, old))
                field_type = cr.fetchone()[0]
            logged_query(cr, """
                ALTER TABLE %(table_name)s
                ADD COLUMN %(new)s %(field_type)s;
                UPDATE %(table_name)s SET %(new)s=%(old)s;
                """ % {
                'table_name': table_name,
                'old': old,
                'field_type': field_type,
                'new': new,
            })


def rename_columns(cr, column_spec):
    """
    Rename table columns. Typically called in the pre script.

    :param column_spec: a hash with table keys, with lists of tuples as \
    values. Tuples consist of (old_name, new_name). Use None for new_name \
    to trigger a conversion of old_name using get_legacy_name()
    """
    for table in column_spec.keys():
        for (old, new) in column_spec[table]:
            if new is None:
                new = get_legacy_name(old)
            logger.info("table %s, column %s: renaming to %s",
                        table, old, new)
            cr.execute(
                'ALTER TABLE "%s" RENAME "%s" TO "%s"' % (table, old, new,))
            cr.execute('DROP INDEX IF EXISTS "%s_%s_index"' % (table, old))


def rename_tables(cr, table_spec):
    """
    Rename tables. Typically called in the pre script.
    This function also renames the id sequence if it exists and if it is
    not modified in the same run.

    :param table_spec: a list of tuples (old table name, new table name). Use \
    None for new_name to trigger a conversion of old_name to the result of \
    get_legacy_name()
    """
    # Append id sequences
    to_rename = [x[0] for x in table_spec]
    for old, new in list(table_spec):
        if new is None:
            new = get_legacy_name(old)
        if (table_exists(cr, old + '_id_seq') and
                old + '_id_seq' not in to_rename):
            table_spec.append((old + '_id_seq', new + '_id_seq'))
    for (old, new) in table_spec:
        if new is None:
            new = get_legacy_name(old)
        logger.info("table %s: renaming to %s",
                    old, new)
        cr.execute('ALTER TABLE "%s" RENAME TO "%s"' % (old, new,))


def rename_models(cr, model_spec):
    """
    Rename models. Typically called in the pre script.
    :param model_spec: a list of tuples (old model name, new model name).

    Use case: if a model changes name, but still implements equivalent
    functionality you will want to update references in for instance
    relation fields.

    """
    for (old, new) in model_spec:
        logger.info("model %s: renaming to %s",
                    old, new)
        cr.execute('UPDATE ir_model SET model = %s '
                   'WHERE model = %s', (new, old,))
        cr.execute('UPDATE ir_model_fields SET relation = %s '
                   'WHERE relation = %s', (new, old,))
        cr.execute('UPDATE ir_model_data SET model = %s '
                   'WHERE model = %s', (new, old,))
    # TODO: signal where the model occurs in references to ir_model


def rename_xmlids(cr, xmlids_spec):
    """
    Rename XML IDs. Typically called in the pre script.
    One usage example is when an ID changes module. In OpenERP 6 for example,
    a number of res_groups IDs moved to module base from other modules (
    although they were still being defined in their respective module).

    :param xmlids_spec: a list of tuples (old module.xmlid, new module.xmlid).
    """
    for (old, new) in xmlids_spec:
        if '.' not in old or '.' not in new:
            logger.error(
                'Cannot rename XMLID %s to %s: need the module '
                'reference to be specified in the IDs' % (old, new))
        else:
            query = ("UPDATE ir_model_data SET module = %s, name = %s "
                     "WHERE module = %s and name = %s")
            logged_query(cr, query, tuple(new.split('.') + old.split('.')))


def add_xmlid(cr, module, xmlid, model, res_id, noupdate=False):
    """
    Adds an entry in ir_model_data. Typically called in the pre script.
    One usage example is when an entry has been add in the XML and there is
    a high probability that the user has already created the entry manually.
    For example, a currency was added in the XML data of the base module
    in OpenERP 6 but the user had already created this missing currency
    by hand in it's 5.0 database. In order to avoid having 2 identical
    currencies (which is in fact blocked by an sql_constraint), you have to
    add the entry in ir_model_data before the upgrade.
    """
    # Check if the XMLID doesn't already exists
    cr.execute(
        "SELECT id FROM ir_model_data WHERE module=%s AND name=%s "
        "AND model=%s",
        (module, xmlid, model))
    already_exists = cr.fetchone()
    if already_exists:
        return False
    else:
        logged_query(
            cr,
            "INSERT INTO ir_model_data (create_uid, create_date, "
            "write_uid, write_date, date_init, date_update, noupdate, "
            "name, module, model, res_id) "
            "VALUES (%s, (now() at time zone 'UTC'), %s, "
            "(now() at time zone 'UTC'), (now() at time zone 'UTC'), "
            "(now() at time zone 'UTC'), %s, %s, %s, %s, %s)", (
                SUPERUSER_ID, SUPERUSER_ID, noupdate,
                xmlid, module, model, res_id))
        return True


def drop_columns(cr, column_spec):
    """
    Drop columns but perform an additional check if a column exists.
    This covers the case of function fields that may or may not be stored.
    Consider that this may not be obvious: an additional module can govern
    a function fields' store properties.

    :param column_spec: a list of (table, column) tuples
    """
    for (table, column) in column_spec:
        logger.info("table %s: drop column %s",
                    table, column)
        if column_exists(cr, table, column):
            cr.execute('ALTER TABLE "%s" DROP COLUMN "%s"' %
                       (table, column))
        else:
            logger.warn("table %s: column %s did not exist",
                        table, column)


def update_workflow_workitems(cr, pool, ref_spec_actions):
    """Find all the workflow items from the target state to set them to
    the wanted state.

    When a workflow action is removed, from model, the objects whose states
    are in these actions need to be set to another to be able to continue the
    workflow properly.

    Run in pre-migration

    :param ref_spec_actions: list of tuples with couple of workflow.action's
        external ids. The first id is replaced with the second.
    :return: None

    .. versionadded:: 7.0
    """
    workflow_workitems = pool['workflow.workitem']
    ir_model_data_model = pool['ir.model.data']

    for (target_external_id, fallback_external_id) in ref_spec_actions:
        target_activity = ir_model_data_model.get_object(
            cr, SUPERUSER_ID,
            target_external_id.split(".")[0],
            target_external_id.split(".")[1],
        )
        fallback_activity = ir_model_data_model.get_object(
            cr, SUPERUSER_ID,
            fallback_external_id.split(".")[0],
            fallback_external_id.split(".")[1],
        )
        ids = workflow_workitems.search(
            cr, SUPERUSER_ID, [('act_id', '=', target_activity.id)]
        )
        if ids:
            logger.info(
                "Moving %d items in the removed workflow action (%s) to a "
                "fallback action (%s): %s",
                len(ids), target_activity.name, fallback_activity.name, ids
            )
            workflow_workitems.write(
                cr, SUPERUSER_ID, ids, {'act_id': fallback_activity.id}
            )


def delete_model_workflow(cr, model):
    """
    Forcefully remove active workflows for obsolete models,
    to prevent foreign key issues when the orm deletes the model.
    """
    logged_query(
        cr,
        "DELETE FROM wkf_workitem WHERE act_id in "
        "( SELECT wkf_activity.id "
        "  FROM wkf_activity, wkf "
        "  WHERE wkf_id = wkf.id AND "
        "  wkf.osv = %s"
        ")", (model,))
    logged_query(
        cr,
        "DELETE FROM wkf WHERE osv = %s", (model,))


def warn_possible_dataloss(cr, pool, old_module, fields):
    """
    Use that function in the following case:
    if a field of a model was moved from a 'A' module to a 'B' module.
    ('B' depend on 'A'),
    This function will test if 'B' is installed.
    If not, count the number of different value and possibly warn the user.
    Use orm, so call from the post script.

    :param old_module: name of the old module
    :param fields: list of dictionary with the following keys:
        'table' : name of the table where the field is.
        'field' : name of the field that are moving.
        'new_module' : name of the new module

    .. versionadded:: 7.0
    """
    module_obj = pool.get('ir.module.module')
    for field in fields:
        module_ids = module_obj.search(
            cr, SUPERUSER_ID, [
                ('name', '=', field['new_module']),
                ('state', 'in', ['installed', 'to upgrade', 'to install'])
            ])
        if not module_ids:
            cr.execute(
                "SELECT count(*) FROM (SELECT %s from %s group by %s) "
                "as tmp" % (
                    field['field'], field['table'], field['field']))
            row = cr.fetchone()
            if row[0] == 1:
                # not a problem, that field wasn't used.
                # Just a loss of functionality
                logger.info(
                    "Field '%s' from module '%s' was moved to module "
                    "'%s' which is not installed: "
                    "No dataloss detected, only loss of functionality"
                    % (field['field'], old_module, field['new_module']))
            else:
                # there is data loss after the migration.
                message(
                    cr, old_module, None, None,
                    "Field '%s' was moved to module "
                    "'%s' which is not installed: "
                    "There were %s distinct values in this field.",
                    field['field'], field['new_module'], row[0])


def set_defaults(cr, pool, default_spec, force=False):
    """
    Set default value. Useful for fields that are newly required. Uses orm, so
    call from the post script.

    :param default_spec: a hash with model names as keys. Values are lists \
    of tuples (field, value). None as a value has a special meaning: it \
    assigns the default value. If this value is provided by a function, the \
    function is called as the user that created the resource.
    :param force: overwrite existing values. To be used for assigning a non- \
    default value (presumably in the case of a new column). The ORM assigns \
    the default value as declared in the model in an earlier stage of the \
    process. Beware of issues with resources loaded from new data that \
    actually do require the model's default, in combination with the post \
    script possible being run multiple times.
    """

    def write_value(ids, field, value):
        logger.debug(
            "model %s, field %s: setting default value of resources %s to %s",
            model, field, ids, unicode(value))
        for res_id in ids:
            # Iterating over ids here as a workaround for lp:1131653
            obj.write(cr, SUPERUSER_ID, [res_id], {field: value})

    for model in default_spec.keys():
        obj = pool.get(model)
        if not obj:
            raise orm.except_orm(
                "Error",
                "Migration: error setting default, no such model: %s" % model)

        for field, value in default_spec[model]:
            domain = not force and [(field, '=', False)] or []
            ids = obj.search(cr, SUPERUSER_ID, domain)
            if not ids:
                continue
            if value is None:
                # Set the value by calling the _defaults of the object.
                # Typically used for company_id on various models, and in that
                # case the result depends on the user associated with the
                # object. We retrieve create_uid for this purpose and need to
                # call the defaults function per resource. Otherwise, write
                # all resources at once.
                if field in obj._defaults:
                    if not callable(obj._defaults[field]):
                        write_value(ids, field, obj._defaults[field])
                    else:
                        cr.execute(
                            "SELECT id, COALESCE(create_uid, 1) FROM %s " %
                            obj._table + "WHERE id in %s", (tuple(ids),))
                        # Execute the function once per user_id
                        user_id_map = {}
                        for row in cr.fetchall():
                            user_id_map.setdefault(row[1], []).append(row[0])
                        for user_id in user_id_map:
                            write_value(
                                user_id_map[user_id], field,
                                obj._defaults[field](obj, cr, user_id, None))
                else:
                    error = (
                        "OpenUpgrade: error setting default, field %s with "
                        "None default value not in %s' _defaults" % (
                            field, model))
                    logger.error(error)
                    # this exeption seems to get lost in a higher up try block
                    orm.except_orm("OpenUpgrade", error)
            else:
                write_value(ids, field, value)


def logged_query(cr, query, args=None):
    """
    Logs query and affected rows at level DEBUG.

    :param query: a query string suitable to pass to cursor.execute()
    :param args: a list, tuple or dictionary passed as substitution values \
to cursor.execute().
    """
    if args is None:
        args = ()
    args = tuple(args) if type(args) == list else args
    cr.execute(query, args)
    logger.debug('Running %s', query % args)
    logger.debug('%s rows affected', cr.rowcount)
    return cr.rowcount


def column_exists(cr, table, column):
    """ Check whether a certain column exists """
    cr.execute(
        'SELECT count(attname) FROM pg_attribute '
        'WHERE attrelid = '
        '( SELECT oid FROM pg_class WHERE relname = %s ) '
        'AND attname = %s',
        (table, column))
    return cr.fetchone()[0] == 1


def update_module_names(cr, namespec):
    """
    Deal with changed module names of certified modules
    in order to prevent  'certificate not unique' error,
    as well as updating the module reference in the
    XML id.

    :param namespec: tuple of (old name, new name)
    """
    for (old_name, new_name) in namespec:
        query = ("UPDATE ir_module_module SET name = %s "
                 "WHERE name = %s")
        logged_query(cr, query, (new_name, old_name))
        query = ("UPDATE ir_model_data SET module = %s "
                 "WHERE module = %s ")
        logged_query(cr, query, (new_name, old_name))
        query = ("UPDATE ir_module_module_dependency SET name = %s "
                 "WHERE name = %s")
        logged_query(cr, query, (new_name, old_name))


def add_ir_model_fields(cr, columnspec):
    """
    Typically, new columns on ir_model_fields need to be added in a very
    early stage in the upgrade process of the base module, in raw sql
    as they need to be in place before any model gets initialized.
    Do not use for fields with additional SQL constraints, such as a
    reference to another table or the cascade constraint, but craft your
    own statement taking them into account.

    :param columnspec: tuple of (column name, column type)
    """
    for column in columnspec:
        query = 'ALTER TABLE ir_model_fields ADD COLUMN %s %s' % (
            column)
        logged_query(cr, query, [])


def get_legacy_name(original_name):
    """
    Returns a versioned name for legacy tables/columns/etc
    Use this function instead of some custom name to avoid
    collisions with future or past legacy tables/columns/etc

    :param original_name: the original name of the column
    :param version: current version as passed to migrate()
    """
    return 'openupgrade_legacy_'+('_').join(
        map(str, release.version_info[0:2]))+'_'+original_name


def m2o_to_x2m(cr, model, table, field, source_field):
    """
    Transform many2one relations into one2many or many2many.
    Use rename_columns in your pre-migrate script to retain the column's old
    value, then call m2o_to_x2m in your post-migrate script.

    WARNING: If converting to one2many, there can be data loss, because only
    one inverse record can be mapped in a one2many, but you can have multiple
    many2one pointing to the same target. Use it when the use case allows this
    conversion.

    :param model: The target model registry object
    :param table: The source table
    :param field: The new field name on the target model
    :param source_field: the (renamed) many2one column on the source table.

    .. versionadded:: 8.0
    """
    if not model._columns.get(field):
        raise orm.except_orm(
            "Error", "m2o_to_x2m: field %s doesn't exist in model %s" % (
                field, model._name))
    if isinstance(model._columns[field], (openerp.osv.fields.many2many,
                                          openerp.fields.Many2many)):
        rel, id1, id2 = openerp.osv.fields.many2many._sql_names(
            model._columns[field], model)
        logged_query(
            cr,
            """
            INSERT INTO %s (%s, %s)
            SELECT id, %s
            FROM %s
            WHERE %s is not null
            """ %
            (rel, id1, id2, source_field, table, source_field))
    elif isinstance(model._columns[field], (openerp.fields.One2many,
                                            openerp.osv.fields.one2many)):
        if isinstance(model._columns[field], openerp.fields.One2many):
            target_table = (
                model.pool[model._columns[field].comodel_name]._table)
            target_field = model._columns[field].inverse_name
        else:
            target_table = model.pool[model._columns[field]._obj]._table
            target_field = model._columns[field]._fields_id
        logged_query(
            cr,
            """
{'fraction_non_alphanumeric': 0.09596991743644241, 'fraction_numerical': 0.0032698438649554486, 'mean_word_length': 2.1772612168041037, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 30}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3718960', 'n_tokens_mistral': 14080, 'n_tokens_neox': 13544, 'n_words': 5158}