<commit_before># Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Forseti db migrator."""

from __future__ import print_function

from builtins import object
import os
import sys

# Importing migrate.changeset adds some new methods to existing SQLAlchemy
# objects but we will not be calling the library directly.
import migrate.changeset  # noqa: F401, pylint: disable=unused-import
from sqlalchemy.exc import OperationalError

import google.cloud.forseti.services.scanner.dao as scanner_dao
import google.cloud.forseti.services.inventory.storage as inventory_dao
import google.cloud.forseti.services.dao as general_dao

from google.cloud.forseti.common.util import logger

DB_NAME = os.environ.get('FORSETI_DB_NAME', 'forseti_security')
DB_USER = os.environ.get('SQL_DB_USER', '')
DB_PASSWORD = os.environ.get('SQL_DB_PASSWORD', '')
DEFAULT_DB_CONN_STR = (f'mysql+pymysql://{DB_USER}:{DB_PASSWORD}@'
                       f'127.0.0.1:3306/{DB_NAME}')
LOGGER = logger.get_logger(__name__)


class ColumnAction(object):
    """Column action class."""
    DROP = 'DROP'
    CREATE = 'CREATE'
    ALTER = 'ALTER'


def create_column(table, column):
    """Create Column.

    Args:
        table (sqlalchemy.schema.Table): The sql alchemy table object.
        column (sqlalchemy.schema.Column): The sql alchemy column object.
    """
    LOGGER.info('Attempting to create column: %s', column.name)
    column.create(table, populate_default=True)


def alter_column(table, old_column, new_column):
    """Alter Column.

    Args:
        table (sqlalchemy.schema.Table): The sql alchemy table object.
        old_column (sqlalchemy.schema.Column): The sql alchemy column object,
            this is the column to be modified.
        new_column (sqlalchemy.schema.Column): The sql alchemy column object,
            this is the column to update to.
    """
    LOGGER.info('Attempting to alter column: %s', old_column.name)

    # bind the old column with the corresponding table.
    old_column.table = table

    old_column.alter(name=new_column.name,
                     type=new_column.type,
                     nullable=new_column.nullable)


def drop_column(table, column):
    """Create Column.

    Args:
        table (sqlalchemy.schema.Table): The sql alchemy table object.
        column (sqlalchemy.schema.Column): The sql alchemy column object.
    """
    LOGGER.info('Attempting to drop column: %s', column.name)
    column.drop(table)


COLUMN_ACTION_MAPPING = {ColumnAction.DROP: drop_column,
                         ColumnAction.CREATE: create_column,
                         ColumnAction.ALTER: alter_column}


def migrate_schema(base, dao_classes):
    """Migrate database schema.

    Args:
        base (Base): Declarative base.
        dao_classes (list): A list of dao classes.
    """

    # Find all the Table objects for each of the classes.
    # The format of tables is: {table_name: Table object}.
    tables = base.metadata.tables

    schema_update_actions_method = 'get_schema_update_actions'

    for dao_class in dao_classes:
        get_schema_update_actions = getattr(dao_class,
                                            schema_update_actions_method,
                                            None)
        if (not callable(get_schema_update_actions) or
                dao_class.__tablename__ not in tables):
            LOGGER.info('Table %s doesn\'t require update.',
                        dao_class.__tablename__)
            continue
        LOGGER.info('Updating table %s', dao_class.__tablename__)
        # schema_update will require the Table object.
        table = tables.get(dao_class.__tablename__)
        schema_update_actions = get_schema_update_actions()
        for column_action, columns in schema_update_actions.items():
            if column_action in [ColumnAction.CREATE, ColumnAction.DROP]:
                _create_or_drop_columns(column_action, columns, table)
            elif column_action in [ColumnAction.ALTER]:<commit_msg>Handle existing columns from db migrator script; currently an exception is raised if a column already exists, this will skip those columns instead. (#3590)<commit_after># Copyright 2020 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Forseti db migrator."""

from __future__ import print_function

from builtins import object
import os
import sys

# Importing migrate.changeset adds some new methods to existing SQLAlchemy
# objects but we will not be calling the library directly.
import migrate.changeset  # noqa: F401, pylint: disable=unused-import
from sqlalchemy.exc import OperationalError

from google.cloud.forseti.common.util import logger
import google.cloud.forseti.services.scanner.dao as scanner_dao
import google.cloud.forseti.services.inventory.storage as inventory_dao
import google.cloud.forseti.services.dao as general_dao


DB_NAME = os.environ.get('FORSETI_DB_NAME', 'forseti_security')
DB_USER = os.environ.get('SQL_DB_USER', '')
DB_PASSWORD = os.environ.get('SQL_DB_PASSWORD', '')
DEFAULT_DB_CONN_STR = (f'mysql+pymysql://{DB_USER}:{DB_PASSWORD}@'
                       f'127.0.0.1:3306/{DB_NAME}')
LOGGER = logger.get_logger(__name__)


class ColumnAction(object):
    """Column action class."""
    DROP = 'DROP'
    CREATE = 'CREATE'
    ALTER = 'ALTER'


def alter_column(table, old_column, new_column):
    """Alter Column.

    Args:
        table (sqlalchemy.schema.Table): The sql alchemy table object.
        old_column (sqlalchemy.schema.Column): The sql alchemy column object,
            this is the column to be modified.
        new_column (sqlalchemy.schema.Column): The sql alchemy column object,
            this is the column to update to.
    """
    LOGGER.info(f'Attempting to alter column {table.name}.{old_column.name}.')

    # bind the old column with the corresponding table.
    old_column.table = table
    old_column.alter(name=new_column.name,
                     type=new_column.type,
                     nullable=new_column.nullable)


def create_column(table, column):
    """Create Column.

    Args:
        table (sqlalchemy.schema.Table): The sql alchemy table object.
        column (sqlalchemy.schema.Column): The sql alchemy column object.
    """
    if column.name not in table.columns:
        LOGGER.info(f'Attempting to create column {table.name}.{column.name}')
        column.create(table, populate_default=True)


def drop_column(table, column):
    """Create Column.

    Args:
        table (sqlalchemy.schema.Table): The sql alchemy table object.
        column (sqlalchemy.schema.Column): The sql alchemy column object.
    """
    LOGGER.info(f'Attempting to drop column {table.name}.{column.name}')
    column.drop(table)


COLUMN_ACTION_MAPPING = {ColumnAction.DROP: drop_column,
                         ColumnAction.CREATE: create_column,
                         ColumnAction.ALTER: alter_column}


def migrate_schema(base, dao_classes):
    """Migrate database schema.

    Args:
        base (Base): Declarative base.
        dao_classes (list): A list of dao classes.
    """
    # Find all the Table objects for each of the classes.
    # The format of tables is: {table_name: Table object}.
    tables = base.metadata.tables
    schema_update_actions_method = 'get_schema_update_actions'

    for dao_class in dao_classes:
        get_schema_update_actions = getattr(dao_class,
                                            schema_update_actions_method,
                                            None)
        if not callable(get_schema_update_actions):
            LOGGER.info(f'Table {dao_class.__tablename__} has not implemented '
                        f'the get_schema_update_actions method.')
            continue
        if dao_class.__tablename__ not in tables:
            LOGGER.info(f'Table {dao_class.__tablename__} not found in '
                        f'existing database tables.')
            continue

        LOGGER.info('Updating table %s', dao_class.__tablename__)
        table = tables.get(dao_class.__tablename__)
        schema_update_actions = get_schema_update_actions()

        for column_action, columns in schema_update_actions.items():
            if column_action in [ColumnAction.CREATE, ColumnAction.DROP]:
                _create_or_drop_columns(column_action, columns, table)
            elif column_action in [ColumnAction.ALTER]:
{'fraction_non_alphanumeric': 0.08558463486703251, 'fraction_numerical': 0.0048543689320388345, 'mean_word_length': 2.975251677852349, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2220927', 'n_tokens_mistral': 2761, 'n_tokens_neox': 2664, 'n_words': 805}