<commit_before># Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Database model"""
import copy
import datetime
import logging
import netaddr
import re
import simplejson as json

from sqlalchemy import BigInteger
from sqlalchemy import Boolean
from sqlalchemy import Column
from sqlalchemy import ColumnDefault
from sqlalchemy import DateTime
from sqlalchemy import Enum
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy import Float
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy.orm import relationship, backref
from sqlalchemy import String
from sqlalchemy import Table
from sqlalchemy import Text
from sqlalchemy.types import TypeDecorator
from sqlalchemy import UniqueConstraint

from compass.db import exception
from compass.utils import util


BASE = declarative_base()


class JSONEncoded(TypeDecorator):
    """Represents an immutable structure as a json-encoded string."""

    impl = Text

    def process_bind_param(self, value, dialect):
        if value is not None:
            value = json.dumps(value)
        return value

    def process_result_value(self, value, dialect):
        if value is not None:
            value = json.loads(value)
        return value


class TimestampMixin(object):
    """Provides table fields for each row created/updated timestamp."""
    created_at = Column(DateTime, default=lambda: datetime.datetime.now())
    updated_at = Column(DateTime, default=lambda: datetime.datetime.now(),
                        onupdate=lambda: datetime.datetime.now())


class HelperMixin(object):
    """Provides general fuctions for all compass table models."""

    def initialize(self):
        self.update()

    def update(self):
        pass

    @staticmethod
    def type_compatible(value, column_type):
        """Check if value type is compatible with the column type."""
        if value is None:
            return True
        if not hasattr(column_type, 'python_type'):
            return True
        column_python_type = column_type.python_type
        if isinstance(value, column_python_type):
            return True
        if issubclass(column_python_type, basestring):
            return isinstance(value, basestring)
        if column_python_type in [int, long]:
            return type(value) in [int, long]
        if column_python_type in [float]:
            return type(value) in [float]
        if column_python_type in [bool]:
            return type(value) in [bool]
        return False

    def validate(self):
        """Generate validate function to make sure the record is legal."""
        columns = self.__mapper__.columns
        for key, column in columns.items():
            value = getattr(self, key)
            if not self.type_compatible(value, column.type):
                raise exception.InvalidParameter(
                    'column %s value %r type is unexpected: %s' % (
                        key, value, column.type
                    )
                )

    def to_dict(self):
        """General function to convert record to dict.

        Convert all columns not starting with '_' to
        {<column_name>: <column_value>}
        """
        keys = self.__mapper__.columns.keys()
        dict_info = {}
        for key in keys:
            if key.startswith('_'):
                continue
            value = getattr(self, key)
            if value is not None:
                if isinstance(value, datetime.datetime):
                    value = util.format_datetime(value)
                dict_info[key] = value
        return dict_info


class StateMixin(TimestampMixin, HelperMixin):
    """Provides general fields and functions for state related table."""

    state = Column(
        Enum(
            'UNINITIALIZED', 'INITIALIZED', 'UPDATE_PREPARING',
            'INSTALLING', 'SUCCESSFUL', 'ERROR'
        ),
        ColumnDefault('UNINITIALIZED')
    )
    percentage = Column(Float, default=0.0)
    message = Column(Text, default='')
    severity = Column(
        Enum('INFO', 'WARNING', 'ERROR'),
        ColumnDefault('INFO')
    )
    ready = Column(Boolean, default=False)

    def update(self):
        # In state table, some field information is redundant.
        # The update function to make sure all related fields
        # are set to correct state.
        if self.ready:
            self.state = 'SUCCESSFUL'
        if self.state in ['UNINITIALIZED', 'INITIALIZED']:
            self.percentage = 0.0
            self.severity = 'INFO'
            self.message = ''
        if self.state == 'INSTALLING':
            if self.severity == 'ERROR':
                self.state = 'ERROR'
            elif self.percentage >= 1.0:
                self.state = 'SUCCESSFUL'
                self.percentage = 1.0
        if self.state == 'SUCCESSFUL':
            self.percentage = 1.0
        super(StateMixin, self).update()


class LogHistoryMixin(TimestampMixin, HelperMixin):
    """Provides general fields and functions for LogHistory related tables."""
    position = Column(Integer, default=0)
    partial_line = Column(Text, default='')
    percentage = Column(Float, default=0.0)
    message = Column(Text, default='')
    severity = Column(
        Enum('ERROR', 'WARNING', 'INFO'),
        ColumnDefault('INFO')
    )
    line_matcher_name = Column(
        String(80), default='start'
    )

    def validate(self):
        # TODO(xicheng): some validation can be moved to column.
        if not self.filename:
            raise exception.InvalidParameter(
                'filename is not set in %s' % self.id
            )


class HostNetwork(BASE, TimestampMixin, HelperMixin):
    """Host network table."""
    __tablename__ = 'host_network'

    id = Column(Integer, primary_key=True)
    host_id = Column(
        Integer,
        ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE')
    )
    interface = Column(
        String(80), nullable=False)
    subnet_id = Column(
        Integer,
        ForeignKey('subnet.id', onupdate='CASCADE', ondelete='CASCADE')
    )
    user_id = Column(Integer, ForeignKey('user.id'))
    ip_int = Column(BigInteger, nullable=False)
    is_mgmt = Column(Boolean, default=False)
    is_promiscuous = Column(Boolean, default=False)

    __table_args__ = (
        UniqueConstraint('host_id', 'interface', name='constraint'),
        UniqueConstraint('ip_int', 'user_id', name='constraint')
    )

    def __init__(self, host_id, interface, user_id, **kwargs):
        self.host_id = host_id<commit_msg>Fix bug in model<commit_after># Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Database model"""
import copy
import datetime
import logging
import netaddr
import re
import simplejson as json

from sqlalchemy import BigInteger
from sqlalchemy import Boolean
from sqlalchemy import Column
from sqlalchemy import ColumnDefault
from sqlalchemy import DateTime
from sqlalchemy import Enum
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy import Float
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy.orm import relationship, backref
from sqlalchemy import String
from sqlalchemy import Table
from sqlalchemy import Text
from sqlalchemy.types import TypeDecorator
from sqlalchemy import UniqueConstraint

from compass.db import exception
from compass.utils import util


BASE = declarative_base()


class JSONEncoded(TypeDecorator):
    """Represents an immutable structure as a json-encoded string."""

    impl = Text

    def process_bind_param(self, value, dialect):
        if value is not None:
            value = json.dumps(value)
        return value

    def process_result_value(self, value, dialect):
        if value is not None:
            value = json.loads(value)
        return value


class TimestampMixin(object):
    """Provides table fields for each row created/updated timestamp."""
    created_at = Column(DateTime, default=lambda: datetime.datetime.now())
    updated_at = Column(DateTime, default=lambda: datetime.datetime.now(),
                        onupdate=lambda: datetime.datetime.now())


class HelperMixin(object):
    """Provides general fuctions for all compass table models."""

    def initialize(self):
        self.update()

    def update(self):
        pass

    @staticmethod
    def type_compatible(value, column_type):
        """Check if value type is compatible with the column type."""
        if value is None:
            return True
        if not hasattr(column_type, 'python_type'):
            return True
        column_python_type = column_type.python_type
        if isinstance(value, column_python_type):
            return True
        if issubclass(column_python_type, basestring):
            return isinstance(value, basestring)
        if column_python_type in [int, long]:
            return type(value) in [int, long]
        if column_python_type in [float]:
            return type(value) in [float]
        if column_python_type in [bool]:
            return type(value) in [bool]
        return False

    def validate(self):
        """Generate validate function to make sure the record is legal."""
        columns = self.__mapper__.columns
        for key, column in columns.items():
            value = getattr(self, key)
            if not self.type_compatible(value, column.type):
                raise exception.InvalidParameter(
                    'column %s value %r type is unexpected: %s' % (
                        key, value, column.type
                    )
                )

    def to_dict(self):
        """General function to convert record to dict.

        Convert all columns not starting with '_' to
        {<column_name>: <column_value>}
        """
        keys = self.__mapper__.columns.keys()
        dict_info = {}
        for key in keys:
            if key.startswith('_'):
                continue
            value = getattr(self, key)
            if value is not None:
                if isinstance(value, datetime.datetime):
                    value = util.format_datetime(value)
                dict_info[key] = value
        return dict_info


class StateMixin(TimestampMixin, HelperMixin):
    """Provides general fields and functions for state related table."""

    state = Column(
        Enum(
            'UNINITIALIZED', 'INITIALIZED', 'UPDATE_PREPARING',
            'INSTALLING', 'SUCCESSFUL', 'ERROR'
        ),
        ColumnDefault('UNINITIALIZED')
    )
    percentage = Column(Float, default=0.0)
    message = Column(Text, default='')
    severity = Column(
        Enum('INFO', 'WARNING', 'ERROR'),
        ColumnDefault('INFO')
    )
    ready = Column(Boolean, default=False)

    def update(self):
        # In state table, some field information is redundant.
        # The update function to make sure all related fields
        # are set to correct state.
        if self.ready:
            self.state = 'SUCCESSFUL'
        if self.state in ['UNINITIALIZED', 'INITIALIZED']:
            self.percentage = 0.0
            self.severity = 'INFO'
            self.message = ''
        if self.state == 'INSTALLING':
            if self.severity == 'ERROR':
                self.state = 'ERROR'
            elif self.percentage >= 1.0:
                self.state = 'SUCCESSFUL'
                self.percentage = 1.0
        if self.state == 'SUCCESSFUL':
            self.percentage = 1.0
        super(StateMixin, self).update()


class LogHistoryMixin(TimestampMixin, HelperMixin):
    """Provides general fields and functions for LogHistory related tables."""
    position = Column(Integer, default=0)
    partial_line = Column(Text, default='')
    percentage = Column(Float, default=0.0)
    message = Column(Text, default='')
    severity = Column(
        Enum('ERROR', 'WARNING', 'INFO'),
        ColumnDefault('INFO')
    )
    line_matcher_name = Column(
        String(80), default='start'
    )

    def validate(self):
        # TODO(xicheng): some validation can be moved to column.
        if not self.filename:
            raise exception.InvalidParameter(
                'filename is not set in %s' % self.id
            )


class HostNetwork(BASE, TimestampMixin, HelperMixin):
    """Host network table."""
    __tablename__ = 'host_network'

    id = Column(Integer, primary_key=True)
    host_id = Column(
        Integer,
        ForeignKey('host.id', onupdate='CASCADE', ondelete='CASCADE')
    )
    interface = Column(
        String(80), nullable=False)
    subnet_id = Column(
        Integer,
        ForeignKey('subnet.id', onupdate='CASCADE', ondelete='CASCADE')
    )
    user_id = Column(Integer, ForeignKey('user.id'))
    ip_int = Column(BigInteger, nullable=False)
    is_mgmt = Column(Boolean, default=False)
    is_promiscuous = Column(Boolean, default=False)

    __table_args__ = (
        UniqueConstraint('host_id', 'interface', name='interface_constraint'),
        UniqueConstraint('ip_int', 'user_id', name='ip_constraint')
    )

    def __init__(self, host_id, interface, user_id, **kwargs):
        self.host_id = host_id
{'fraction_non_alphanumeric': 0.09154633394083836, 'fraction_numerical': 0.0035048366746109633, 'mean_word_length': 2.597327281896117, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3996144', 'n_tokens_mistral': 3938, 'n_tokens_neox': 3781, 'n_words': 1228}