<commit_before>#!/usr/bin/env python
# -*- coding: utf-8; tab-width: 4; indent-tabs-mode: t -*-
#
# NetProfile: Custom database fields
# © Copyright 2013-2014 Alex 'Unik' Unigovsky
#
# This file is part of NetProfile.
# NetProfile is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later
# version.
#
# NetProfile is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General
# Public License along with NetProfile. If not, see
# <http://www.gnu.org/licenses/>.

from __future__ import (
	unicode_literals,
	print_function,
	absolute_import,
	division
)

from sqlalchemy import (
	and_,
	schema,
	type_coerce,
	types,
	util
)

from sqlalchemy.dialects import (
	mssql,
	mysql,
	oracle,
	postgresql
)

from sqlalchemy.sql import (
	expression,
	sqltypes
)
from sqlalchemy.ext.compiler import compiles

from netprofile.db import processors
from netprofile.common import ipaddr

import sys
import binascii


if sys.version < '3':
	from netprofile.db.enum2 import (
		EnumSymbol,
		EnumMeta,
		DeclEnum
	)
else:
	from netprofile.db.enum3 import (
		EnumSymbol,
		EnumMeta,
		DeclEnum
	)

import re

_D_MYSQL = frozenset([
	mysql.mysqlconnector.dialect,
	mysql.mysqldb.dialect,
	mysql.oursql.dialect,
	mysql.pymysql.dialect,
	mysql.pyodbc.dialect,
	mysql.zxjdbc.dialect
])

_D_PGSQL = frozenset([
	postgresql.pg8000.dialect,
	postgresql.psycopg2.dialect,
	postgresql.pypostgresql.dialect,
	postgresql.zxjdbc.dialect
])

_D_ORA = frozenset([
	oracle.cx_oracle.dialect,
	oracle.zxjdbc.dialect
])

_D_MSSQL = frozenset([
	mssql.adodbapi.dialect,
	mssql.mxodbc.dialect,
	mssql.pymssql.dialect,
	mssql.pyodbc.dialect,
	mssql.zxjdbc.dialect
])

def _is_mysql(d):
	return d.__class__ in _D_MYSQL

def _is_pgsql(d):
	return d.__class__ in _D_PGSQL

def _is_ora(d):
	return d.__class__ in _D_ORA

def _is_mssql(d):
	return d.__class__ in _D_MSSQL

class LargeBLOB(types.TypeDecorator):
	"""
	Large binary object.
	"""
	impl = types.LargeBinary

	def load_dialect_impl(self, dialect):
		if _is_mysql(dialect):
			return mysql.LONGBLOB()
		return self.impl

	@property
	def python_type(self):
		return bytes

class IPv4Address(types.TypeDecorator):
	"""
	Hybrid IPv4 address.
	"""
	impl = types.Integer

	def load_dialect_impl(self, dialect):
		if _is_mysql(dialect):
			return mysql.INTEGER(unsigned=True)
		if _is_pgsql(dialect):
			return postgresql.INET()
		return self.impl

	@property
	def python_type(self):
		return ipaddr.IPv4Address

	def process_bind_param(self, value, dialect):
		if value is None:
			return None
		if _is_pgsql(dialect):
			return str(value)
		return int(value)

	def process_result_value(self, value, dialect):
		if value is None:
			return None
		return ipaddr.IPv4Address(value)

class IPv6Address(types.TypeDecorator):
	"""
	Hybrid IPv6 address.
	"""
	impl = types.BINARY(16)

	def load_dialect_impl(self, dialect):
		if _is_pgsql(dialect):
			return postgresql.INET()
		return self.impl

	@property
	def python_type(self):
		return ipaddr.IPv6Address

	def process_bind_param(self, value, dialect):
		if value is None:
			return None
		if _is_pgsql(dialect):
			return str(value)
		return value.packed()

	def process_result_value(self, value, dialect):
		if value is None:
			return None
		return ipaddr.IPv6Address(value)

class IPv6Offset(types.TypeDecorator):
	"""
	IPv6 address offset.
	"""
	impl = types.Numeric(39, 0)
	MIN_VALUE = 0
	MAX_VALUE = 340282366920938463463374607431768211456

	def load_dialect_impl(self, dialect):
		if _is_mysql(dialect):
			return mysql.DECIMAL(precision=39, scale=0, unsigned=True)
		return self.impl

	@property
	def python_type(self):
		return int

class Money(types.TypeDecorator):
	"""
	Money amount.
	"""
	impl = types.Numeric(20, 8)

class Traffic(types.TypeDecorator):
	"""
	Amount of traffic in bytes.
	"""
	impl = types.Numeric(16, 0)
	MIN_VALUE = 0
	MAX_VALUE = 9999999999999999

	def load_dialect_impl(self, dialect):
		if _is_mysql(dialect):
			return mysql.DECIMAL(precision=16, scale=0, unsigned=True)
		return self.impl

	@property
	def python_type(self):
		return int

class PercentFraction(types.TypeDecorator):
	"""
	Highly accurate percent fraction.
	"""
	impl = types.Numeric(11, 10)

	def load_dialect_impl(self, dialect):
		if _is_mysql(dialect):
			return mysql.DECIMAL(precision=11, scale=10, unsigned=True)
		return self.impl

	@property
	def python_type(self):
		return int

class MACAddress(types.TypeDecorator):
	"""
	MAC address
	"""
	impl = types.String(17)
	
	@property
	def python_type(self):
		return types.String

	def process_bind_param(self, value, dialect):
		if value is None:
			return None
		return binascii.unhexlify(bytes(value.replace(':', ''), 'utf-8'))

	def process_result_value(self, value, dialect):
		if value is None:
			return None
		return ':'.join( [ "%02X" % x for x in value ] )



class NPBoolean(types.TypeDecorator, types.SchemaType):
	"""
	An almost-normal boolean type with a special case for MySQL.
	"""
	impl = types.Boolean

	def __init__(self, **kw):
		types.TypeDecorator.__init__(self, **kw)
		types.SchemaType.__init__(self, **kw)
<commit_msg>Fixed MACAddress type definition<commit_after>#!/usr/bin/env python
# -*- coding: utf-8; tab-width: 4; indent-tabs-mode: t -*-
#
# NetProfile: Custom database fields
# © Copyright 2013-2015 Alex 'Unik' Unigovsky
#
# This file is part of NetProfile.
# NetProfile is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later
# version.
#
# NetProfile is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General
# Public License along with NetProfile. If not, see
# <http://www.gnu.org/licenses/>.

from __future__ import (
	unicode_literals,
	print_function,
	absolute_import,
	division
)

from sqlalchemy import (
	and_,
	schema,
	type_coerce,
	types,
	util
)

from sqlalchemy.dialects import (
	mssql,
	mysql,
	oracle,
	postgresql
)

from sqlalchemy.sql import (
	expression,
	sqltypes
)
from sqlalchemy.ext.compiler import compiles

from netprofile.db import processors
from netprofile.common import ipaddr

import sys
import binascii


if sys.version < '3':
	from netprofile.db.enum2 import (
		EnumSymbol,
		EnumMeta,
		DeclEnum
	)
else:
	from netprofile.db.enum3 import (
		EnumSymbol,
		EnumMeta,
		DeclEnum
	)

import re

_D_MYSQL = frozenset([
	mysql.mysqlconnector.dialect,
	mysql.mysqldb.dialect,
	mysql.oursql.dialect,
	mysql.pymysql.dialect,
	mysql.pyodbc.dialect,
	mysql.zxjdbc.dialect
])

_D_PGSQL = frozenset([
	postgresql.pg8000.dialect,
	postgresql.psycopg2.dialect,
	postgresql.pypostgresql.dialect,
	postgresql.zxjdbc.dialect
])

_D_ORA = frozenset([
	oracle.cx_oracle.dialect,
	oracle.zxjdbc.dialect
])

_D_MSSQL = frozenset([
	mssql.adodbapi.dialect,
	mssql.mxodbc.dialect,
	mssql.pymssql.dialect,
	mssql.pyodbc.dialect,
	mssql.zxjdbc.dialect
])

def _is_mysql(d):
	return d.__class__ in _D_MYSQL

def _is_pgsql(d):
	return d.__class__ in _D_PGSQL

def _is_ora(d):
	return d.__class__ in _D_ORA

def _is_mssql(d):
	return d.__class__ in _D_MSSQL

class LargeBLOB(types.TypeDecorator):
	"""
	Large binary object.
	"""
	impl = types.LargeBinary

	def load_dialect_impl(self, dialect):
		if _is_mysql(dialect):
			return mysql.LONGBLOB()
		return self.impl

	@property
	def python_type(self):
		return bytes

class IPv4Address(types.TypeDecorator):
	"""
	Hybrid IPv4 address.
	"""
	impl = types.Integer

	def load_dialect_impl(self, dialect):
		if _is_mysql(dialect):
			return mysql.INTEGER(unsigned=True)
		if _is_pgsql(dialect):
			return postgresql.INET()
		return self.impl

	@property
	def python_type(self):
		return ipaddr.IPv4Address

	def process_bind_param(self, value, dialect):
		if value is None:
			return None
		if _is_pgsql(dialect):
			return str(value)
		return int(value)

	def process_result_value(self, value, dialect):
		if value is None:
			return None
		return ipaddr.IPv4Address(value)

class IPv6Address(types.TypeDecorator):
	"""
	Hybrid IPv6 address.
	"""
	impl = types.BINARY(16)

	def load_dialect_impl(self, dialect):
		if _is_pgsql(dialect):
			return postgresql.INET()
		return self.impl

	@property
	def python_type(self):
		return ipaddr.IPv6Address

	def process_bind_param(self, value, dialect):
		if value is None:
			return None
		if _is_pgsql(dialect):
			return str(value)
		return value.packed()

	def process_result_value(self, value, dialect):
		if value is None:
			return None
		return ipaddr.IPv6Address(value)

class IPv6Offset(types.TypeDecorator):
	"""
	IPv6 address offset.
	"""
	impl = types.Numeric(39, 0)
	MIN_VALUE = 0
	MAX_VALUE = 340282366920938463463374607431768211456

	def load_dialect_impl(self, dialect):
		if _is_mysql(dialect):
			return mysql.DECIMAL(precision=39, scale=0, unsigned=True)
		return self.impl

	@property
	def python_type(self):
		return int

class Money(types.TypeDecorator):
	"""
	Money amount.
	"""
	impl = types.Numeric(20, 8)

class Traffic(types.TypeDecorator):
	"""
	Amount of traffic in bytes.
	"""
	impl = types.Numeric(16, 0)
	MIN_VALUE = 0
	MAX_VALUE = 9999999999999999

	def load_dialect_impl(self, dialect):
		if _is_mysql(dialect):
			return mysql.DECIMAL(precision=16, scale=0, unsigned=True)
		return self.impl

	@property
	def python_type(self):
		return int

class PercentFraction(types.TypeDecorator):
	"""
	Highly accurate percent fraction.
	"""
	impl = types.Numeric(11, 10)

	def load_dialect_impl(self, dialect):
		if _is_mysql(dialect):
			return mysql.DECIMAL(precision=11, scale=10, unsigned=True)
		return self.impl

	@property
	def python_type(self):
		return int

class MACAddress(types.TypeDecorator):
	"""
	MAC address
	"""
	impl = types.BINARY(6)

	def load_dialect_impl(self, dialect):
		if _is_pgsql(dialect):
			return postgresql.MACADDR()
		return self.impl
	
	@property
	def python_type(self):
		return str

	def process_bind_param(self, value, dialect):
		if value is None:
			return None
		if _is_pgsql(dialect):
			return str(value)
		return binascii.unhexlify(bytes(value.replace(':', ''), 'utf-8'))

	def process_result_value(self, value, dialect):
		if value is None:
			return None
		if _is_pgsql(dialect):
			return value
		return ':'.join('%02x' % x for x in value)



class NPBoolean(types.TypeDecorator, types.SchemaType):
	"""
	An almost-normal boolean type with a special case for MySQL.
	"""
	impl = types.Boolean

	def __init__(self, **kw):
		types.TypeDecorator.__init__(self, **kw)
		types.SchemaType.__init__(self, **kw)

{'fraction_non_alphanumeric': 0.10239397471532323, 'fraction_numerical': 0.020711916076392, 'mean_word_length': 4.459618208516887, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4104032', 'n_tokens_mistral': 4508, 'n_tokens_neox': 4031, 'n_words': 1039}