<commit_before>from __future__ import absolute_import,unicode_literals

import re
import json
import time
import base64
import socket
import struct
import datetime

import synapse.compat as s_compat
import synapse.lib.socket as s_socket
import synapse.lib.urlhelp as s_urlhelp
import synapse.swarm.syntax as s_syntax

from synapse.common import *

class DataType:

    subprops = ()

    def __init__(self, tlib, name, **info):
        self.tlib = tlib
        self.name = name
        self.info = info

    def _raiseBadValu(self, valu, **info):
        raise BadTypeValu(name=self.name, valu=valu, **info)

    def subs(self):
        '''
        Implement if the presence of a property with this type requires sub props.
        '''
        return self.subprops

    def chop(self, valu):
        '''
        Returns a tuple of (norm,subs) for the given valu.
        '''
        return self.norm(valu),{}

    def extend(self, name, **info):
        '''
        Construct a new subtype from this instance.
        '''
        for k,v in self.info.items():
            info.setdefault(k,v)

        return self.__class__(self.tlib, name,**info)

class StrType(DataType):

    def __init__(self, tlib, name, **info):
        DataType.__init__(self, tlib, name, **info)

        self.regex = None

        regex = info.get('regex')
        if regex != None:
            self.regex = re.compile(regex)

    def repr(self, valu):
        return valu

    def norm(self, valu):
        if self.info.get('lower'):
            valu = valu.lower()

        enums = self.info.get('enums')
        if enums != None and valu not in enums:
            self._raiseBadValu(valu,enums=enums)

        if self.regex != None and not self.regex.match(valu):
            self._raiseBadValu(valu,regex=self.info.get('regex'))

        return valu

    def parse(self, valu):
        return self.norm(valu)

class IntType(DataType):

    def __init__(self, tlib, name, **info):
        DataType.__init__(self, tlib, name, **info)

        self.fmt = info.get('fmt','%d')
        #self.modval = info.get('mod',None)
        self.minval = info.get('min',None)
        self.maxval = info.get('max',None)

    def repr(self, valu):
        return self.fmt % valu

    def norm(self, valu):
<commit_msg>use isint() in IntType norm() method to raise on non-int types<commit_after>from __future__ import absolute_import,unicode_literals

import re
import json
import time
import base64
import socket
import struct
import datetime

import synapse.compat as s_compat
import synapse.lib.socket as s_socket
import synapse.lib.urlhelp as s_urlhelp
import synapse.swarm.syntax as s_syntax

from synapse.common import *

class DataType:

    subprops = ()

    def __init__(self, tlib, name, **info):
        self.tlib = tlib
        self.name = name
        self.info = info

    def _raiseBadValu(self, valu, **info):
        raise BadTypeValu(name=self.name, valu=valu, **info)

    def subs(self):
        '''
        Implement if the presence of a property with this type requires sub props.
        '''
        return self.subprops

    def chop(self, valu):
        '''
        Returns a tuple of (norm,subs) for the given valu.
        '''
        return self.norm(valu),{}

    def extend(self, name, **info):
        '''
        Construct a new subtype from this instance.
        '''
        for k,v in self.info.items():
            info.setdefault(k,v)

        return self.__class__(self.tlib, name,**info)

class StrType(DataType):

    def __init__(self, tlib, name, **info):
        DataType.__init__(self, tlib, name, **info)

        self.regex = None

        regex = info.get('regex')
        if regex != None:
            self.regex = re.compile(regex)

    def repr(self, valu):
        return valu

    def norm(self, valu):
        if self.info.get('lower'):
            valu = valu.lower()

        enums = self.info.get('enums')
        if enums != None and valu not in enums:
            self._raiseBadValu(valu,enums=enums)

        if self.regex != None and not self.regex.match(valu):
            self._raiseBadValu(valu,regex=self.info.get('regex'))

        return valu

    def parse(self, valu):
        return self.norm(valu)

class IntType(DataType):

    def __init__(self, tlib, name, **info):
        DataType.__init__(self, tlib, name, **info)

        self.fmt = info.get('fmt','%d')
        #self.modval = info.get('mod',None)
        self.minval = info.get('min',None)
        self.maxval = info.get('max',None)

    def repr(self, valu):
        return self.fmt % valu

    def norm(self, valu):

        if not s_compat.isint(valu):
            self._raiseBadValu(valu)

{'fraction_non_alphanumeric': 0.11784799316823227, 'fraction_numerical': 0.0008539709649871904, 'mean_word_length': 2.35361488904796, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5284834', 'n_tokens_mistral': 1522, 'n_tokens_neox': 1470, 'n_words': 402}