<commit_before>import logging
import threading
from socket import AF_UNSPEC
from pyroute2.common import basestring
from pyroute2.netlink import nlmsg
from pyroute2.netlink.rtnl.rtmsg import rtmsg
from pyroute2.netlink.rtnl.rtmsg import nh as rtmsg_nh
from pyroute2.netlink.rtnl.req import IPRouteRequest
from pyroute2.ipdb.transactional import Transactional
from pyroute2.ipdb.linkedset import LinkedSet


class Metrics(Transactional):

    _fields = [rtmsg.metrics.nla2name(i[0]) for i in rtmsg.metrics.nla_map]


class NextHopSet(LinkedSet):

    def __init__(self, prime=None):
        super(NextHopSet, self).__init__()
        prime = prime or []
        for v in prime:
            self.add(v)

    def __sub__(self, vs):
        ret = type(self)()
        sub = set(self.raw.keys()) - set(vs.raw.keys())
        for v in sub:
            ret.add(self[v], raw=self.raw[v])
        return ret

    def __make_nh(self, prime):
        return (prime.get('flags', 0),
                prime.get('hops', 0),
                prime.get('ifindex', 0),
                prime.get('gateway'))

    def __getitem__(self, key):
        return dict(zip(('flags', 'hops', 'ifindex', 'gateway'), key))

    def __iter__(self):
        def NHIterator():
            for x in tuple(self.raw.keys()):
                yield self[x]
        return NHIterator()

    def add(self, prime, raw=None):
        return super(NextHopSet, self).add(self.__make_nh(prime))

    def remove(self, prime, raw=None):
        hit = False
        for nh in self:
            for key in prime:
                if prime[key] != nh.get(key):
                    break
            else:
                hit = True
                super(NextHopSet, self).remove(self.__make_nh(nh))
        if not hit:
            raise KeyError('nexthop not found')


class WatchdogKey(dict):
    '''
    Construct from a route a dictionary that could be used as
    a match for IPDB watchdogs.
    '''
    def __init__(self, route):
        dict.__init__(self, [x for x in IPRouteRequest(route).items()
                             if x[0] in ('dst', 'dst_len', 'oif',
                                         'iif', 'table')])


def RouteKey(msg):
    '''
    Construct from a netlink message a key that can be used
    to locate the route in the table
    '''
    if isinstance(msg, nlmsg):
        src = None
        # calculate dst
        if msg.get_attr('RTA_DST', None) is not None:
            dst = '%s/%s' % (msg.get_attr('RTA_DST'),
                             msg['dst_len'])
        else:
            dst = 'default'
        # use output | input interfaces as key also
        iif = msg.get_attr(msg.name2nla('iif'))
        oif = msg.get_attr(msg.name2nla('oif'))
    elif isinstance(msg, Transactional):
        src = None
        dst = msg.get('dst')
        iif = msg.get('iif')
        oif = msg.get('oif')
    else:
        raise TypeError('prime not supported')
    # key: src, dst, iif, oif
    return (src, dst, iif, oif)


class Route(Transactional):
    '''
    Persistent transactional route object
    '''

    _fields = [rtmsg.nla2name(i[0]) for i in rtmsg.nla_map]
    _fields.append('flags')
    _fields.append('src_len')
    _fields.append('dst_len')
    _fields.append('table')
    _fields.append('removal')
    _virtual_fields = ['ipdb_scope', 'ipdb_priority']
    _fields.extend(_virtual_fields)
    _linked_sets = ['multipath', ]
    cleanup = ('attrs',
               'header',
               'event',
               'cacheinfo')

    def __init__(self, ipdb, mode=None, parent=None, uid=None):
        Transactional.__init__(self, ipdb, mode, parent, uid)
        self._load_event = threading.Event()
        with self._direct_state:
            for i in self._fields:
                self[i] = None
            self['metrics'] = Metrics(parent=self)
            self['multipath'] = NextHopSet()
            self['ipdb_priority'] = 0

    def add_nh(self, prime):
        with self._write_lock:
            tx = self.get_tx()
            with tx._direct_state:
                tx['multipath'].add(prime)

    def del_nh(self, prime):
        with self._write_lock:
            tx = self.get_tx()
            with tx._direct_state:
                tx['multipath'].remove(prime)

    def load_netlink(self, msg):
        with self._direct_state:
            if self['ipdb_scope'] == 'locked':
                # do not touch locked interfaces
                return

            self['ipdb_scope'] = 'system'
            self.update(msg)

            # re-init metrics
            metrics = self.get('metrics', Metrics(parent=self))
            with metrics._direct_state:
                for metric in tuple(metrics.keys()):
                    del metrics[metric]
            self['metrics'] = metrics

            # merge key
            for (name, value) in msg['attrs']:
                norm = rtmsg.nla2name(name)
                # normalize RTAX
                if norm == 'metrics':
                    with self['metrics']._direct_state:
                        for (rtax, rtax_value) in value['attrs']:
                            rtax_norm = rtmsg.metrics.nla2name(rtax)
                            self['metrics'][rtax_norm] = rtax_value
                elif norm == 'multipath':
                    self['multipath'] = NextHopSet()
                    for v in value:
                        nh = {}
                        for name in [x[0] for x in rtmsg_nh.fields]:
                            nh[name] = v[name]
                        for (rta, rta_value) in v['attrs']:
                            rta_norm = rtmsg.nla2name(rta)
                            nh[rta_norm] = rta_value
                        self['multipath'].add(nh)
                else:<commit_msg>ipdb: route nh attrs are optional<commit_after>import logging
import threading
from socket import AF_UNSPEC
from pyroute2.common import basestring
from pyroute2.netlink import nlmsg
from pyroute2.netlink.rtnl.rtmsg import rtmsg
from pyroute2.netlink.rtnl.rtmsg import nh as rtmsg_nh
from pyroute2.netlink.rtnl.req import IPRouteRequest
from pyroute2.ipdb.transactional import Transactional
from pyroute2.ipdb.linkedset import LinkedSet


class Metrics(Transactional):

    _fields = [rtmsg.metrics.nla2name(i[0]) for i in rtmsg.metrics.nla_map]


class NextHopSet(LinkedSet):

    def __init__(self, prime=None):
        super(NextHopSet, self).__init__()
        prime = prime or []
        for v in prime:
            self.add(v)

    def __sub__(self, vs):
        ret = type(self)()
        sub = set(self.raw.keys()) - set(vs.raw.keys())
        for v in sub:
            ret.add(self[v], raw=self.raw[v])
        return ret

    def __make_nh(self, prime):
        return (prime.get('flags', 0),
                prime.get('hops', 0),
                prime.get('ifindex', 0),
                prime.get('gateway'))

    def __getitem__(self, key):
        return dict(zip(('flags', 'hops', 'ifindex', 'gateway'), key))

    def __iter__(self):
        def NHIterator():
            for x in tuple(self.raw.keys()):
                yield self[x]
        return NHIterator()

    def add(self, prime, raw=None):
        return super(NextHopSet, self).add(self.__make_nh(prime))

    def remove(self, prime, raw=None):
        hit = False
        for nh in self:
            for key in prime:
                if prime[key] != nh.get(key):
                    break
            else:
                hit = True
                super(NextHopSet, self).remove(self.__make_nh(nh))
        if not hit:
            raise KeyError('nexthop not found')


class WatchdogKey(dict):
    '''
    Construct from a route a dictionary that could be used as
    a match for IPDB watchdogs.
    '''
    def __init__(self, route):
        dict.__init__(self, [x for x in IPRouteRequest(route).items()
                             if x[0] in ('dst', 'dst_len', 'oif',
                                         'iif', 'table')])


def RouteKey(msg):
    '''
    Construct from a netlink message a key that can be used
    to locate the route in the table
    '''
    if isinstance(msg, nlmsg):
        src = None
        # calculate dst
        if msg.get_attr('RTA_DST', None) is not None:
            dst = '%s/%s' % (msg.get_attr('RTA_DST'),
                             msg['dst_len'])
        else:
            dst = 'default'
        # use output | input interfaces as key also
        iif = msg.get_attr(msg.name2nla('iif'))
        oif = msg.get_attr(msg.name2nla('oif'))
    elif isinstance(msg, Transactional):
        src = None
        dst = msg.get('dst')
        iif = msg.get('iif')
        oif = msg.get('oif')
    else:
        raise TypeError('prime not supported')
    # key: src, dst, iif, oif
    return (src, dst, iif, oif)


class Route(Transactional):
    '''
    Persistent transactional route object
    '''

    _fields = [rtmsg.nla2name(i[0]) for i in rtmsg.nla_map]
    _fields.append('flags')
    _fields.append('src_len')
    _fields.append('dst_len')
    _fields.append('table')
    _fields.append('removal')
    _virtual_fields = ['ipdb_scope', 'ipdb_priority']
    _fields.extend(_virtual_fields)
    _linked_sets = ['multipath', ]
    cleanup = ('attrs',
               'header',
               'event',
               'cacheinfo')

    def __init__(self, ipdb, mode=None, parent=None, uid=None):
        Transactional.__init__(self, ipdb, mode, parent, uid)
        self._load_event = threading.Event()
        with self._direct_state:
            for i in self._fields:
                self[i] = None
            self['metrics'] = Metrics(parent=self)
            self['multipath'] = NextHopSet()
            self['ipdb_priority'] = 0

    def add_nh(self, prime):
        with self._write_lock:
            tx = self.get_tx()
            with tx._direct_state:
                tx['multipath'].add(prime)

    def del_nh(self, prime):
        with self._write_lock:
            tx = self.get_tx()
            with tx._direct_state:
                tx['multipath'].remove(prime)

    def load_netlink(self, msg):
        with self._direct_state:
            if self['ipdb_scope'] == 'locked':
                # do not touch locked interfaces
                return

            self['ipdb_scope'] = 'system'
            self.update(msg)

            # re-init metrics
            metrics = self.get('metrics', Metrics(parent=self))
            with metrics._direct_state:
                for metric in tuple(metrics.keys()):
                    del metrics[metric]
            self['metrics'] = metrics

            # merge key
            for (name, value) in msg['attrs']:
                norm = rtmsg.nla2name(name)
                # normalize RTAX
                if norm == 'metrics':
                    with self['metrics']._direct_state:
                        for (rtax, rtax_value) in value['attrs']:
                            rtax_norm = rtmsg.metrics.nla2name(rtax)
                            self['metrics'][rtax_norm] = rtax_value
                elif norm == 'multipath':
                    self['multipath'] = NextHopSet()
                    for v in value:
                        nh = {}
                        for name in [x[0] for x in rtmsg_nh.fields]:
                            nh[name] = v[name]
                        for (rta, rta_value) in v.get('attrs', ()):
                            rta_norm = rtmsg.nla2name(rta)
                            nh[rta_norm] = rta_value
                        self['multipath'].add(nh)
                else:
{'fraction_non_alphanumeric': 0.11477253070403044, 'fraction_numerical': 0.003805569970593323, 'mean_word_length': 1.7322778827977316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3673554', 'n_tokens_mistral': 3532, 'n_tokens_neox': 3393, 'n_words': 846}