<commit_before># Copyright 2015 PLUMgrid
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import atexit
from collections import MutableMapping
import ctypes as ct
import fcntl
import json
import os
from subprocess import Popen, PIPE
import sys
basestring = (unicode if sys.version_info[0] < 3 else str)

lib = ct.CDLL("libbcc.so")

# keep in sync with bpf_common.h
lib.bpf_module_create_b.restype = ct.c_void_p
lib.bpf_module_create_b.argtypes = [ct.c_char_p, ct.c_char_p, ct.c_uint]
lib.bpf_module_create_c.restype = ct.c_void_p
lib.bpf_module_create_c.argtypes = [ct.c_char_p, ct.c_uint]
lib.bpf_module_create_c_from_string.restype = ct.c_void_p
lib.bpf_module_create_c_from_string.argtypes = [ct.c_char_p, ct.c_uint]
lib.bpf_module_destroy.restype = None
lib.bpf_module_destroy.argtypes = [ct.c_void_p]
lib.bpf_module_license.restype = ct.c_char_p
lib.bpf_module_license.argtypes = [ct.c_void_p]
lib.bpf_module_kern_version.restype = ct.c_uint
lib.bpf_module_kern_version.argtypes = [ct.c_void_p]
lib.bpf_num_functions.restype = ct.c_ulonglong
lib.bpf_num_functions.argtypes = [ct.c_void_p]
lib.bpf_function_name.restype = ct.c_char_p
lib.bpf_function_name.argtypes = [ct.c_void_p, ct.c_ulonglong]
lib.bpf_function_start.restype = ct.c_void_p
lib.bpf_function_start.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_function_size.restype = ct.c_size_t
lib.bpf_function_size.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_table_id.restype = ct.c_ulonglong
lib.bpf_table_id.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_table_fd.restype = ct.c_int
lib.bpf_table_fd.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_table_type_id.restype = ct.c_int
lib.bpf_table_type_id.argtypes = [ct.c_void_p, ct.c_ulonglong]
lib.bpf_table_key_desc.restype = ct.c_char_p
lib.bpf_table_key_desc.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_table_leaf_desc.restype = ct.c_char_p
lib.bpf_table_leaf_desc.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_table_key_snprintf.restype = ct.c_int
lib.bpf_table_key_snprintf.argtypes = [ct.c_void_p, ct.c_ulonglong,
        ct.c_char_p, ct.c_ulonglong, ct.c_void_p]
lib.bpf_table_leaf_snprintf.restype = ct.c_int
lib.bpf_table_leaf_snprintf.argtypes = [ct.c_void_p, ct.c_ulonglong,
        ct.c_char_p, ct.c_ulonglong, ct.c_void_p]
lib.bpf_table_key_sscanf.restype = ct.c_int
lib.bpf_table_key_sscanf.argtypes = [ct.c_void_p, ct.c_ulonglong,
        ct.c_char_p, ct.c_void_p]
lib.bpf_table_leaf_sscanf.restype = ct.c_int
lib.bpf_table_leaf_sscanf.argtypes = [ct.c_void_p, ct.c_ulonglong,
        ct.c_char_p, ct.c_void_p]

# keep in sync with libbpf.h
lib.bpf_get_next_key.restype = ct.c_int
lib.bpf_get_next_key.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p]
lib.bpf_lookup_elem.restype = ct.c_int
lib.bpf_lookup_elem.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p]
lib.bpf_update_elem.restype = ct.c_int
lib.bpf_update_elem.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p,
        ct.c_ulonglong]
lib.bpf_delete_elem.restype = ct.c_int
lib.bpf_delete_elem.argtypes = [ct.c_int, ct.c_void_p]
lib.bpf_open_raw_sock.restype = ct.c_int
lib.bpf_open_raw_sock.argtypes = [ct.c_char_p]
lib.bpf_attach_socket.restype = ct.c_int
lib.bpf_attach_socket.argtypes = [ct.c_int, ct.c_int]
lib.bpf_prog_load.restype = ct.c_int
lib.bpf_prog_load.argtypes = [ct.c_int, ct.c_void_p, ct.c_size_t,
        ct.c_char_p, ct.c_uint, ct.c_char_p, ct.c_uint]
lib.bpf_attach_kprobe.restype = ct.c_int
lib.bpf_attach_kprobe.argtypes = [ct.c_int, ct.c_char_p,
        ct.c_char_p, ct.c_int, ct.c_int, ct.c_int]
lib.bpf_detach_kprobe.restype = ct.c_int
lib.bpf_detach_kprobe.argtypes = [ct.c_char_p]

open_kprobes = {}
tracefile = None
TRACEFS = "/sys/kernel/debug/tracing"
KALLSYMS = "/proc/kallsyms"
ksym_addrs = []
ksym_names = []
ksym_loaded = 0
stars_max = 38

@atexit.register
def cleanup_kprobes():
    for k, v in open_kprobes.items():
        os.close(v)
        desc = "-:kprobes/%s" % k
        lib.bpf_detach_kprobe(desc.encode("ascii"))
    if tracefile:
        tracefile.close()

class BPF(object):
    SOCKET_FILTER = 1
    KPROBE = 2
    SCHED_CLS = 3
    SCHED_ACT = 4

    HASH = 1
    ARRAY = 2
    PROG_ARRAY = 3

    class Function(object):
        def __init__(self, bpf, name, fd):
            self.bpf = bpf
            self.name = name
            self.fd = fd

    class Table(MutableMapping):
        def __init__(self, bpf, map_id, map_fd, keytype, leaftype):
            self.bpf = bpf
            self.map_id = map_id
            self.map_fd = map_fd
            self.Key = keytype
            self.Leaf = leaftype
            self.ttype = lib.bpf_table_type_id(self.bpf.module, self.map_id)

        def key_sprintf(self, key):
            key_p = ct.pointer(key)
            buf = ct.create_string_buffer(ct.sizeof(self.Key) * 8)
            res = lib.bpf_table_key_snprintf(self.bpf.module, self.map_id,
                    buf, len(buf), key_p)
            if res < 0:
                raise Exception("Could not printf key")
            return buf.value

        def leaf_sprintf(self, leaf):
            leaf_p = ct.pointer(leaf)
            buf = ct.create_string_buffer(ct.sizeof(self.Leaf) * 8)
            res = lib.bpf_table_leaf_snprintf(self.bpf.module, self.map_id,
                    buf, len(buf), leaf_p)
            if res < 0:
                raise Exception("Could not printf leaf")
            return buf.value

        def key_scanf(self, key_str):
            key = self.Key()
            key_p = ct.pointer(key)
            res = lib.bpf_table_key_sscanf(self.bpf.module, self.map_id,
                    key_str, key_p)
            if res < 0:
                raise Exception("Could not scanf key")
            return key

        def leaf_scanf(self, leaf_str):
            leaf = self.Leaf()
            leaf_p = ct.pointer(leaf)
            res = lib.bpf_table_leaf_sscanf(self.bpf.module, self.map_id,
                    leaf_str, leaf_p)
            if res < 0:
                raise Exception("Could not scanf leaf")
            return leaf

        def __getitem__(self, key):
            key_p = ct.pointer(key)
            leaf = self.Leaf()
            leaf_p = ct.pointer(leaf)
            res = lib.bpf_lookup_elem(self.map_fd,
                    ct.cast(key_p, ct.c_void_p),
                    ct.cast(leaf_p, ct.c_void_p))
            if res < 0:
                raise KeyError
            return leaf

        def __setitem__(self, key, leaf):
            key_p = ct.pointer(key)
            leaf_p = ct.pointer(leaf)
            res = lib.bpf_update_elem(self.map_fd,
                    ct.cast(key_p, ct.c_void_p),
                    ct.cast(leaf_p, ct.c_void_p), 0)
            if res < 0:
                raise Exception("Could not update table")

        def __len__(self):
            i = 0
            for k in self: i += 1
            return i

        def __delitem__(self, key):
            key_p = ct.pointer(key)
            ttype = lib.bpf_table_type_id(self.bpf.module, self.map_id)
            # Deleting from array type maps does not have an effect, so
            # zero out the entry instead.
            if ttype in (BPF.ARRAY, BPF.PROG_ARRAY):
                leaf = self.Leaf()
                leaf_p = ct.pointer(leaf)
                res = lib.bpf_update_elem(self.map_fd,
                        ct.cast(key_p, ct.c_void_p),
                        ct.cast(leaf_p, ct.c_void_p), 0)
                if res < 0:
                    raise Exception("Could not clear item")
            else:
                res = lib.bpf_delete_elem(self.map_fd,
                        ct.cast(key_p, ct.c_void_p))
                if res < 0:
                    raise KeyError

        def clear(self):
            if self.ttype in (BPF.ARRAY, BPF.PROG_ARRAY):
                # Special case clear, since this class is currently behaving
                # like a dict but popitem on an array causes an infinite loop.
                # TODO: derive Table from array.array instead
                for k in self.keys():
                    self.__delitem__(k)
            else:
                super(BPF.Table, self).clear()

        @staticmethod
        def _stars(val, val_max, width):
            i = 0
            text = ""
            while (1):
                if (i > (width * val / val_max) - 1) or (i > width - 1):
                    break
                text += "*"
                i += 1
            if val > val_max:
                text = text[:-1] + "+"
            return text

        def print_log2_hist(self, val_type="value"):
            """print_log2_hist(type=value)

            Prints a table as a log2 histogram. The table must be stored as
            log2. The type argument is optional, and is a column header.
            """
            global stars_max
            log2_dist_max = 64
            idx_max = -1
            val_max = 0
            for i in range(1, log2_dist_max + 1):
                try:
                    val = self[ct.c_int(i)].value
                    if (val > 0):
                        idx_max = i
                    if (val > val_max):
                        val_max = val
                except:
                    break
            if idx_max > 0:
                print("     %-15s : count     distribution" % val_type);
            for i in range(1, idx_max + 1):
                low = (1 << i) >> 1
                high = (1 << i) - 1
                if (low == high):
                    low -= 1
                try:
                    val = self[ct.c_int(i)].value
                    print("%8d -> %-8d : %-8d |%-*s|" % (low, high, val,
                        stars_max, self._stars(val, val_max, stars_max)))
                except:
                    break


        def __iter__(self):
            return BPF.Table.Iter(self, self.Key)

        def iter(self): return self.__iter__()
        def keys(self): return self.__iter__()

        class Iter(object):
            def __init__(self, table, keytype):
                self.Key = keytype
                self.table = table
                self.key = self.Key()
            def __iter__(self):
                return self
            def __next__(self):
                return self.next()
            def next(self):
                self.key = self.table.next(self.key)
                return self.key

        def next(self, key):
            next_key = self.Key()
            next_key_p = ct.pointer(next_key)
            key_p = ct.pointer(key)
            res = lib.bpf_get_next_key(self.map_fd,
                    ct.cast(key_p, ct.c_void_p),
                    ct.cast(next_key_p, ct.c_void_p))
            if res < 0:
                raise StopIteration()
            return next_key

    @staticmethod
    def _find_file(filename):
        """ If filename is invalid, search in ./ of argv[0] """
        if filename:
            if not os.path.isfile(filename):
                t = "/".join([os.path.abspath(os.path.dirname(sys.argv[0])), filename])
                if os.path.isfile(t):
                    filename = t
                else:
                    raise Exception("Could not find file %s" % filename)
        return filename

    def __init__(self, src_file="", hdr_file="", text=None, debug=0):
        """Create a a new BPF module with the given source code.

        Note:
            All fields are marked as optional, but either `src_file` or `text`
            must be supplied, and not both.

        Args:
            src_file (Optional[str]): Path to a source file for the module
            hdr_file (Optional[str]): Path to a helper header file for the `src_file`
            text (Optional[str]): Contents of a source file for the module
            debug (Optional[int]): Flags used for debug prints, can be |'d together
                0x1: print LLVM IR to stderr
                0x2: print BPF bytecode to stderr
        """

        self.debug = debug
        self.funcs = {}
        self.tables = {}
        if text:
            self.module = lib.bpf_module_create_c_from_string(text.encode("ascii"), self.debug)
        else:
            src_file = BPF._find_file(src_file)
            hdr_file = BPF._find_file(hdr_file)
            if src_file.endswith(".b"):
                self.module = lib.bpf_module_create_b(src_file.encode("ascii"),
                        hdr_file.encode("ascii"), self.debug)
            else:
                self.module = lib.bpf_module_create_c(src_file.encode("ascii"),
                        self.debug)

        if self.module == None:
            raise Exception("Failed to compile BPF module %s" % src_file)

        # If any "kprobe__" prefixed functions were defined, they will be
        # loaded and attached here.
        self._trace_autoload()

    def load_funcs(self, prog_type=KPROBE):
        """load_funcs(prog_type=KPROBE)

        Load all functions in this BPF module with the given type.
        Returns a list of the function handles."""

        fns = []
        for i in range(0, lib.bpf_num_functions(self.module)):
            func_name = lib.bpf_function_name(self.module, i).decode()
            fns.append(self.load_func(func_name, prog_type))

        return fns

    def load_func(self, func_name, prog_type):
        if func_name in self.funcs:
            return self.funcs[func_name]

        if lib.bpf_function_start(self.module, func_name.encode("ascii")) == None:
            raise Exception("Unknown program %s" % func_name)

        log_buf = ct.create_string_buffer(65536) if self.debug else None

        fd = lib.bpf_prog_load(prog_type,
                lib.bpf_function_start(self.module, func_name.encode("ascii")),
                lib.bpf_function_size(self.module, func_name.encode("ascii")),
                lib.bpf_module_license(self.module),
                lib.bpf_module_kern_version(self.module),
                log_buf, ct.sizeof(log_buf) if log_buf else 0)

        if self.debug & 0x2:
            print(log_buf.value.decode(), file=sys.stderr)

        if fd < 0:
            raise Exception("Failed to load BPF program %s" % func_name)

        fn = BPF.Function(self, func_name, fd)
        self.funcs[func_name] = fn

        return fn

    str2ctype = {
        u"_Bool": ct.c_bool,
        u"char": ct.c_char,
        u"wchar_t": ct.c_wchar,
        u"char": ct.c_byte,
        u"unsigned char": ct.c_ubyte,<commit_msg>add ebpf method to retrieve bytecode<commit_after># Copyright 2015 PLUMgrid
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import atexit
from collections import MutableMapping
import ctypes as ct
import fcntl
import json
import os
from subprocess import Popen, PIPE
import sys
basestring = (unicode if sys.version_info[0] < 3 else str)

lib = ct.CDLL("libbcc.so")

# keep in sync with bpf_common.h
lib.bpf_module_create_b.restype = ct.c_void_p
lib.bpf_module_create_b.argtypes = [ct.c_char_p, ct.c_char_p, ct.c_uint]
lib.bpf_module_create_c.restype = ct.c_void_p
lib.bpf_module_create_c.argtypes = [ct.c_char_p, ct.c_uint]
lib.bpf_module_create_c_from_string.restype = ct.c_void_p
lib.bpf_module_create_c_from_string.argtypes = [ct.c_char_p, ct.c_uint]
lib.bpf_module_destroy.restype = None
lib.bpf_module_destroy.argtypes = [ct.c_void_p]
lib.bpf_module_license.restype = ct.c_char_p
lib.bpf_module_license.argtypes = [ct.c_void_p]
lib.bpf_module_kern_version.restype = ct.c_uint
lib.bpf_module_kern_version.argtypes = [ct.c_void_p]
lib.bpf_num_functions.restype = ct.c_ulonglong
lib.bpf_num_functions.argtypes = [ct.c_void_p]
lib.bpf_function_name.restype = ct.c_char_p
lib.bpf_function_name.argtypes = [ct.c_void_p, ct.c_ulonglong]
lib.bpf_function_start.restype = ct.c_void_p
lib.bpf_function_start.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_function_size.restype = ct.c_size_t
lib.bpf_function_size.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_table_id.restype = ct.c_ulonglong
lib.bpf_table_id.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_table_fd.restype = ct.c_int
lib.bpf_table_fd.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_table_type_id.restype = ct.c_int
lib.bpf_table_type_id.argtypes = [ct.c_void_p, ct.c_ulonglong]
lib.bpf_table_key_desc.restype = ct.c_char_p
lib.bpf_table_key_desc.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_table_leaf_desc.restype = ct.c_char_p
lib.bpf_table_leaf_desc.argtypes = [ct.c_void_p, ct.c_char_p]
lib.bpf_table_key_snprintf.restype = ct.c_int
lib.bpf_table_key_snprintf.argtypes = [ct.c_void_p, ct.c_ulonglong,
        ct.c_char_p, ct.c_ulonglong, ct.c_void_p]
lib.bpf_table_leaf_snprintf.restype = ct.c_int
lib.bpf_table_leaf_snprintf.argtypes = [ct.c_void_p, ct.c_ulonglong,
        ct.c_char_p, ct.c_ulonglong, ct.c_void_p]
lib.bpf_table_key_sscanf.restype = ct.c_int
lib.bpf_table_key_sscanf.argtypes = [ct.c_void_p, ct.c_ulonglong,
        ct.c_char_p, ct.c_void_p]
lib.bpf_table_leaf_sscanf.restype = ct.c_int
lib.bpf_table_leaf_sscanf.argtypes = [ct.c_void_p, ct.c_ulonglong,
        ct.c_char_p, ct.c_void_p]

# keep in sync with libbpf.h
lib.bpf_get_next_key.restype = ct.c_int
lib.bpf_get_next_key.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p]
lib.bpf_lookup_elem.restype = ct.c_int
lib.bpf_lookup_elem.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p]
lib.bpf_update_elem.restype = ct.c_int
lib.bpf_update_elem.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p,
        ct.c_ulonglong]
lib.bpf_delete_elem.restype = ct.c_int
lib.bpf_delete_elem.argtypes = [ct.c_int, ct.c_void_p]
lib.bpf_open_raw_sock.restype = ct.c_int
lib.bpf_open_raw_sock.argtypes = [ct.c_char_p]
lib.bpf_attach_socket.restype = ct.c_int
lib.bpf_attach_socket.argtypes = [ct.c_int, ct.c_int]
lib.bpf_prog_load.restype = ct.c_int
lib.bpf_prog_load.argtypes = [ct.c_int, ct.c_void_p, ct.c_size_t,
        ct.c_char_p, ct.c_uint, ct.c_char_p, ct.c_uint]
lib.bpf_attach_kprobe.restype = ct.c_int
lib.bpf_attach_kprobe.argtypes = [ct.c_int, ct.c_char_p,
        ct.c_char_p, ct.c_int, ct.c_int, ct.c_int]
lib.bpf_detach_kprobe.restype = ct.c_int
lib.bpf_detach_kprobe.argtypes = [ct.c_char_p]

open_kprobes = {}
tracefile = None
TRACEFS = "/sys/kernel/debug/tracing"
KALLSYMS = "/proc/kallsyms"
ksym_addrs = []
ksym_names = []
ksym_loaded = 0
stars_max = 38

@atexit.register
def cleanup_kprobes():
    for k, v in open_kprobes.items():
        os.close(v)
        desc = "-:kprobes/%s" % k
        lib.bpf_detach_kprobe(desc.encode("ascii"))
    if tracefile:
        tracefile.close()

class BPF(object):
    SOCKET_FILTER = 1
    KPROBE = 2
    SCHED_CLS = 3
    SCHED_ACT = 4

    HASH = 1
    ARRAY = 2
    PROG_ARRAY = 3

    class Function(object):
        def __init__(self, bpf, name, fd):
            self.bpf = bpf
            self.name = name
            self.fd = fd

    class Table(MutableMapping):
        def __init__(self, bpf, map_id, map_fd, keytype, leaftype):
            self.bpf = bpf
            self.map_id = map_id
            self.map_fd = map_fd
            self.Key = keytype
            self.Leaf = leaftype
            self.ttype = lib.bpf_table_type_id(self.bpf.module, self.map_id)

        def key_sprintf(self, key):
            key_p = ct.pointer(key)
            buf = ct.create_string_buffer(ct.sizeof(self.Key) * 8)
            res = lib.bpf_table_key_snprintf(self.bpf.module, self.map_id,
                    buf, len(buf), key_p)
            if res < 0:
                raise Exception("Could not printf key")
            return buf.value

        def leaf_sprintf(self, leaf):
            leaf_p = ct.pointer(leaf)
            buf = ct.create_string_buffer(ct.sizeof(self.Leaf) * 8)
            res = lib.bpf_table_leaf_snprintf(self.bpf.module, self.map_id,
                    buf, len(buf), leaf_p)
            if res < 0:
                raise Exception("Could not printf leaf")
            return buf.value

        def key_scanf(self, key_str):
            key = self.Key()
            key_p = ct.pointer(key)
            res = lib.bpf_table_key_sscanf(self.bpf.module, self.map_id,
                    key_str, key_p)
            if res < 0:
                raise Exception("Could not scanf key")
            return key

        def leaf_scanf(self, leaf_str):
            leaf = self.Leaf()
            leaf_p = ct.pointer(leaf)
            res = lib.bpf_table_leaf_sscanf(self.bpf.module, self.map_id,
                    leaf_str, leaf_p)
            if res < 0:
                raise Exception("Could not scanf leaf")
            return leaf

        def __getitem__(self, key):
            key_p = ct.pointer(key)
            leaf = self.Leaf()
            leaf_p = ct.pointer(leaf)
            res = lib.bpf_lookup_elem(self.map_fd,
                    ct.cast(key_p, ct.c_void_p),
                    ct.cast(leaf_p, ct.c_void_p))
            if res < 0:
                raise KeyError
            return leaf

        def __setitem__(self, key, leaf):
            key_p = ct.pointer(key)
            leaf_p = ct.pointer(leaf)
            res = lib.bpf_update_elem(self.map_fd,
                    ct.cast(key_p, ct.c_void_p),
                    ct.cast(leaf_p, ct.c_void_p), 0)
            if res < 0:
                raise Exception("Could not update table")

        def __len__(self):
            i = 0
            for k in self: i += 1
            return i

        def __delitem__(self, key):
            key_p = ct.pointer(key)
            ttype = lib.bpf_table_type_id(self.bpf.module, self.map_id)
            # Deleting from array type maps does not have an effect, so
            # zero out the entry instead.
            if ttype in (BPF.ARRAY, BPF.PROG_ARRAY):
                leaf = self.Leaf()
                leaf_p = ct.pointer(leaf)
                res = lib.bpf_update_elem(self.map_fd,
                        ct.cast(key_p, ct.c_void_p),
                        ct.cast(leaf_p, ct.c_void_p), 0)
                if res < 0:
                    raise Exception("Could not clear item")
            else:
                res = lib.bpf_delete_elem(self.map_fd,
                        ct.cast(key_p, ct.c_void_p))
                if res < 0:
                    raise KeyError

        def clear(self):
            if self.ttype in (BPF.ARRAY, BPF.PROG_ARRAY):
                # Special case clear, since this class is currently behaving
                # like a dict but popitem on an array causes an infinite loop.
                # TODO: derive Table from array.array instead
                for k in self.keys():
                    self.__delitem__(k)
            else:
                super(BPF.Table, self).clear()

        @staticmethod
        def _stars(val, val_max, width):
            i = 0
            text = ""
            while (1):
                if (i > (width * val / val_max) - 1) or (i > width - 1):
                    break
                text += "*"
                i += 1
            if val > val_max:
                text = text[:-1] + "+"
            return text

        def print_log2_hist(self, val_type="value"):
            """print_log2_hist(type=value)

            Prints a table as a log2 histogram. The table must be stored as
            log2. The type argument is optional, and is a column header.
            """
            global stars_max
            log2_dist_max = 64
            idx_max = -1
            val_max = 0
            for i in range(1, log2_dist_max + 1):
                try:
                    val = self[ct.c_int(i)].value
                    if (val > 0):
                        idx_max = i
                    if (val > val_max):
                        val_max = val
                except:
                    break
            if idx_max > 0:
                print("     %-15s : count     distribution" % val_type);
            for i in range(1, idx_max + 1):
                low = (1 << i) >> 1
                high = (1 << i) - 1
                if (low == high):
                    low -= 1
                try:
                    val = self[ct.c_int(i)].value
                    print("%8d -> %-8d : %-8d |%-*s|" % (low, high, val,
                        stars_max, self._stars(val, val_max, stars_max)))
                except:
                    break


        def __iter__(self):
            return BPF.Table.Iter(self, self.Key)

        def iter(self): return self.__iter__()
        def keys(self): return self.__iter__()

        class Iter(object):
            def __init__(self, table, keytype):
                self.Key = keytype
                self.table = table
                self.key = self.Key()
            def __iter__(self):
                return self
            def __next__(self):
                return self.next()
            def next(self):
                self.key = self.table.next(self.key)
                return self.key

        def next(self, key):
            next_key = self.Key()
            next_key_p = ct.pointer(next_key)
            key_p = ct.pointer(key)
            res = lib.bpf_get_next_key(self.map_fd,
                    ct.cast(key_p, ct.c_void_p),
                    ct.cast(next_key_p, ct.c_void_p))
            if res < 0:
                raise StopIteration()
            return next_key

    @staticmethod
    def _find_file(filename):
        """ If filename is invalid, search in ./ of argv[0] """
        if filename:
            if not os.path.isfile(filename):
                t = "/".join([os.path.abspath(os.path.dirname(sys.argv[0])), filename])
                if os.path.isfile(t):
                    filename = t
                else:
                    raise Exception("Could not find file %s" % filename)
        return filename

    def __init__(self, src_file="", hdr_file="", text=None, debug=0):
        """Create a a new BPF module with the given source code.

        Note:
            All fields are marked as optional, but either `src_file` or `text`
            must be supplied, and not both.

        Args:
            src_file (Optional[str]): Path to a source file for the module
            hdr_file (Optional[str]): Path to a helper header file for the `src_file`
            text (Optional[str]): Contents of a source file for the module
            debug (Optional[int]): Flags used for debug prints, can be |'d together
                0x1: print LLVM IR to stderr
                0x2: print BPF bytecode to stderr
        """

        self.debug = debug
        self.funcs = {}
        self.tables = {}
        if text:
            self.module = lib.bpf_module_create_c_from_string(text.encode("ascii"), self.debug)
        else:
            src_file = BPF._find_file(src_file)
            hdr_file = BPF._find_file(hdr_file)
            if src_file.endswith(".b"):
                self.module = lib.bpf_module_create_b(src_file.encode("ascii"),
                        hdr_file.encode("ascii"), self.debug)
            else:
                self.module = lib.bpf_module_create_c(src_file.encode("ascii"),
                        self.debug)

        if self.module == None:
            raise Exception("Failed to compile BPF module %s" % src_file)

        # If any "kprobe__" prefixed functions were defined, they will be
        # loaded and attached here.
        self._trace_autoload()

    def load_funcs(self, prog_type=KPROBE):
        """load_funcs(prog_type=KPROBE)

        Load all functions in this BPF module with the given type.
        Returns a list of the function handles."""

        fns = []
        for i in range(0, lib.bpf_num_functions(self.module)):
            func_name = lib.bpf_function_name(self.module, i).decode()
            fns.append(self.load_func(func_name, prog_type))

        return fns

    def load_func(self, func_name, prog_type):
        if func_name in self.funcs:
            return self.funcs[func_name]

        if lib.bpf_function_start(self.module, func_name.encode("ascii")) == None:
            raise Exception("Unknown program %s" % func_name)

        log_buf = ct.create_string_buffer(65536) if self.debug else None

        fd = lib.bpf_prog_load(prog_type,
                lib.bpf_function_start(self.module, func_name.encode("ascii")),
                lib.bpf_function_size(self.module, func_name.encode("ascii")),
                lib.bpf_module_license(self.module),
                lib.bpf_module_kern_version(self.module),
                log_buf, ct.sizeof(log_buf) if log_buf else 0)

        if self.debug & 0x2:
            print(log_buf.value.decode(), file=sys.stderr)

        if fd < 0:
            raise Exception("Failed to load BPF program %s" % func_name)

        fn = BPF.Function(self, func_name, fd)
        self.funcs[func_name] = fn

        return fn

    def ebpf(self, func_name):
        """
        Return the eBPF bytecodes for the specified function as a string
        """
        if lib.bpf_function_start(self.module, func_name.encode("ascii")) == None:
            raise Exception("Unknown program %s" % func_name)

        start, = lib.bpf_function_start(self.module, func_name.encode("ascii")),
        size, = lib.bpf_function_size(self.module, func_name.encode("ascii")),
        return ct.string_at(start, size)

    str2ctype = {
        u"_Bool": ct.c_bool,
        u"char": ct.c_char,
        u"wchar_t": ct.c_wchar,
        u"char": ct.c_byte,
        u"unsigned char": ct.c_ubyte,
{'fraction_non_alphanumeric': 0.10057803468208093, 'fraction_numerical': 0.005615194054500413, 'mean_word_length': 1.9705651491365777, 'pattern_counts': {'":': 10, '<': 33, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4981146', 'n_tokens_mistral': 10807, 'n_tokens_neox': 10149, 'n_words': 2216}