<commit_before>#!/usr/bin/env python
# -*- coding:UTF-8 -*-
"""Find the position of the kernel in memory

The following locations are used.

To read live kernel position:
* /proc/iomem, which contains "Kernel code", "Kernel data" and "Kernel bss".
* /proc/kallsyms, which contains the current position of symbols
  but may be unavailable for security reasons.

To read static kernel symbols:
* nm /lib/modules/$(uname -r)/build/vmlinux, when vmlinux file is available.
* /boot/System.map-$(uname -r), which contains the result of the previous
  command at build time.

It is also possible to extract vmlinux from an installed vmlinux file with
https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/scripts/extract-vmlinux
and then use "readelf -S" to read the section header of the extracted kernel to
get the static location of .text, .data and .bss segments.
"""
import collections
import errno
import itertools
import logging
import os.path
import platform
import re
import struct
import subprocess
import sys


if sys.version_info < (2, 7):
    sys.stderr.write("This program cannot be run in Python<2.7 mode.\n")
    sys.exit(0)

logger = logging.getLogger(__name__)


# Internal name of kernel sections with symbols defining the start and end
KERNEL_SECTIONS = collections.OrderedDict((
    ('code', ('_text', '_etext')),
    ('data', ('_sdata', '_edata')),
    ('bss', ('__bss_start', '__bss_stop')),
))
KERNEL_SYMNAMES = list(itertools.chain(*KERNEL_SECTIONS.values()))


def symbols2positions(symbols):
    """Convert a symbols dict to section postions"""
    def dec_or_none(value):
        return value - 1 if value is not None else None
    return dict((
        (name, (symbols[syms[0]], dec_or_none(symbols[syms[1]])))
        for name, syms in KERNEL_SECTIONS.items()))


def show_positions_diff(pos1, pos2):
    """Show the differences between positions collected by 2 ways"""
    for symname in KERNEL_SECTIONS.keys():
        print("  {}:".format(symname))
        for endidx, endname in enumerate(('start', 'end  ')):
            p1 = pos1[symname][endidx]
            p2 = pos2[symname][endidx]
            if p1 is None and p2 is None:
                print("    {}: ?".format(endname))
            elif p1 is None:
                print("    {}: ? - {:x}".format(endname, p2))
            elif p2 is None:
                print("    {}: {:x} - ?".format(endname, p1))
            else:
                print("    {}: {:x} - {:x} ({}{:#x})".format(
                    endname, p1, p2, '+' if p2 > p1 else '', p2 - p1))


def get_pa2va_offset(machine=None):
    """Get the physical address to kernel virtual address offset for the
    specified architecture
    """
    if machine is None:
        machine = platform.machine()

    if re.match(r'armv[1-9]+l$', machine):
        # https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/arm/memory.txt
        return 0xc0000000  # PAGE_OFFSET
    elif re.match(r'i[3-6]86$', machine):
        return 0xc0000000  # CONFIG_PAGE_OFFSET default value
    elif machine == 'x86_64':
        # https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/x86/x86_64/mm.txt
        return 0xffffffff80000000  # __START_KERNEL_map

    logger.warning("Unknown pa2va offset for %s", machine)
    return 0


def get_kernel_pos_from_iomem(machine=None):
    """Get the position of some kernel segments using /proc/iomem"""
    if not os.path.exists('/proc/iomem'):
        logger.warning("/proc/iomem does not exist")
        return
    logger.debug("reading live symbols from /proc/iomem")
    offset = get_pa2va_offset(machine)
    kernel_pos = dict(((name, None) for name in KERNEL_SECTIONS.keys()))
    with open('/proc/iomem', 'r') as fiomem:
        for line in fiomem:
            line = line.strip()
            matches = re.match(r'([0-9a-f]+)-([0-9a-f]+) : Kernel (.*)$', line)
            if matches is None:
                continue
            start, end, name = matches.groups()
            if name not in kernel_pos:
                logger.warning("unknown kernel part %s in /proc/iomem", name)
                continue
            kernel_pos[name] = (int(start, 16) + offset, int(end, 16) + offset)

    # Some architectures (like ARM) merge data and bss segments
    if kernel_pos['bss'] is None and kernel_pos['data'] is not None:
        kernel_pos['bss'] = (None, kernel_pos['data'][1])
        kernel_pos['data'] = (kernel_pos['data'][0], None)

    if None in kernel_pos.values():
        notfound = [name for name, addr in kernel_pos.items() if addr is None]
        logger.warning("unable to find kernel %s in /proc/iomem",
                       ', '.join(notfound))
        for name in notfound:
            kernel_pos[name] = (None, None)
    return kernel_pos


def get_pos_symbols_from_systemmap(path=None):
    """Get the symbols from a System.map file"""
    if path is None:
        path = '/boot/System.map-{}'.format(platform.release())
        if not os.path.exists(path):
            return
    logger.debug("reading symbols from %s", path)
    symbols = dict(((name, None) for name in KERNEL_SYMNAMES))
    try:
        with open(path, 'r') as fsysmap:
            for line in fsysmap:
                matches = re.match(r'([0-9a-f]+) . ([0-9a-zA-Z_]+)', line)
                if matches is None:<commit_msg>linux/kernel_position: recover gracefully when access to /proc/iomem is denied<commit_after>#!/usr/bin/env python
# -*- coding:UTF-8 -*-
"""Find the position of the kernel in memory

The following locations are used.

To read live kernel position:
* /proc/iomem, which contains "Kernel code", "Kernel data" and "Kernel bss".
* /proc/kallsyms, which contains the current position of symbols
  but may be unavailable for security reasons.

To read static kernel symbols:
* nm /lib/modules/$(uname -r)/build/vmlinux, when vmlinux file is available.
* /boot/System.map-$(uname -r), which contains the result of the previous
  command at build time.

It is also possible to extract vmlinux from an installed vmlinux file with
https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/scripts/extract-vmlinux
and then use "readelf -S" to read the section header of the extracted kernel to
get the static location of .text, .data and .bss segments.
"""
import collections
import errno
import itertools
import logging
import os.path
import platform
import re
import struct
import subprocess
import sys


if sys.version_info < (2, 7):
    sys.stderr.write("This program cannot be run in Python<2.7 mode.\n")
    sys.exit(0)

logger = logging.getLogger(__name__)


# Internal name of kernel sections with symbols defining the start and end
KERNEL_SECTIONS = collections.OrderedDict((
    ('code', ('_text', '_etext')),
    ('data', ('_sdata', '_edata')),
    ('bss', ('__bss_start', '__bss_stop')),
))
KERNEL_SYMNAMES = list(itertools.chain(*KERNEL_SECTIONS.values()))


def symbols2positions(symbols):
    """Convert a symbols dict to section postions"""
    def dec_or_none(value):
        return value - 1 if value is not None else None
    return dict((
        (name, (symbols[syms[0]], dec_or_none(symbols[syms[1]])))
        for name, syms in KERNEL_SECTIONS.items()))


def show_positions_diff(pos1, pos2):
    """Show the differences between positions collected by 2 ways"""
    for symname in KERNEL_SECTIONS.keys():
        print("  {}:".format(symname))
        for endidx, endname in enumerate(('start', 'end  ')):
            p1 = pos1[symname][endidx]
            p2 = pos2[symname][endidx]
            if p1 is None and p2 is None:
                print("    {}: ?".format(endname))
            elif p1 is None:
                print("    {}: ? - {:x}".format(endname, p2))
            elif p2 is None:
                print("    {}: {:x} - ?".format(endname, p1))
            else:
                print("    {}: {:x} - {:x} ({}{:#x})".format(
                    endname, p1, p2, '+' if p2 > p1 else '', p2 - p1))


def get_pa2va_offset(machine=None):
    """Get the physical address to kernel virtual address offset for the
    specified architecture
    """
    if machine is None:
        machine = platform.machine()

    if re.match(r'armv[1-9]+l$', machine):
        # https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/arm/memory.txt
        return 0xc0000000  # PAGE_OFFSET
    elif re.match(r'i[3-6]86$', machine):
        return 0xc0000000  # CONFIG_PAGE_OFFSET default value
    elif machine == 'x86_64':
        # https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/x86/x86_64/mm.txt
        return 0xffffffff80000000  # __START_KERNEL_map

    logger.warning("Unknown pa2va offset for %s", machine)
    return 0


def get_kernel_pos_from_iomem(machine=None):
    """Get the position of some kernel segments using /proc/iomem"""
    if not os.path.exists('/proc/iomem'):
        logger.warning("/proc/iomem does not exist")
        return
    logger.debug("reading live symbols from /proc/iomem")
    offset = get_pa2va_offset(machine)
    kernel_pos = dict(((name, None) for name in KERNEL_SECTIONS.keys()))
    try:
        with open('/proc/iomem', 'r') as fiomem:
            for line in fiomem:
                line = line.strip()
                matches = re.match(r'([0-9a-f]+)-([0-9a-f]+) : Kernel (.*)$',
                                   line)
                if matches is None:
                    continue
                start, end, name = matches.groups()
                if name not in kernel_pos:
                    logger.warning("unknown kernel part %s in /proc/iomem",
                                   name)
                    continue
                kernel_pos[name] = (int(start, 16) + offset,
                                    int(end, 16) + offset)
    except IOError as exc:
        if exc.errno == errno.EACCES:
            logger.warning("/proc/iomem access is not permitted")
            return
        raise

    # Some architectures (like ARM) merge data and bss segments
    if kernel_pos['bss'] is None and kernel_pos['data'] is not None:
        kernel_pos['bss'] = (None, kernel_pos['data'][1])
        kernel_pos['data'] = (kernel_pos['data'][0], None)

    if None in kernel_pos.values():
        notfound = [name for name, addr in kernel_pos.items() if addr is None]
        logger.warning("unable to find kernel %s in /proc/iomem",
                       ', '.join(notfound))
        for name in notfound:
            kernel_pos[name] = (None, None)
    return kernel_pos


def get_pos_symbols_from_systemmap(path=None):
    """Get the symbols from a System.map file"""
    if path is None:
        path = '/boot/System.map-{}'.format(platform.release())
        if not os.path.exists(path):
            return
    logger.debug("reading symbols from %s", path)
    symbols = dict(((name, None) for name in KERNEL_SYMNAMES))
    try:
        with open(path, 'r') as fsysmap:
            for line in fsysmap:
                matches = re.match(r'([0-9a-f]+) . ([0-9a-zA-Z_]+)', line)
                if matches is None:
{'fraction_non_alphanumeric': 0.12747213232650126, 'fraction_numerical': 0.015821646889608056, 'mean_word_length': 2.5691369906961823, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2325966', 'n_tokens_mistral': 3568, 'n_tokens_neox': 3405, 'n_words': 1028}