<commit_before>#!/usr/bin/env python
"""Dump CPU affinity information as a tree."""


import sys
import collections
import re
import json


ORDER = {"physical id": 0, "core id": 1, "processor": 2}
PATTERN = "^(" + "|".join(ORDER.keys()) + ")"
FILTER_RE = re.compile(PATTERN)
ISLEAF = 0b001
ISFIRST = 0b010
ISLAST = 0b100


class DictTree(object):
    """Tree stucture implemented using dict. No need for a Node() class, but
    each key must be hashable, and sibling of the same level must be unique.
    The order of siblings is lost.
    """

    def __init__(self):
        """Create root node."""
        self._root = {}
        return None

    def add(self, inputlist):
        """Add the nodes specified in inputlist to the tree.The nth element of
        inputlist is added in the nth level.
        """
        _add(self._root, inputlist)
        return None

    def dumpjson(self):
        """Return a string dump in JSON format."""
        return json.dumps(self._root, sort_keys=True)

    def dumpascii(self, *args, **kwargs):
        """Return a string dump of an ascii drawing of the tree."""
        return _dumpascii(_tokenize(self._root, *args, **kwargs))


class CpuTree(DictTree):
    """Tree representing the CPUs found in /proc/cpuinfo."""

    def __init__(self):
        super(CpuTree, self).__init__()
        for cpu in _part3(_filter_cpuinfo()):
            self.add(cpu)
        self._sortkey = lambda x: int(x[0].rsplit(":", 1)[1].strip())
        return None

    def dumpascii(self):
        return super(CpuTree, self).dumpascii(self._sortkey)


def _add(node, inputlist):
    """Add elements in inputlist to the tree, rooted at given node.  Leaf nodes
    have an empty dictionary as child.  The nth element of inputlist is added
    to the nth level descending from node.
    """
    try:
        head = inputlist[0]
    except IndexError:
        return None
    tail = inputlist[1:]
    nextnode = node.setdefault(head, {})
    _add(nextnode, tail)


def _tokenize(node, sortkey=None):
    """Generator that traverses a tree in pre-order from node.
    Each yielded value is a tuple of (nodekey, depth, flags, symbol).
    flags is a bitwise-or'ed mask of following bits:
        0b001: node is a leaf
        0b010: node is first among siblings
        0b100: node is last among siblings (can be set with 0b010).
    """
    trackstack = []
    this = node
    depth = 0
    yield "*", 0, (ISFIRST | ISLAST)  # root
    while trackstack or this:
        children = this.items()
        if children:
            children.sort(key=sortkey, reverse=True)
            numchi = len(children)
            chiflags = [0] * numchi
            chiflags[0] |= ISLAST
            chiflags[-1] |= ISFIRST
            depth += 1
            tmp = children[-1]  # first child
            thisflag = chiflags[-1]
            tail = children[:-1]
            trackstack.extend(zip(tail,
                                  [depth] * (numchi - 1),
                                  chiflags[:-1]))
        else:
            tmp, depth, thisflag = trackstack.pop()  # Load depth and flags.
        yield tmp[0], depth, thisflag | (ISLEAF if not tmp[1] else 0)
        this = tmp[1]   # Advance to next.


def _symbol(label, flags, taillen=1):
    """Generate drawing symbol using a token's label and flags."""
    tail = "\n" if (flags & ISLEAF) else "-" * taillen
    isonlychild = (flags & ISFIRST) and (flags & ISLAST)
    if isonlychild:
        stem = "-"
    elif flags & ISFIRST:
        stem = "+"
    elif flags & ISLAST:
        stem = "`"
    else:
        stem = "|"
    return "%s-%s%s" % (stem, label, tail)


def _dumplines(tokenstream):
    """Draw using the input tokenstream iterator.  Return a list containing
    separate lines of the drawing.
    """
    text = []
    columnpos = collections.defaultdict(lambda: 0)
    for token in tokenstream:
        label, depth, flags = token
        symbol = _symbol(label, flags)
        if flags & ISFIRST:
            # need to set position for next column
            columnpos[depth + 1] = columnpos[depth] + len(symbol)
            indentstring = ""
        else:  # need indent
            indent = []
            for i in xrange(depth):
                indent.extend([" "] * (columnpos[i + 1] - columnpos[i]))
            indentstring = "".join(indent)
        text.append("%s%s" % (indentstring, symbol))
    return ("".join(text)).splitlines(True)


def _fudge_vertical_bars(lines):
    """Draw vertical bars to connect lines.  Return the "fudged" lines
    joined together as a string.
    """
    for lineno, line in enumerate(lines):
        vbidx = [i for (i, letter) in enumerate(line) if letter in ("|", "+")]
        nxlineno = lineno + 1
        try:    # try peeking next line
            nxline = lines[nxlineno]
        except IndexError:<commit_msg>Simplify indent width reckoning<commit_after>#!/usr/bin/env python
"""Dump CPU affinity information as a tree."""


import sys
import collections
import re
import json


ORDER = {"physical id": 0, "core id": 1, "processor": 2}
PATTERN = "^(" + "|".join(ORDER.keys()) + ")"
FILTER_RE = re.compile(PATTERN)
ISLEAF = 0b001
ISFIRST = 0b010
ISLAST = 0b100


class DictTree(object):
    """Tree stucture implemented using dict. No need for a Node() class, but
    each key must be hashable, and sibling of the same level must be unique.
    The order of siblings is lost.
    """

    def __init__(self):
        """Create root node."""
        self._root = {}
        return None

    def add(self, inputlist):
        """Add the nodes specified in inputlist to the tree.The nth element of
        inputlist is added in the nth level.
        """
        _add(self._root, inputlist)
        return None

    def dumpjson(self):
        """Return a string dump in JSON format."""
        return json.dumps(self._root, sort_keys=True)

    def dumpascii(self, *args, **kwargs):
        """Return a string dump of an ascii drawing of the tree."""
        return _dumpascii(_tokenize(self._root, *args, **kwargs))


class CpuTree(DictTree):
    """Tree representing the CPUs found in /proc/cpuinfo."""

    def __init__(self):
        super(CpuTree, self).__init__()
        for cpu in _part3(_filter_cpuinfo()):
            self.add(cpu)
        self._sortkey = lambda x: int(x[0].rsplit(":", 1)[1].strip())
        return None

    def dumpascii(self):
        return super(CpuTree, self).dumpascii(self._sortkey)


def _add(node, inputlist):
    """Add elements in inputlist to the tree, rooted at given node.  Leaf nodes
    have an empty dictionary as child.  The nth element of inputlist is added
    to the nth level descending from node.
    """
    try:
        head = inputlist[0]
    except IndexError:
        return None
    tail = inputlist[1:]
    nextnode = node.setdefault(head, {})
    _add(nextnode, tail)


def _tokenize(node, sortkey=None):
    """Generator that traverses a tree in pre-order from node.
    Each yielded value is a tuple of (nodekey, depth, flags, symbol).
    flags is a bitwise-or'ed mask of following bits:
        0b001: node is a leaf
        0b010: node is first among siblings
        0b100: node is last among siblings (can be set with 0b010).
    """
    trackstack = []
    this = node
    depth = 0
    yield "*", 0, (ISFIRST | ISLAST)  # root
    while trackstack or this:
        children = this.items()
        if children:
            children.sort(key=sortkey, reverse=True)
            numchi = len(children)
            chiflags = [0] * numchi
            chiflags[0] |= ISLAST
            chiflags[-1] |= ISFIRST
            depth += 1
            tmp = children[-1]  # first child
            thisflag = chiflags[-1]
            tail = children[:-1]
            trackstack.extend(zip(tail,
                                  [depth] * (numchi - 1),
                                  chiflags[:-1]))
        else:
            tmp, depth, thisflag = trackstack.pop()  # Load depth and flags.
        yield tmp[0], depth, thisflag | (ISLEAF if not tmp[1] else 0)
        this = tmp[1]   # Advance to next.


def _symbol(label, flags, taillen=1):
    """Generate drawing symbol using a token's label and flags."""
    tail = "\n" if (flags & ISLEAF) else "-" * taillen
    isonlychild = (flags & ISFIRST) and (flags & ISLAST)
    if isonlychild:
        stem = "-"
    elif flags & ISFIRST:
        stem = "+"
    elif flags & ISLAST:
        stem = "`"
    else:
        stem = "|"
    return "%s-%s%s" % (stem, label, tail)


def _dumplines(tokenstream):
    """Draw using the input tokenstream iterator.  Return a list containing
    separate lines of the drawing.
    """
    text = []
    columnwidths = collections.defaultdict(lambda: 0)
    for token in tokenstream:
        label, depth, flags = token
        symbol = _symbol(label, flags)
        if flags & ISFIRST:
            # need to set position for next column
            columnwidths[depth] = len(symbol)
            indentstring = ""
        else:  # need indent
            indent = []
            for i in xrange(depth):
                indent.extend([" "] * columnwidths[i])
            indentstring = "".join(indent)
        text.append("%s%s" % (indentstring, symbol))
    return ("".join(text)).splitlines(True)


def _fudge_vertical_bars(lines):
    """Draw vertical bars to connect lines.  Return the "fudged" lines
    joined together as a string.
    """
    for lineno, line in enumerate(lines):
        vbidx = [i for (i, letter) in enumerate(line) if letter in ("|", "+")]
        nxlineno = lineno + 1
        try:    # try peeking next line
            nxline = lines[nxlineno]
        except IndexError:
{'fraction_non_alphanumeric': 0.11442734689662294, 'fraction_numerical': 0.011566663224207374, 'mean_word_length': 2.2119402985074625, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1002785', 'n_tokens_mistral': 3025, 'n_tokens_neox': 2868, 'n_words': 976}