<commit_before>#!/usr/bin/env python

import os
import re

from collections import defaultdict
from subprocess import Popen, PIPE

from tenderloin.plugin import TenderloinPlugin


class BasePlugin(TenderloinPlugin):
    def __init__(self, *args, **kwargs):
        self.prev_cpu = {}
        self.prev_diskstats = {}
        super(BasePlugin, self).__init__(*args, **kwargs)

    def fmt(self, f):
        return "%.2f" % f

    def rateof(self, a, b):
        a = float(a)
        b = float(b)

        try:
            return (b - a) / self.interval if (b - a) != 0 else 0
        except ZeroDivisionError:
            if a:
                return -a
            return b

    def get_cpu(self):
        cpudata = defaultdict(dict)
        cur_cpu = {}
        to_read = "/proc/stat"
        wanted = [
            "user",
            "nice",
            "system",
            "idle",
            "iowait",
            "irq",
            "softirq"
        ]

        if os.path.exists(to_read):
            jiffy = os.sysconf(os.sysconf_names["SC_CLK_TCK"])

            with open(to_read, "r") as f:
                for line in f:
                    if line.startswith("cpu"):
                        fields = line.split()
                        cur_cpu[fields[0]] = fields[1:len(wanted) + 1]
                    else:
                        break

            if self.prev_cpu:
                for cpu in cur_cpu:
                    if cpu in self.prev_cpu:
                        for fld_idx in xrange(len(wanted) + 1):
                            try:
                                prev = float(self.prev_cpu[cpu][fld_idx])
                                cur = float(cur_cpu[cpu][fld_idx])
                                cpudata[cpu][wanted[fld_idx]] = ((cur - prev) / self.interval * 100 / jiffy)
                            except ZeroDivisionError:
                                if prev:
                                    cpudata[cpu][wanted[fld_idx]] = -prev
                                else:
                                    cpudata[cpu][wanted[fld_idx]] = cur
                            except IndexError:
                                continue
                self["cpu"] = cpudata
            self.prev_cpu = cur_cpu
        else:
            self["cpu"] = {}

    def get_diskstats(self):
        diskstats = defaultdict(dict)
        cur_diskstats = defaultdict(dict)
        to_read = "/proc/diskstats"

        if os.path.exists(to_read):
            stat_fields = [
                "rd",
                "rd_m",
                "rd_s",
                "rd_t",
                "wr",
                "wr_m",
                "wr_s",
                "wr_t",
                "running",
                "use",
                "aveq"
            ]
            r1 = re.compile("[a-z]d[a-z]$")

            with open(to_read, "r") as f:
                for line in f:
                    fields = line.strip().split()
                    device = fields[2]
                    fields = fields[3:]
<commit_msg>support lvm devices in diskstats<commit_after>#!/usr/bin/env python

import os
import re

from collections import defaultdict
from subprocess import Popen, PIPE

from tenderloin.plugin import TenderloinPlugin


class BasePlugin(TenderloinPlugin):
    def __init__(self, *args, **kwargs):
        self.prev_cpu = {}
        self.prev_diskstats = {}
        super(BasePlugin, self).__init__(*args, **kwargs)

    def fmt(self, f):
        return "%.2f" % f

    def rateof(self, a, b):
        a = float(a)
        b = float(b)

        try:
            return (b - a) / self.interval if (b - a) != 0 else 0
        except ZeroDivisionError:
            if a:
                return -a
            return b

    def get_cpu(self):
        cpudata = defaultdict(dict)
        cur_cpu = {}
        to_read = "/proc/stat"
        wanted = [
            "user",
            "nice",
            "system",
            "idle",
            "iowait",
            "irq",
            "softirq"
        ]

        if os.path.exists(to_read):
            jiffy = os.sysconf(os.sysconf_names["SC_CLK_TCK"])

            with open(to_read, "r") as f:
                for line in f:
                    if line.startswith("cpu"):
                        fields = line.split()
                        cur_cpu[fields[0]] = fields[1:len(wanted) + 1]
                    else:
                        break

            if self.prev_cpu:
                for cpu in cur_cpu:
                    if cpu in self.prev_cpu:
                        for fld_idx in xrange(len(wanted) + 1):
                            try:
                                prev = float(self.prev_cpu[cpu][fld_idx])
                                cur = float(cur_cpu[cpu][fld_idx])
                                cpudata[cpu][wanted[fld_idx]] = ((cur - prev) / self.interval * 100 / jiffy)
                            except ZeroDivisionError:
                                if prev:
                                    cpudata[cpu][wanted[fld_idx]] = -prev
                                else:
                                    cpudata[cpu][wanted[fld_idx]] = cur
                            except IndexError:
                                continue
                self["cpu"] = cpudata
            self.prev_cpu = cur_cpu
        else:
            self["cpu"] = {}

    def get_diskstats(self):
        diskstats = defaultdict(dict)
        cur_diskstats = defaultdict(dict)
        to_read = "/proc/diskstats"

        if os.path.exists(to_read):
            stat_fields = [
                "rd",
                "rd_m",
                "rd_s",
                "rd_t",
                "wr",
                "wr_m",
                "wr_s",
                "wr_t",
                "running",
                "use",
                "aveq"
            ]
            r1 = re.compile("^([a-z]d[a-z]|dm-[0-9]+)$")

            with open(to_read, "r") as f:
                for line in f:
                    fields = line.strip().split()
                    device = fields[2]
                    fields = fields[3:]

{'fraction_non_alphanumeric': 0.10546108567691302, 'fraction_numerical': 0.004578155657292348, 'mean_word_length': 1.0302024560238965, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4237153', 'n_tokens_mistral': 1800, 'n_tokens_neox': 1685, 'n_words': 370}