<commit_before>import os
import re
import sys
import json
from urllib2 import Request
from urllib2 import urlopen
from urlparse import urljoin
from commands import getstatusoutput
from traceback import format_exc
from collections import defaultdict

def run(cmdline, expected_status=0):
    status, output = getstatusoutput(cmdline)
    if status != expected_status:
        raise RuntimeError('Error while running {0!r}: {1}'.format(cmdline, output))
    return output

def collect_net_info():
    net_info = defaultdict(dict)
    for iface in run('ip --oneline link').splitlines():
        mo = re.match(r'^\d:\s+(?P<iface_name>\w+):.*link/(?P<iface_type>\w+)\s+(?P<iface_addr>[\w:]+).*$', iface)
        if mo is not None:
            iface_info = mo.groupdict()
            if iface_info['iface_type'] == 'loopback':
                continue
            net_info[iface_info['iface_name']]['mac'] = iface_info['iface_addr']
    for iface in run('ip --oneline addr').splitlines():
        _, iface_name, iface_info = iface.split(' ', 2)
        if iface_name == 'lo':
            continue
        iface_info = iface_info.strip().split()
        if iface_info[0] == 'inet':
            net_info[iface_name].setdefault('inet', []).append(iface_info[1])
        if iface_info[0] == 'inet6':
            net_info[iface_name].setdefault('inet6', []).append(iface_info[1])
    return [dict(dev=dev, **fields) for dev, fields in net_info.iteritems()]

def collect_pci_info():
    lspci = run('lspci -mm')
    pci_devices = []
    for line in lspci.splitlines():
        mo = re.match(r'^(?P<address>[:\w.]+)\s+"(?P<type>[^"]+)" "(?P<vendor>[^"]+)" "(?P<name>[^"]+)".*$', line)
        if mo is not None:
            pci_devices.append(mo.groupdict())
    return pci_devices

def collect_disk_info():
    lsblk = run('lsblk --bytes --nodeps --output-all --pairs')
    disks = []
    for line in lsblk.splitlines():
        fields = {}
        for mo in re.finditer(r'(?P<key>[^=]+)="(?P<value>[^"]*)"', line):
            groupdict = mo.groupdict()
            key, value = groupdict['key'].strip().lower(), groupdict['value'].strip()
            try:
                value = int(value)
            except ValueError:
                pass
            fields[key] = value
        disks.append(fields)
    return disks

def collect_cpu_info():
    cpus_info = open('/proc/cpuinfo').read()
    return [dict(map(lambda x: x.strip(), line.split(':', 1)) for line in cpu_info.splitlines() if line.strip()) for cpu_info in cpus_info.split('\n\n') if cpu_info.strip()]

MEM_KEYS = (
    'MemTotal',
    'MemFree',
    'MemAvailable',
    'SwapFree',
    'SwapFree',
    'SwapCached',
)

def collect_mem_info():
    mem_info = {}
    for line in open('/proc/meminfo').read().splitlines():
        key, value = map(lambda x: x.strip(), line.split(':', 1))
        if key not in MEM_KEYS:
            continue
        try:
            value = int(value)
        except ValueError:
            if value.endswith('kB'):
                value = int(value[:-len('kB')]) * 1024
        mem_info[key] = value
    return mem_info

def collect_fs_info():
    df = run('df --block-size=1 /')
    lines = df.splitlines()[1:]
    fields = df.splitlines()[1].split()
    return {'/': dict(
        dev             = fields[0],
        total_bytes     = int(fields[1]),
        used_bytes      = int(fields[2]),
        available_bytes = int(fields[3]),
    )}

info_collectors = dict(
    net         = collect_net_info,
    pci_devices = collect_pci_info,
    disks       = collect_disk_info,
    cpu         = collect_cpu_info,
    mem         = collect_mem_info,
    fs          = collect_fs_info,
)

def collect_info():
    info = dict(
        hostname = open('/proc/sys/kernel/hostname').read().strip(),
        hw_info  = {},
        errors   = [],
    )
    for key, collect_func in info_collectors.iteritems():
        try:
            info['hw_info'][key] = collect_func()
        except Exception as error:
            info['hw_info'][key] = None
            info['errors'].append(dict(info=key, error=str(error), traceback=format_exc()))
    return info

def send(data):
    request = Request(WAREHAUS_HEARTBEAT_POST_URL, json.dumps(data), {'Content-Type': 'application/json'})
    response = urlopen(request).read()

info = collect_info()
send(info)
<commit_msg>Remove --output-all from heartbeat code (not supported on AWS linux)<commit_after>import os
import re
import sys
import json
from urllib2 import Request
from urllib2 import urlopen
from urlparse import urljoin
from commands import getstatusoutput
from traceback import format_exc
from collections import defaultdict

def run(cmdline, expected_status=0):
    status, output = getstatusoutput(cmdline)
    if status != expected_status:
        raise RuntimeError('Error while running {0!r}: {1}'.format(cmdline, output))
    return output

def collect_net_info():
    net_info = defaultdict(dict)
    for iface in run('ip --oneline link').splitlines():
        mo = re.match(r'^\d:\s+(?P<iface_name>\w+):.*link/(?P<iface_type>\w+)\s+(?P<iface_addr>[\w:]+).*$', iface)
        if mo is not None:
            iface_info = mo.groupdict()
            if iface_info['iface_type'] == 'loopback':
                continue
            net_info[iface_info['iface_name']]['mac'] = iface_info['iface_addr']
    for iface in run('ip --oneline addr').splitlines():
        _, iface_name, iface_info = iface.split(' ', 2)
        if iface_name == 'lo':
            continue
        iface_info = iface_info.strip().split()
        if iface_info[0] == 'inet':
            net_info[iface_name].setdefault('inet', []).append(iface_info[1])
        if iface_info[0] == 'inet6':
            net_info[iface_name].setdefault('inet6', []).append(iface_info[1])
    return [dict(dev=dev, **fields) for dev, fields in net_info.iteritems()]

def collect_pci_info():
    lspci = run('lspci -mm')
    pci_devices = []
    for line in lspci.splitlines():
        mo = re.match(r'^(?P<address>[:\w.]+)\s+"(?P<type>[^"]+)" "(?P<vendor>[^"]+)" "(?P<name>[^"]+)".*$', line)
        if mo is not None:
            pci_devices.append(mo.groupdict())
    return pci_devices

def collect_disk_info():
    lsblk = run('lsblk --bytes --nodeps --pairs')
    disks = []
    for line in lsblk.splitlines():
        fields = {}
        for mo in re.finditer(r'(?P<key>[^=]+)="(?P<value>[^"]*)"', line):
            groupdict = mo.groupdict()
            key, value = groupdict['key'].strip().lower(), groupdict['value'].strip()
            try:
                value = int(value)
            except ValueError:
                pass
            fields[key] = value
        disks.append(fields)
    return disks

def collect_cpu_info():
    cpus_info = open('/proc/cpuinfo').read()
    return [dict(map(lambda x: x.strip(), line.split(':', 1)) for line in cpu_info.splitlines() if line.strip()) for cpu_info in cpus_info.split('\n\n') if cpu_info.strip()]

MEM_KEYS = (
    'MemTotal',
    'MemFree',
    'MemAvailable',
    'SwapFree',
    'SwapFree',
    'SwapCached',
)

def collect_mem_info():
    mem_info = {}
    for line in open('/proc/meminfo').read().splitlines():
        key, value = map(lambda x: x.strip(), line.split(':', 1))
        if key not in MEM_KEYS:
            continue
        try:
            value = int(value)
        except ValueError:
            if value.endswith('kB'):
                value = int(value[:-len('kB')]) * 1024
        mem_info[key] = value
    return mem_info

def collect_fs_info():
    df = run('df --block-size=1 /')
    lines = df.splitlines()[1:]
    fields = df.splitlines()[1].split()
    return {'/': dict(
        dev             = fields[0],
        total_bytes     = int(fields[1]),
        used_bytes      = int(fields[2]),
        available_bytes = int(fields[3]),
    )}

info_collectors = dict(
    net         = collect_net_info,
    pci_devices = collect_pci_info,
    disks       = collect_disk_info,
    cpu         = collect_cpu_info,
    mem         = collect_mem_info,
    fs          = collect_fs_info,
)

def collect_info():
    info = dict(
        hostname = open('/proc/sys/kernel/hostname').read().strip(),
        hw_info  = {},
        errors   = [],
    )
    for key, collect_func in info_collectors.iteritems():
        try:
            info['hw_info'][key] = collect_func()
        except Exception as error:
            info['hw_info'][key] = None
            info['errors'].append(dict(info=key, error=str(error), traceback=format_exc()))
    return info

def send(data):
    request = Request(WAREHAUS_HEARTBEAT_POST_URL, json.dumps(data), {'Content-Type': 'application/json'})
    response = urlopen(request).read()

info = collect_info()
send(info)

{'fraction_non_alphanumeric': 0.15754522555530112, 'fraction_numerical': 0.00572475383558507, 'mean_word_length': 2.61099627945432, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2493107', 'n_tokens_mistral': 2881, 'n_tokens_neox': 2859, 'n_words': 616}