<commit_before># -*- coding: utf-8 -*-
import os
import argparse
import json
import subprocess
import urllib2
import getpass
import socket

from simocollector.sender import BaseSender
from simocollector.collectors import system_info_collector
from simocollector.utils import slugify
from simocollector.sample_config import default_config


class ServerInfoSender(BaseSender):
    name = 'server'

    def __init__(self, config, ip, name):
        if 'server_id' not in config:
            config['server_id'] = ''
        super(ServerInfoSender, self).__init__(config)
        self.ip = ip
        self.server_name = slugify(unicode(name))

    def get_data(self):
        raw_data = system_info_collector.get_system_info()
        data = {
            'distribution': raw_data['distro'].get('distribution', 'unknown'),
            'release': raw_data['distro'].get('release', 'unknown'),
            'cpu_model_name': raw_data['processor'].get('model-name', 'unknown'),
            'cpu_number_of_cores': raw_data['processor'].get('cpu-cores', 0),
            'name': self.server_name,
            'slug': self.server_name,
            'ip_address': self.ip,
        }
        return data

    def _add_addtional_data(self, data):
        return data


def _get_hostname():
    return subprocess.Popen(['hostname'], stdout=subprocess.PIPE, close_fds=True).communicate()[0].strip()


def _get_host_ip_address():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(('google.com', 80))
    ip = s.getsockname()[0]
    s.close()

    return ip


def create_cron_jobs():
    publisher_path = subprocess.Popen(['which', 'simo-collection-publish.py'],
                                      stdout=subprocess.PIPE, close_fds=True).communicate()[0].strip()
    jobs = [
        '*/5 * * * * root {} -t loadavg'.format(publisher_path),
        '*/10 * * * * root {} -t cpu'.format(publisher_path),
        '*/15 * * * * root {} -t memory'.format(publisher_path),
    ]

    filename = '/etc/cron.d/simo-collector'

    if not os.path.exists(os.path.dirname(filename)):
        os.makedirs(os.path.dirname(filename))

    with open(filename, 'w') as f:
        f.write('\n'.join(jobs))
        f.close()


def main():
    create_cron_jobs()
    parser = argparse.ArgumentParser(description='Create sample configuration of SIMO Collector')
    parser.add_argument('path', default='/etc/simo/collector.conf', type=str, nargs='?',
                        help='path to configuration file (default /etc/simo/collector.conf).')

    args = parser.parse_args()

    actual_user = getpass.getuser()
    host_ip_address = _get_host_ip_address()
    current_hostname = _get_hostname()

    username = raw_input('Username [{}]: '.format(actual_user)) or actual_user
    password = <PASSWORD>('Password: ')
    server = raw_input('SIMO url: ')
    ip_address = raw_input('Write server ip address [{}]: '.format(host_ip_address)) or host_ip_address
    hostname = raw_input('Server name [{}]: '.format(current_hostname)) or current_hostname

    config = {
        'username': username,
        'password': password,
        'server': server
    }

    try:
        sender = ServerInfoSender(config, ip_address, hostname)
        response = sender.send()
        response_data = json.load(response)
    except urllib2.HTTPError, e:
        print(e.fp.read())
        exit(1)

    if 'id' not in response_data:
        print(response_data)
        exit(1)

    default_config['server_id'] = response_data['id']
    default_config['server'] = server
    default_config['username'] = username
    default_config['password'] = password

    config_path = args.path

    if not os.path.exists(os.path.dirname(config_path)):
        os.makedirs(os.path.dirname(config_path))

    with open(config_path, 'w') as f:
        f.write(json.dumps(default_config))

    print('Creating configuration on path {}'.format(config_path))

    create_cron_jobs()

if __name__ == '__main__':
    main()<commit_msg>Added registration for network devices and hard disks<commit_after># -*- coding: utf-8 -*-
import os
import argparse
import json
import subprocess
import urllib2
import getpass
import socket

from simocollector.sender import BaseObjectSender, BaseMultiObjectSender
from simocollector.collectors import system_info_collector
from simocollector.utils import slugify
from simocollector.sample_config import default_config


CRON_JOB_FILENAME = '/etc/cron.d/simo-collector'
CONFIG_FILE_DEFAULT_PATH = '/etc/simo/collector.conf'


class ServerInfoSender(BaseObjectSender):
    name = 'server'

    def __init__(self, config, ip, name):
        if 'server_id' not in config:
            config['server_id'] = ''
        super(ServerInfoSender, self).__init__(config)
        self.ip = ip
        self.server_name = slugify(unicode(name))

    def get_data(self):
        raw_data = system_info_collector.get_system_info()
        data = {
            'distribution': raw_data['distro'].get('distribution', 'unknown'),
            'release': raw_data['distro'].get('release', 'unknown'),
            'cpu_model_name': raw_data['processor'].get('model-name', 'unknown'),
            'cpu_number_of_cores': raw_data['processor'].get('cpu-cores', 0),
            'name': self.server_name,
            'slug': self.server_name,
            'ip_address': self.ip,
        }
        return data

    def add_additional_data(self, data):
        return data


class DiskRegisterSender(BaseMultiObjectSender):
    name = 'disk'

    def get_data(self):
        raw_data = system_info_collector.get_disk_usage()
        data = []
        for partition_name, partition_data in raw_data.iteritems():
            disk_data = {
                'partition_name': partition_name,
                'path': partition_data['path'],
                'total': partition_data['total'],
                'volume': partition_data['volume']
            }
            data.append(self.add_additional_data(disk_data))

        return data


class NetDeviceRegisterSender(BaseMultiObjectSender):
    name = 'netdevice'

    def get_data(self):
        raw_data = system_info_collector.get_network_traffic()
        data = []
        for device_name in raw_data.iterkeys():
            device_data = {
                'name': device_name,
            }
            data.append(self.add_additional_data(device_data))

        return data


def _get_hostname():
    return subprocess.Popen(['hostname'], stdout=subprocess.PIPE, close_fds=True).communicate()[0].strip()


def _get_host_ip_address():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(('google.com', 80))
    ip = s.getsockname()[0]
    s.close()

    return ip


def create_cron_jobs():
    publisher_path = subprocess.Popen(['which', 'simo-collection-publish.py'],
                                      stdout=subprocess.PIPE, close_fds=True).communicate()[0].strip()
    jobs = [
        '*/5 * * * * root {} -t loadavg'.format(publisher_path),
        '*/5 * * * * root {} -t networktraffic'.format(publisher_path),
        '*/10 * * * * root {} -t cpu'.format(publisher_path),
        '*/15 * * * * root {} -t memory'.format(publisher_path),
        '* */3 * * * root {} -t diskusage'.format(publisher_path),
    ]

    if not os.path.exists(os.path.dirname(CRON_JOB_FILENAME)):
        os.makedirs(os.path.dirname(CRON_JOB_FILENAME))

    with open(CRON_JOB_FILENAME, 'w') as f:
        f.write('\n'.join(jobs))
        f.close()


def _wrap_with(code):
    def inner(text, bold=False):
        c = code
        if bold:
            c = "1;%s" % c
        return "\033[%sm%s\033[0m" % (c, text)

    return inner


red = _wrap_with('31')
green = _wrap_with('32')
yellow = _wrap_with('33')
blue = _wrap_with('34')
magenta = _wrap_with('35')
cyan = _wrap_with('36')
white = _wrap_with('37')


def main():
    parser = argparse.ArgumentParser(description='SIMO Collector installer.')
    parser.add_argument('path', default=CONFIG_FILE_DEFAULT_PATH, type=str, nargs='?',
                        help='path to configuration file (default {}).'.format(CONFIG_FILE_DEFAULT_PATH))

    args = parser.parse_args()

    actual_user = getpass.getuser()
    host_ip_address = _get_host_ip_address()
    current_hostname = _get_hostname()

    username = raw_input('Username [{}]: '.format(actual_user)) or actual_user
    password = <PASSWORD>('Password: ')
    server = raw_input('SIMO url: ')
    ip_address = raw_input('Write server ip address [{}]: '.format(host_ip_address)) or host_ip_address
    hostname = raw_input('Server name [{}]: '.format(current_hostname)) or current_hostname

    config = {
        'username': username,
        'password': password,
        'server': server
    }

    print('\n\n')

    try:
        sender = ServerInfoSender(config, ip_address, hostname)
        response = sender.send()
        response_data = json.loads(response)
    except urllib2.HTTPError, e:
        print(e.fp.read())
        exit(1)

    if 'url' not in response_data:
        print(red('Bad data response:'))
        print(response_data)
        exit(1)

    default_config['server_id'] = response_data['url']
    default_config['server'] = server
    default_config['username'] = username
    default_config['password'] = password

    config_path = args.path

    if not os.path.exists(os.path.dirname(config_path)):
        os.makedirs(os.path.dirname(config_path))

    try:
        response = DiskRegisterSender(default_config).send()
    except urllib2.HTTPError, e:
        print(red('Problem in disk registration process'))
        print(e.fp.read())
        exit(1)

    default_config['disk'] = {}
    for data in response:
        disk = json.loads(data)
        partition_name = disk['partition_name']
        print('{}: {}'.format(green('Disk partition has been registred'), partition_name))
        default_config['disk'][partition_name] = disk['url']

    try:
        response = NetDeviceRegisterSender(default_config).send()
    except urllib2.HTTPError, e:
        print(red('Problem in disk registration process'))
        print(e.fp.read())
        exit(1)

    default_config['networkdevices'] = {}
    for data in response:
        device_data = json.loads(data)
        device_name = device_data['name']
        print('{}: {}'.format(green('Network device has been registred'), device_name))
        default_config['networkdevices'][device_name] = device_data['url']

    with open(config_path, 'w') as f:
        f.write(json.dumps(default_config, indent=4, sort_keys=True))

    print('{}: {}'.format(green('Creating configuration on path'), config_path))

    print('{}: {}'.format(green('Creating cron jobs'), CRON_JOB_FILENAME))
    create_cron_jobs()

    print('\n\n{}'.format(green('Successfully installed SIMO Collector')))


def mmain():
    conf = '/home/rbas/simocollector.conf'
    with open(conf, 'r') as f:
        default_config = json.load(f)

    try:
        response = NetDeviceRegisterSender(default_config).send()
    except urllib2.HTTPError, e:
        print(e.fp.read())
        exit(1)

    default_config['networkdevices'] = {}
    for data in response:
        device_data = json.loads(data)
        default_config['networkdevices'][device_data['name']] = device_data['url']

    json_config = json.dumps(default_config, indent=4, sort_keys=True)
    with open(conf, 'w') as f:
        f.write(json_config)


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.13547766803601083, 'fraction_numerical': 0.005593916615680448, 'mean_word_length': 3.0260380014074597, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6334822', 'n_tokens_mistral': 3646, 'n_tokens_neox': 3546, 'n_words': 696}