<commit_before># coding: utf-8

from fabric.api import env, task, hosts, cd
import re, os, json, sys
from types import *
import datetime
import util, conf, testtools
from api import *

@task
@hosts('localhost')
def node(option=None, host_pattern=None, edit_key=None, edit_value=None):
    if option == 'create':
        host_pattern = check_host_pattern(host_pattern)
        env.hosts = util.get_expanded_hosts(host_pattern)
        print env.hosts

        if util.confirm('Are you sure you want to create above nodes?', 'Canceled'):
            for host in env.hosts:
                if not util.exists_json(host):
                    util.dump_json(conf.get_initial_json(host), host)
                else:
                    print '{0} is already created.'.format(host)
            print_hosts()

        return

    elif option == 'remove':
        host_pattern = check_host_pattern(host_pattern)
        env.hosts = util.get_available_hosts(host_pattern)
        print env.hosts

        if util.confirm('Are you sure you want to remove above nodes?', 'Canceled'):
            for host in env.hosts:
                util.remove_json(host)

            print '{0} removed.'.format(host_pattern)

        return

    elif option == 'edit':
        host_pattern = check_host_pattern(host_pattern)
        env.hosts = util.get_available_hosts(host_pattern)
        print_hosts()

        print '\n\nEdit above nodes.'
        if not edit_key and type(edit_value) is not StringType:
            edit_key = raw_input('Enter key: ')

        if not edit_value and type(edit_value) is not StringType:
            edit_value = raw_input('Enter value: ')

        for host in env.hosts:
            host_json = util.load_json(host)
            host_json.update({edit_key: edit_value})
            util.dump_json(host_json, host)

        print_hosts()

    elif option == 'upload':
        host_pattern = check_host_pattern(host_pattern)
        env.hosts = util.get_available_hosts(host_pattern)
        print_hosts()

        if util.confirm('Are you sure you want to upload above nodes?', 'Canceled'):
            for host in env.hosts:
                if util.exists_json(host):
                    print cmd('knife node from file {0}/{1}.json'.format(conf.NODE_DIR, host))[1]
        return

    elif option == 'download':
        host_pattern = check_host_pattern(host_pattern)
        searched_nodes = cmd('knife search node "name:{0}" -F json'.format(host_pattern))[1]
        if env.is_test:
            searched_nodes = testtools.get_searched_nodes(host_pattern)

        nodes = json.loads(searched_nodes)['rows']
        for node in nodes:
            print node['name']

        if util.confirm('Are you sure you want to save above nodes?', 'Canceled'):
            for node in nodes:
                host = node['name']
                node_json = util.load_json(host)
                node_json.update(node)
                util.dump_json(node_json, host)
                print 'saved {0}'.format(host)

        return

    elif option == 'check':
        warning_nodes = cmd('grep \' \[.*[^0]\]\' -r nodes/*'.format(conf.NODE_DIR))[1]
        if warning_nodes == '':
            print 'No warning nodes.'
        else:
            print warning_nodes[1]
        return

    else:
        is_verbose = False
        if option:
            if host_pattern == 'v':
                is_verbose = True
            host_pattern = option
        else:
            host_pattern = '*'

        env.hosts = util.get_available_hosts(host_pattern)
        print_hosts(is_verbose)
        RE_ROLE = re.compile('role\[(.+)\]')
        for host in env.hosts:
            host_json = util.load_json(host)
            for run_list in host_json['run_list']:
                role = RE_ROLE.match(run_list)
                if role:
                    role_name = role.group(1)
                    role_hosts = env.roledefs.get(role_name, [])
                    role_hosts.append(host)
                    env.roledefs.update({
                        role_name: role_hosts
                    })

        for task in env.tasks:
            is_prepare = task.find('prepare') == 0<commit_msg>updated check node<commit_after># coding: utf-8

from fabric.api import env, task, hosts, cd
import re, os, json, sys
from types import *
import datetime
import util, conf, testtools
from api import *

@task
@hosts('localhost')
def node(option=None, host_pattern=None, edit_key=None, edit_value=None):
    if option == 'create':
        host_pattern = check_host_pattern(host_pattern)
        env.hosts = util.get_expanded_hosts(host_pattern)
        print env.hosts

        if util.confirm('Are you sure you want to create above nodes?', 'Canceled'):
            for host in env.hosts:
                if not util.exists_json(host):
                    util.dump_json(conf.get_initial_json(host), host)
                else:
                    print '{0} is already created.'.format(host)
            print_hosts()

        return

    elif option == 'remove':
        host_pattern = check_host_pattern(host_pattern)
        env.hosts = util.get_available_hosts(host_pattern)
        print env.hosts

        if util.confirm('Are you sure you want to remove above nodes?', 'Canceled'):
            for host in env.hosts:
                util.remove_json(host)

            print '{0} removed.'.format(host_pattern)

        return

    elif option == 'edit':
        host_pattern = check_host_pattern(host_pattern)
        env.hosts = util.get_available_hosts(host_pattern)
        print_hosts()

        print '\n\nEdit above nodes.'
        if not edit_key and type(edit_value) is not StringType:
            edit_key = raw_input('Enter key: ')

        if not edit_value and type(edit_value) is not StringType:
            edit_value = raw_input('Enter value: ')

        for host in env.hosts:
            host_json = util.load_json(host)
            host_json.update({edit_key: edit_value})
            util.dump_json(host_json, host)

        print_hosts()

    elif option == 'upload':
        host_pattern = check_host_pattern(host_pattern)
        env.hosts = util.get_available_hosts(host_pattern)
        print_hosts()

        if util.confirm('Are you sure you want to upload above nodes?', 'Canceled'):
            for host in env.hosts:
                if util.exists_json(host):
                    print cmd('knife node from file {0}/{1}.json'.format(conf.NODE_DIR, host))[1]
        return

    elif option == 'download':
        host_pattern = check_host_pattern(host_pattern)
        searched_nodes = cmd('knife search node "name:{0}" -F json'.format(host_pattern))[1]
        if env.is_test:
            searched_nodes = testtools.get_searched_nodes(host_pattern)

        nodes = json.loads(searched_nodes)['rows']
        for node in nodes:
            print node['name']

        if util.confirm('Are you sure you want to save above nodes?', 'Canceled'):
            for node in nodes:
                host = node['name']
                node_json = util.load_json(host)
                node_json.update(node)
                util.dump_json(node_json, host)
                print 'saved {0}'.format(host)

        return

    elif option == 'check':
        warning_nodes    = cmd('grep \' \[.*[^0]\]\' -r {0}/*'.format(conf.NODE_DIR))[1]
        failed_ssh_nodes = cmd('grep \'"ssh": "failed"\' -r {0}/*'.format(conf.NODE_DIR))[1]
        if warning_nodes == '' and failed_ssh_nodes == '':
            print 'No warning nodes.'
        else:
            print warning_nodes
            print failed_ssh_nodes

        return

    else:
        is_verbose = False
        if option:
            if host_pattern == 'v':
                is_verbose = True
            host_pattern = option
        else:
            host_pattern = '*'

        env.hosts = util.get_available_hosts(host_pattern)
        print_hosts(is_verbose)
        RE_ROLE = re.compile('role\[(.+)\]')
        for host in env.hosts:
            host_json = util.load_json(host)
            for run_list in host_json['run_list']:
                role = RE_ROLE.match(run_list)
                if role:
                    role_name = role.group(1)
                    role_hosts = env.roledefs.get(role_name, [])
                    role_hosts.append(host)
                    env.roledefs.update({
                        role_name: role_hosts
                    })

        for task in env.tasks:
            is_prepare = task.find('prepare') == 0
{'fraction_non_alphanumeric': 0.10334310850439883, 'fraction_numerical': 0.0035190615835777126, 'mean_word_length': 1.9339298004129388, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7082596', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2418, 'n_words': 599}