<commit_before>#!/usr/bin/env python -i

# Fabric - Pythonic remote deployment tool.
# Copyright (C) 2008  <NAME>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import getpass
import os
import os.path
import re
import signal
import subprocess
import sys
import threading
import time
import types
import datetime

try:
    import paramiko as ssh
except ImportError:
    print("Error: paramiko is a required module. Please install it:")
    print("  $ sudo easy_install paramiko")
    exit(1)

__version__ = '0.0.4'
__author__ = '<NAME>'
__author_email__ = '<EMAIL>'
__url__ = 'https://savannah.nongnu.org/projects/fab/'
__license__ = 'GPL-2'
__greeter__ = '''\
   Fabric v. %(fab_version)s, Copyright (C) 2008 %(fab_author)s.
   Fabric comes with ABSOLUTELY NO WARRANTY; for details type `fab warranty'.
   This is free software, and you are welcome to redistribute it
   under certain conditions; type `fab license' for details.
'''

ENV = {
    'fab_version':__version__,
    'fab_author':__author__,
    'fab_mode':'rolling',
    'fab_port':22,
    'fab_user':None,
    'fab_pkey':None,
    'fab_key_filename':None,
    'fab_new_host_key':'accept',
    'fab_shell':'/bin/bash -l -c "%s"',
    'fab_timestamp':datetime.datetime.utcnow().strftime('%F_%H-%M-%S'),
    'fab_debug':False,
}

CONNECTIONS = []
COMMANDS = {}
OPERATIONS = {}
STRATEGIES = {}
_LAZY_FORMAT_SUBSTITUTER = re.compile(r'\$\((?P<var>\w+?)\)')

#
# Helper decorators:
#
def new_registering_decorator(registry):
    def registering_decorator(first_arg=None):
        if callable(first_arg):
            registry[first_arg.__name__] = first_arg
            return first_arg
        else:
            def sub_decorator(f):
                registry[first_arg] = f
                return f
            return sub_decorator
    return registering_decorator
command = new_registering_decorator(COMMANDS)
operation = new_registering_decorator(OPERATIONS)
strategy = new_registering_decorator(STRATEGIES)

def run_per_host(op_fn):
    def wrapper(*args, **kvargs):
        if not CONNECTIONS:
            _connect()
        _on_hosts_do(op_fn, *args, **kvargs)
    wrapper.__doc__ = op_fn.__doc__
    wrapper.__name__ = op_fn.__name__
    return wrapper

#
# Standard fabfile operations:
#
@operation
def set(**variables):
    """Set a number of Fabric environment variables.
    
    Set takes a number of keyword arguments, and defines or updates the
    variables that corrosponds to each keyword with the respective value.
    
    The values can be of any type, but strings are used for most variables.
    If the value is a string and contain any eager variable references, such as
    %(fab_user)s, then these will be expanded to their corrosponding value.
    Lazy references, those beginning with a $ rather than a %, will not be
    expanded.
    
    Example:
        set(fab_user='joe.shmoe', fab_mode='rolling')
    
    """
    for k, v in variables.items():
        if isinstance(v, types.StringTypes):
            ENV[k] = (v % ENV)
        else:
            ENV[k] = v

@operation
def get(name, otherwise=None):
    """Get the value of a given Fabric environment variable.
    
    If the variable isn't found, then this operation returns the
    value of the 'otherwise' parameter, which is None unless set.
    
    """
    return name in ENV and ENV[name] or otherwise

@operation
def require(var, **kvargs):
    """Make sure that a certain environmet variable is available.
    
    The 'var' parameter is a string that names the variable to check for.
    Two other optional kvargs are supported:
        * 'used_for' is a string that gets injected into, and then printed, as
          something like this string: "This variable is used for %s".
        * 'provided_by' is a list of strings that name commands which the user
          can run in order to satisfy the requirement.
    
    If the required variable is not found in the current environment, then the
    operation is stopped and Fabric halts.
    
    Example:
        require('project_name',
            used_for='finding the target deployment dir.',
            provided_by=['staging', 'production'],
        )
    
    """
    if var in ENV:
        return
    print(
        ("The '%(fab_cur_command)s' command requires a '" + var
        + "' variable.") % ENV
    )
    if 'used_for' in kvargs:
        print("This variable is used for %s" % _lazy_format(kvargs['used_for']))
    if 'provided_by' in kvargs:
        print("Get the variable by running one of these commands:")
        to_s = lambda obj: getattr(obj, '__name__', str(obj))
        provided_by = [to_s(obj) for obj in kvargs['provided_by']]
        print('\t' + ('\n\t'.join(provided_by)))
    exit(1)

@operation
@run_per_host
def put(host, client, env, localpath, remotepath, **kvargs):
    """Upload a file to the current hosts.
    
    The 'localpath' parameter is the relative or absolute path to the file on
    your localhost that you wish to upload to the fab_hosts.
    The 'remotepath' parameter is the destination path on the individual
    fab_hosts, and relative paths are relative to the fab_user's home
    directory.
    
    May take an additional 'fail' keyword argument with one of these values:
        * ignore - do nothing on failure
        * warn - print warning on failure
        * abort - terminate fabric on failure
    
    Example:
        put('bin/project.zip', '/tmp/project.zip')
    
    """
    localpath = _lazy_format(localpath, env)
    remotepath = _lazy_format(remotepath, env)
    if not os.path.exists(localpath):
        return False
    ftp = client.open_sftp()
    print("[%s] put: %s -> %s" % (host, localpath, remotepath))
    ftp.put(localpath, remotepath)
    return True

@operation
@run_per_host
def download(host, client, env, remotepath, localpath, **kvargs):
    """Download a file from the remote hosts.
    
    The 'remotepath' parameter is the relative or absolute path to the files
    to download from the fab_hosts. The 'localpath' parameter will be suffixed
    with the individual hostname from which they were downloaded, and the
    downloaded files will then be stored in those respective paths.
    
    May take an additional 'fail' keyword argument with one of these values:
        * ignore - do nothing on failure
        * warn - print warning on failure
        * abort - terminate fabric on failure
    
    Example:
        set(fab_hosts=['node1.cluster.com','node2.cluster.com'])
        download('/var/log/server.log', 'server.log')
    
    The above code will produce two files on your local system, called
    "server.log.node1.cluster.com" and "server.log.node2.cluster.com"
    respectively.
    
    """
    ftp = client.open_sftp()
    localpath = _lazy_format(localpath) + '.' + host
    remotepath = _lazy_format(remotepath)
    print("[%s] download: %s <- %s" % (host, localpath, remotepath))
    ftp.get(remotepath, localpath)
    return True

@operation
@run_per_host
def run(host, client, env, cmd, **kvargs):
    """Run a shell command on the current fab_hosts.
    
    The provided command is executed with the permisions of fab_user, and the
    exact execution environ is determined by the fab_shell variable.
    
    May take an additional 'fail' keyword argument with one of these values:
        * ignore - do nothing on failure
        * warn - print warning on failure
        * abort - terminate fabric on failure
    
    Example:
        run("ls")
    
    """
    cmd = env['fab_shell'] % _lazy_format(cmd, env).replace('"', '\\"')
    if not _confirm_proceed('run', host, kvargs):
        return False # TODO: should we return False in fail??
    print("[%s] run: %s" % (host, cmd))
    stdin, stdout, stderr = client.exec_command(cmd)
    out_th = _start_outputter("[%s] out" % host, stdout)
    err_th = _start_outputter("[%s] err" % host, stderr)
    out_th.join()
    err_th.join()
    return True

@operation
@run_per_host
def sudo(host, client, env, cmd, **kvargs):
    """Run a sudo (root privileged) command on the current hosts.
    
    The provided command is executed with root permisions, provided that
    fab_user is in the sudoers file in the remote host. The exact execution
    environ is determined by the fab_shell variable - the 'sudo' part is
    injected into this variable.
    
    May take an additional 'fail' keyword argument with one of these values:
        * ignore - do nothing on failure
        * warn - print warning on failure
        * abort - terminate fabric on failure
    
    Example:
        sudo("install_script.py")
    
    """
    cmd = _lazy_format(cmd, env)
    real_cmd = env['fab_shell'] % ("sudo -S " + cmd.replace('"', '\\"'))
    cmd = env['fab_debug'] and real_cmd or cmd
    if not _confirm_proceed('sudo', host, kvargs):
        return False # TODO: should we return False in fail??
    print("[%s] sudo: %s" % (host, cmd))
    stdin, stdout, stderr = client.exec_command(real_cmd)<commit_msg>Changed variable fab_debug -> fab_print_real_sudo as it was too generic.<commit_after>#!/usr/bin/env python -i

# Fabric - Pythonic remote deployment tool.
# Copyright (C) 2008  <NAME>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import getpass
import os
import os.path
import re
import signal
import subprocess
import sys
import threading
import time
import types
import datetime

try:
    import paramiko as ssh
except ImportError:
    print("Error: paramiko is a required module. Please install it:")
    print("  $ sudo easy_install paramiko")
    exit(1)

__version__ = '0.0.4'
__author__ = '<NAME>'
__author_email__ = '<EMAIL>'
__url__ = 'https://savannah.nongnu.org/projects/fab/'
__license__ = 'GPL-2'
__greeter__ = '''\
   Fabric v. %(fab_version)s, Copyright (C) 2008 %(fab_author)s.
   Fabric comes with ABSOLUTELY NO WARRANTY; for details type `fab warranty'.
   This is free software, and you are welcome to redistribute it
   under certain conditions; type `fab license' for details.
'''

ENV = {
    'fab_version':__version__,
    'fab_author':__author__,
    'fab_mode':'rolling',
    'fab_port':22,
    'fab_user':None,
    'fab_pkey':None,
    'fab_key_filename':None,
    'fab_new_host_key':'accept',
    'fab_shell':'/bin/bash -l -c "%s"',
    'fab_timestamp':datetime.datetime.utcnow().strftime('%F_%H-%M-%S'),
    'fab_print_real_sudo':False,
}

CONNECTIONS = []
COMMANDS = {}
OPERATIONS = {}
STRATEGIES = {}
_LAZY_FORMAT_SUBSTITUTER = re.compile(r'\$\((?P<var>\w+?)\)')

#
# Helper decorators:
#
def new_registering_decorator(registry):
    def registering_decorator(first_arg=None):
        if callable(first_arg):
            registry[first_arg.__name__] = first_arg
            return first_arg
        else:
            def sub_decorator(f):
                registry[first_arg] = f
                return f
            return sub_decorator
    return registering_decorator
command = new_registering_decorator(COMMANDS)
operation = new_registering_decorator(OPERATIONS)
strategy = new_registering_decorator(STRATEGIES)

def run_per_host(op_fn):
    def wrapper(*args, **kvargs):
        if not CONNECTIONS:
            _connect()
        _on_hosts_do(op_fn, *args, **kvargs)
    wrapper.__doc__ = op_fn.__doc__
    wrapper.__name__ = op_fn.__name__
    return wrapper

#
# Standard fabfile operations:
#
@operation
def set(**variables):
    """Set a number of Fabric environment variables.
    
    Set takes a number of keyword arguments, and defines or updates the
    variables that corrosponds to each keyword with the respective value.
    
    The values can be of any type, but strings are used for most variables.
    If the value is a string and contain any eager variable references, such as
    %(fab_user)s, then these will be expanded to their corrosponding value.
    Lazy references, those beginning with a $ rather than a %, will not be
    expanded.
    
    Example:
        set(fab_user='joe.shmoe', fab_mode='rolling')
    
    """
    for k, v in variables.items():
        if isinstance(v, types.StringTypes):
            ENV[k] = (v % ENV)
        else:
            ENV[k] = v

@operation
def get(name, otherwise=None):
    """Get the value of a given Fabric environment variable.
    
    If the variable isn't found, then this operation returns the
    value of the 'otherwise' parameter, which is None unless set.
    
    """
    return name in ENV and ENV[name] or otherwise

@operation
def require(var, **kvargs):
    """Make sure that a certain environmet variable is available.
    
    The 'var' parameter is a string that names the variable to check for.
    Two other optional kvargs are supported:
        * 'used_for' is a string that gets injected into, and then printed, as
          something like this string: "This variable is used for %s".
        * 'provided_by' is a list of strings that name commands which the user
          can run in order to satisfy the requirement.
    
    If the required variable is not found in the current environment, then the
    operation is stopped and Fabric halts.
    
    Example:
        require('project_name',
            used_for='finding the target deployment dir.',
            provided_by=['staging', 'production'],
        )
    
    """
    if var in ENV:
        return
    print(
        ("The '%(fab_cur_command)s' command requires a '" + var
        + "' variable.") % ENV
    )
    if 'used_for' in kvargs:
        print("This variable is used for %s" % _lazy_format(kvargs['used_for']))
    if 'provided_by' in kvargs:
        print("Get the variable by running one of these commands:")
        to_s = lambda obj: getattr(obj, '__name__', str(obj))
        provided_by = [to_s(obj) for obj in kvargs['provided_by']]
        print('\t' + ('\n\t'.join(provided_by)))
    exit(1)

@operation
@run_per_host
def put(host, client, env, localpath, remotepath, **kvargs):
    """Upload a file to the current hosts.
    
    The 'localpath' parameter is the relative or absolute path to the file on
    your localhost that you wish to upload to the fab_hosts.
    The 'remotepath' parameter is the destination path on the individual
    fab_hosts, and relative paths are relative to the fab_user's home
    directory.
    
    May take an additional 'fail' keyword argument with one of these values:
        * ignore - do nothing on failure
        * warn - print warning on failure
        * abort - terminate fabric on failure
    
    Example:
        put('bin/project.zip', '/tmp/project.zip')
    
    """
    localpath = _lazy_format(localpath, env)
    remotepath = _lazy_format(remotepath, env)
    if not os.path.exists(localpath):
        return False
    ftp = client.open_sftp()
    print("[%s] put: %s -> %s" % (host, localpath, remotepath))
    ftp.put(localpath, remotepath)
    return True

@operation
@run_per_host
def download(host, client, env, remotepath, localpath, **kvargs):
    """Download a file from the remote hosts.
    
    The 'remotepath' parameter is the relative or absolute path to the files
    to download from the fab_hosts. The 'localpath' parameter will be suffixed
    with the individual hostname from which they were downloaded, and the
    downloaded files will then be stored in those respective paths.
    
    May take an additional 'fail' keyword argument with one of these values:
        * ignore - do nothing on failure
        * warn - print warning on failure
        * abort - terminate fabric on failure
    
    Example:
        set(fab_hosts=['node1.cluster.com','node2.cluster.com'])
        download('/var/log/server.log', 'server.log')
    
    The above code will produce two files on your local system, called
    "server.log.node1.cluster.com" and "server.log.node2.cluster.com"
    respectively.
    
    """
    ftp = client.open_sftp()
    localpath = _lazy_format(localpath) + '.' + host
    remotepath = _lazy_format(remotepath)
    print("[%s] download: %s <- %s" % (host, localpath, remotepath))
    ftp.get(remotepath, localpath)
    return True

@operation
@run_per_host
def run(host, client, env, cmd, **kvargs):
    """Run a shell command on the current fab_hosts.
    
    The provided command is executed with the permisions of fab_user, and the
    exact execution environ is determined by the fab_shell variable.
    
    May take an additional 'fail' keyword argument with one of these values:
        * ignore - do nothing on failure
        * warn - print warning on failure
        * abort - terminate fabric on failure
    
    Example:
        run("ls")
    
    """
    cmd = env['fab_shell'] % _lazy_format(cmd, env).replace('"', '\\"')
    if not _confirm_proceed('run', host, kvargs):
        return False # TODO: should we return False in fail??
    print("[%s] run: %s" % (host, cmd))
    stdin, stdout, stderr = client.exec_command(cmd)
    out_th = _start_outputter("[%s] out" % host, stdout)
    err_th = _start_outputter("[%s] err" % host, stderr)
    out_th.join()
    err_th.join()
    return True

@operation
@run_per_host
def sudo(host, client, env, cmd, **kvargs):
    """Run a sudo (root privileged) command on the current hosts.
    
    The provided command is executed with root permisions, provided that
    fab_user is in the sudoers file in the remote host. The exact execution
    environ is determined by the fab_shell variable - the 'sudo' part is
    injected into this variable.
    
    May take an additional 'fail' keyword argument with one of these values:
        * ignore - do nothing on failure
        * warn - print warning on failure
        * abort - terminate fabric on failure
    
    Example:
        sudo("install_script.py")
    
    """
    cmd = _lazy_format(cmd, env)
    real_cmd = env['fab_shell'] % ("sudo -S " + cmd.replace('"', '\\"'))
    cmd = env['fab_print_real_sudo'] and real_cmd or cmd
    if not _confirm_proceed('sudo', host, kvargs):
        return False # TODO: should we return False in fail??
    print("[%s] sudo: %s" % (host, cmd))
    stdin, stdout, stderr = client.exec_command(real_cmd)
{'fraction_non_alphanumeric': 0.09577712306817594, 'fraction_numerical': 0.003308006409262418, 'mean_word_length': 3.0091172813924576, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5030750', 'n_tokens_mistral': 5845, 'n_tokens_neox': 5704, 'n_words': 2109}