<commit_before>import os
import os.path
import urllib
import stat
import socket
import time
import httplib
import xmlrpclib
import paramiko
import string
import random
from threading import Thread

import BaseHTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler
import posixpath

import logging
logger = logging.getLogger('devops.helpers')

class TimeoutError(Exception): pass
class AuthenticationError(Exception): pass

def get_free_port():
    ports = range(32000, 32100)
    random.shuffle(ports)
    for port in ports:
        if not tcp_ping('localhost', port):
            return port
    raise Error, "No free ports available"

def icmp_ping(host, timeout=1):
    "icmp_ping(host, timeout=1) - returns True if host is pingable; False - otherwise."
    return os.system("ping -c 1 -W '%(timeout)d' '%(host)s' 1>/dev/null 2>&1" % { 'host': str(host), 'timeout': timeout}) == 0

def tcp_ping(host, port):
    "tcp_ping(host, port) - returns True if TCP connection to specified host and port can be established; False - otherwise."
    s = socket.socket()
    try:
        s.connect((str(host), int(port)))
    except socket.error:
        return False
    s.close()
    return True

def wait(predicate, interval=5, timeout=None):
    """
      wait(predicate, interval=5, timeout=None) - wait until predicate will become True. Returns number of seconds that is left or 0 if timeout is None.
      Options:
        interval - seconds between checks.
        timeout  - raise TimeoutError if predicate won't become True after this amount of seconds. 'None' disables timeout.
    """
    start_time = time.time()
    while not predicate():
        if timeout and start_time + timeout < time.time():
            raise TimeoutError, "Waiting timed out"

        seconds_to_sleep = interval
        if timeout:
            seconds_to_sleep = max(0, min(seconds_to_sleep, start_time + timeout - time.time()))
        time.sleep(seconds_to_sleep)

    return timeout + start_time - time.time() if timeout else 0

def http(host='localhost', port=80, method='GET', url='/', waited_code=200):
    try:
        conn = httplib.HTTPConnection(str(host), int(port))
        conn.request(method, url)
        res = conn.getresponse()
        
        if res.status == waited_code:
            return True
        return False
    except:
        return False


class KeyPolicy(paramiko.WarningPolicy):
    def missing_host_key(self, client, hostname, key):
        return
    
class SSHClient(object):
    class get_sudo(object):
        def __init__(self, ssh):
            self.ssh = ssh

        def __enter__(self):
            self.ssh.sudo_mode = True

        def __exit__(self, type, value, traceback):
            self.ssh.sudo_mode = False

    def __init__(self, host, port=22, username=None, password=<PASSWORD>):
        self.host = str(host)
        self.port = int(port)
        self.username = username
        self.password = password

        self.sudo_mode = False
        self.sudo = self.get_sudo(self)

        self.reconnect()

    def __del__(self):
        self._sftp.close()
        self._ssh.close()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        pass

    def reconnect(self):
        self._ssh = paramiko.SSHClient()
        self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        self._ssh.connect(self.host, port=self.port, username=self.username, password=self.password)
        self._sftp = self._ssh.open_sftp()

    def execute(self, command):
        logger.debug("Executing command: '%s'" % command.rstrip())
        chan = self._ssh.get_transport().open_session()
        stdin = chan.makefile('wb')
        stdout = chan.makefile('rb')
        stderr = chan.makefile_stderr('rb')
        cmd = "%s\n" % command
        if self.sudo_mode:
            cmd = 'sudo -S bash -c "%s"' % cmd.replace('"', '\\"')
        chan.exec_command(cmd)
        if stdout.channel.closed is False:
            stdin.write('%s\n' % self.password)
            stdin.flush()
        result = {
            'stdout': [],
            'stderr': [],
            'exit_code': chan.recv_exit_status()
        }
        for line in stdout:
            result['stdout'].append(line)
        for line in stderr:
            result['stderr'].append(line)

        chan.close()

        return result

    def mkdir(self, path):
        if self.exists(path):
            return
        logger.debug("Creating directory: %s" % path)
        self.execute("mkdir %s\n" % path)

    def rm_rf(self, path):
        logger.debug("Removing directory: %s" % path)
        self.execute("rm -rf %s" % path)

    def open(self, path, mode='r'):
        return self._sftp.open(path, mode)

    def upload(self, source, target):
        logger.debug("Copying '%s' -> '%s'" % (source, target))

        if self.isdir(target):
            target = os.path.join(target, os.path.basename(source))

        if not os.path.isdir(source):
            self._sftp.put(source, target)
            return<commit_msg>[devops] Updated devops ssh helper to use  -p switch for mkdir<commit_after>import os
import os.path
import urllib
import stat
import socket
import time
import httplib
import xmlrpclib
import paramiko
import string
import random
from threading import Thread

import BaseHTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler
import posixpath

import logging
logger = logging.getLogger('devops.helpers')

class TimeoutError(Exception): pass
class AuthenticationError(Exception): pass

def get_free_port():
    ports = range(32000, 32100)
    random.shuffle(ports)
    for port in ports:
        if not tcp_ping('localhost', port):
            return port
    raise Error, "No free ports available"

def icmp_ping(host, timeout=1):
    "icmp_ping(host, timeout=1) - returns True if host is pingable; False - otherwise."
    return os.system("ping -c 1 -W '%(timeout)d' '%(host)s' 1>/dev/null 2>&1" % { 'host': str(host), 'timeout': timeout}) == 0

def tcp_ping(host, port):
    "tcp_ping(host, port) - returns True if TCP connection to specified host and port can be established; False - otherwise."
    s = socket.socket()
    try:
        s.connect((str(host), int(port)))
    except socket.error:
        return False
    s.close()
    return True

def wait(predicate, interval=5, timeout=None):
    """
      wait(predicate, interval=5, timeout=None) - wait until predicate will become True. Returns number of seconds that is left or 0 if timeout is None.
      Options:
        interval - seconds between checks.
        timeout  - raise TimeoutError if predicate won't become True after this amount of seconds. 'None' disables timeout.
    """
    start_time = time.time()
    while not predicate():
        if timeout and start_time + timeout < time.time():
            raise TimeoutError, "Waiting timed out"

        seconds_to_sleep = interval
        if timeout:
            seconds_to_sleep = max(0, min(seconds_to_sleep, start_time + timeout - time.time()))
        time.sleep(seconds_to_sleep)

    return timeout + start_time - time.time() if timeout else 0

def http(host='localhost', port=80, method='GET', url='/', waited_code=200):
    try:
        conn = httplib.HTTPConnection(str(host), int(port))
        conn.request(method, url)
        res = conn.getresponse()
        
        if res.status == waited_code:
            return True
        return False
    except:
        return False


class KeyPolicy(paramiko.WarningPolicy):
    def missing_host_key(self, client, hostname, key):
        return
    
class SSHClient(object):
    class get_sudo(object):
        def __init__(self, ssh):
            self.ssh = ssh

        def __enter__(self):
            self.ssh.sudo_mode = True

        def __exit__(self, type, value, traceback):
            self.ssh.sudo_mode = False

    def __init__(self, host, port=22, username=None, password=<PASSWORD>):
        self.host = str(host)
        self.port = int(port)
        self.username = username
        self.password = password

        self.sudo_mode = False
        self.sudo = self.get_sudo(self)

        self.reconnect()

    def __del__(self):
        self._sftp.close()
        self._ssh.close()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        pass

    def reconnect(self):
        self._ssh = paramiko.SSHClient()
        self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        self._ssh.connect(self.host, port=self.port, username=self.username, password=self.password)
        self._sftp = self._ssh.open_sftp()

    def execute(self, command):
        logger.debug("Executing command: '%s'" % command.rstrip())
        chan = self._ssh.get_transport().open_session()
        stdin = chan.makefile('wb')
        stdout = chan.makefile('rb')
        stderr = chan.makefile_stderr('rb')
        cmd = "%s\n" % command
        if self.sudo_mode:
            cmd = 'sudo -S bash -c "%s"' % cmd.replace('"', '\\"')
        chan.exec_command(cmd)
        if stdout.channel.closed is False:
            stdin.write('%s\n' % self.password)
            stdin.flush()
        result = {
            'stdout': [],
            'stderr': [],
            'exit_code': chan.recv_exit_status()
        }
        for line in stdout:
            result['stdout'].append(line)
        for line in stderr:
            result['stderr'].append(line)

        chan.close()

        return result

    def mkdir(self, path):
        if self.exists(path):
            return
        logger.debug("Creating directory: %s" % path)
        self.execute("mkdir -p %s\n" % path)

    def rm_rf(self, path):
        logger.debug("Removing directory: %s" % path)
        self.execute("rm -rf %s" % path)

    def open(self, path, mode='r'):
        return self._sftp.open(path, mode)

    def upload(self, source, target):
        logger.debug("Copying '%s' -> '%s'" % (source, target))

        if self.isdir(target):
            target = os.path.join(target, os.path.basename(source))

        if not os.path.isdir(source):
            self._sftp.put(source, target)
            return
{'fraction_non_alphanumeric': 0.1163657770800628, 'fraction_numerical': 0.005690737833594976, 'mean_word_length': 2.480027313076135, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '63356', 'n_tokens_mistral': 3149, 'n_tokens_neox': 2992, 'n_words': 824}