<commit_before>from subprocess import Popen
import os
from time import sleep

import paramiko
import boto.ec2

from brome.core.utils import (
    string_to_filename,
    devnull,
    create_dir_if_doesnt_exist
)
from brome.runner.base_instance import BaseInstance
from brome.core.settings import BROME_CONFIG


class EC2Instance(BaseInstance):
    """EC2 instance

    Attributes:
        runner (object)
        browser_config (object)
        index (int)
    """

    def __init__(self, runner, browser_config, index):
        self.runner = runner
        self.browser_config = browser_config
        self.index = index

    def get_ip(self):
        """Return the ip address of the node
        """

        return self.private_ip

    def scp_file_remote_to_local(self, remote_path, local_path):
        """Scp a remote file to local

        Args:
            remote_path (str)
            local_path (str)
        """

        scp_command = [
            'scp',
            '-o',
            'StrictHostKeyChecking=no',
            '-i',
            self.browser_config.get('ssh_key_path'),
            '%s@%s:"%s"' %
            (
                self.browser_config.get('username'),
                self.get_ip(),
                remote_path
            ),
            local_path
        ]
        self.info_log(
            "executing command: %s" %
            ' '.join(scp_command)
        )
        p = Popen(scp_command)
        p.wait()

    def execute_command(self, command):
        """Execute a command on the node

        Args:
            command (str)
        """

        self.info_log("executing command: %s" % command)

        try:
            ssh = paramiko.SSHClient()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

            k = paramiko.RSAKey.from_private_key_file(
                self.browser_config.get('ssh_key_path')
            )
            ssh.connect(
                self.private_ip,
                username=self.browser_config.get('username'),
                pkey=k
            )

            sleep_time = 0.1
            stdout = []
            stderr = []

            ssh_transport = ssh.get_transport()
            channel = ssh_transport.open_session()
            channel.setblocking(0)
            channel.exec_command(command)

            while True:

                while channel.recv_ready():
                    stdout.append(channel.recv(1000))

                while channel.recv_stderr_ready():
                    stderr.append(channel.recv_stderr(1000))

                if channel.exit_status_ready():
                    break

                sleep(sleep_time)

            # ret = channel.recv_exit_status()
            ssh_transport.close()

            ssh.close()

            return ''.join(stdout), ''.join(stderr)

        except Exception as e:
            msg = "Execute_command exception: %s" % str(e)
            self.error_log(msg)
            raise Exception(msg)<commit_msg>fix ;<commit_after>from subprocess import Popen
import os
from time import sleep

import paramiko
import boto.ec2

from brome.core.utils import (
    string_to_filename,
    devnull,
    create_dir_if_doesnt_exist
)
from brome.runner.base_instance import BaseInstance
from brome.core.settings import BROME_CONFIG


class EC2Instance(BaseInstance):
    """EC2 instance

    Attributes:
        runner (object)
        browser_config (object)
        index (int)
    """

    def __init__(self, runner, browser_config, index):
        self.runner = runner
        self.browser_config = browser_config
        self.index = index

    def get_ip(self):
        """Return the ip address of the node
        """

        return self.private_ip

    def scp_file_remote_to_local(self, remote_path, local_path):
        """Scp a remote file to local

        Args:
            remote_path (str)
            local_path (str)
        """

        scp_command = [
            'scp',
            '-o',
            'StrictHostKeyChecking=no',
            '-i',
            self.browser_config.get('ssh_key_path'),
            '%s@%s:"%s"' %
            (
                self.browser_config.get('username'),
                self.get_ip(),
                remote_path
            ),
            local_path
        ]
        self.info_log(
            "executing command: %s" %
            ' '.join(scp_command)
        )
        p = Popen(scp_command)
        p.wait()

    def execute_command(self, command):
        """Execute a command on the node

        Args:
            command (str)
        """

        self.info_log("executing command: %s" % command)

        try:
            ssh = paramiko.SSHClient()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

            k = paramiko.RSAKey.from_private_key_file(
                self.browser_config.get('ssh_key_path')
            )
            ssh.connect(
                self.private_ip,
                username=self.browser_config.get('username'),
                pkey=k
            )

            sleep_time = 0.1
            stdout = []
            stderr = []

            ssh_transport = ssh.get_transport()
            channel = ssh_transport.open_session()
            channel.setblocking(0)
            channel.exec_command(command)

            while True:

                while channel.recv_ready():
                    stdout.append(channel.recv(1000))

                while channel.recv_stderr_ready():
                    stderr.append(channel.recv_stderr(1000))

                if channel.exit_status_ready():
                    break

                sleep(sleep_time)

            # ret = channel.recv_exit_status()
            ssh_transport.close()

            ssh.close()

            return b''.join(stdout), b''.join(stderr)

        except Exception as e:
            msg = "Execute_command exception: %s" % str(e)
            self.error_log(msg)
            raise Exception(msg)
{'fraction_non_alphanumeric': 0.09358198924731183, 'fraction_numerical': 0.004704301075268817, 'mean_word_length': 1.5571305841924399, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4795274', 'n_tokens_mistral': 1738, 'n_tokens_neox': 1608, 'n_words': 326}