<commit_before>import argparse
import doctest
import sys
import subprocess
import time
import os
import os.path
import random
from datetime import datetime

def getDockerRunnerTypeAndConfigStr(dockerRunnerArg):
    """
    >>> getDockerRunnerTypeAndConfigStr('drType:drConfig')
    ('drType', 'drConfig')
    """
    idx = dockerRunnerArg.find(":")
    assert idx != -1
    drType = dockerRunnerArg[:idx]
    drConfig = dockerRunnerArg[idx + 1:]
    return (drType, drConfig)

class DockerRunner(object):
    def __init__(self, configStr, args):
        self._configStr = configStr
        self._args = args
        self._dist = None 

    """
    Get ips for hosts (ordered - with respect to hostId)
    """
    def getHostIps(self):
        raise NotImplementedError

    """
    Prepare environment
    """
    def prepare(self, dist):
        raise NotImplementedError

    """
    Run process on hostId
    Returns processId
    """
    def runDockerCmd(self, hostId, cmd): 
        raise NotImplementedError

    """
    Cleanup environment
    """
    def cleanup(self):
        raise NotImplementedError

class NativeDockerRunner(DockerRunner):
    def __init__(self, configStr, args):
        super(NativeDockerRunner, self).__init__(configStr, args)
        self._numHosts = int(self._configStr)

    def getHostIps(self):
        return ['127.0.0.1'] * self._numHosts

    def prepare(self, dist):
        self._dist = dist
        self.cleanup()
        for ins in self._dist:
            zkStorageDir=format("/var/jopamaTest/storage/%d/ZOOKEEPER" % ins.gId)
            subprocess.check_call('mkdir -p %s' % zkStorageDir, shell=True)
            zkLogsDir=format("/var/jopamaTest/logs/%d/ZOOKEEPER" % ins.gId)
            subprocess.check_call('mkdir -p %s' % zkLogsDir, shell=True)
            if ins.runTP:
                tpLogsDir=format("/var/jopamaTest/logs/%d/TP" % ins.gId)
                subprocess.check_call('mkdir -p %s' % tpLogsDir, shell=True)
            if ins.runTC:
                tcLogsDir=format("/var/jopamaTest/logs/%d/TC" % ins.gId)
                subprocess.check_call('mkdir -p %s' % tcLogsDir, shell=True)
        ccLogsDir=format("/var/jopamaTest/logs/CC")
        tvLogsDir=format("/var/jopamaTest/logs/TV")
        for dirToCreate in [ccLogsDir, tvLogsDir]:
            subprocess.check_call('mkdir -p %s' % dirToCreate, shell=True)

    def runDockerCmd(self, hostId, cmd):
        print('NDR::runDockerCmd, hostId: %d cmd: %s' % (hostId, cmd, ))
        return subprocess.check_output(cmd, shell=True)

    def saveResults(self):
        for ins in self._dist:
            if ins.runTP:
                remoteStatsPath=format("/var/jopamaTest/logs/%d/TP/stats.log" % ins.gId)
                statsTargetDir=format("%s/stats/%d/TP" % (self._args.outputDir, ins.gId))
                subprocess.check_call('mkdir -p %s' % statsTargetDir, shell=True)
                subprocess.check_call('cp %s %s' % (remoteStatsPath, statsTargetDir), shell=True)
            if self._args.alllogs:
                if ins.runTP:
                    tpLogsPath=format("/var/jopamaTest/logs/%d/TP" % ins.gId)
                    tpTargetDirPath=format("%s/%d" % (self._args.outputDir, ins.gId))
                    subprocess.check_call('mkdir -p %s' % tpTargetDirPath, shell=True)
                    subprocess.check_call('cp -r %s %s' % (tpLogsPath, tpTargetDirPath), shell=True)
                tvLogsPath="/var/jopamaTest/logs/TV"
                tvTargetDirPath=self._args.outputDir
                subprocess.check_call('cp -r %s %s' % (tvLogsPath, tvTargetDirPath), shell=True)

    def cleanup(self):
        subprocess.check_call("docker ps -aq | while read line; do docker kill $line; docker rm $line; done", shell=True)
        subprocess.check_call("sudo rm -fr /var/jopamaTest/*", shell=True)

class DockerMachineDockerRunner(DockerRunner):
    def __init__(self, configStr, args):
        super(DockerMachineDockerRunner, self).__init__(configStr, args)
        args = self._configStr.split(';')
        assert len(args) == 2
        self._ifName = args[0]
        self._dmNames = args[1].split(',')
        self._dmIps = []
        for dmName in self._dmNames:
            ip = subprocess.check_output('docker-machine ssh %s ip -4 -o addr show dev %s | cut -d" " -f7 | cut -d"/" -f1' % (dmName, self._ifName), shell=True)
            self._dmIps.append(ip.strip())

    def getHostIps(self):
        return self._dmIps

    def _execOnHostId(self, hostId, cmd):
        print('DMDR::_execOnHostId, hostId: %d cmd: %s' % (hostId, cmd, ))
        return subprocess.check_output("docker-machine ssh %s sh -c \"\\\"%s\\\"\"" % (self._dmNames[hostId], cmd), shell=True)

    def prepare(self, dist):
        print("prepare")
        self._dist = dist
        print("cleanup")
        self.cleanup()
        print("hostIdsAndDirs init")
        hostIdsAndDirs = []
        for ins in self._dist:
            zkStorageDir=format("/var/jopamaTest/storage/%d/ZOOKEEPER" % ins.gId)
            hostIdsAndDirs.append((ins.hostId, zkStorageDir))
            zkLogsDir=format("/var/jopamaTest/logs/%d/ZOOKEEPER" % ins.gId)
            hostIdsAndDirs.append((ins.hostId, zkLogsDir))
            if ins.runTP:
                tpLogsDir=format("/var/jopamaTest/logs/%d/TP" % ins.gId)
                hostIdsAndDirs.append((ins.hostId, tpLogsDir))
            if ins.runTC:
                tcLogsDir=format("/var/jopamaTest/logs/%d/TC" % ins.gId)
                hostIdsAndDirs.append((ins.hostId, tcLogsDir))
        ccLogsDir=format("/var/jopamaTest/logs/CC")
        hostIdsAndDirs.append((0, ccLogsDir)) 
        tvLogsDir=format("/var/jopamaTest/logs/TV")
        hostIdsAndDirs.append((0, tvLogsDir)) 
        for hostId, dirPath in hostIdsAndDirs:
            self._execOnHostId(hostId, "mkdir -p %s" % (dirPath, )) 

    def runDockerCmd(self, hostId, cmd):
        print('DMDR::runDockerCmd, hostId: %d cmd: %s' % (hostId, cmd, ))
        return subprocess.check_output("eval $(docker-machine env %s); %s; eval $(docker-machine env -u);" % (self._dmNames[hostId], cmd), shell=True)

    def saveResults(self):
        for ins in self._dist:
            if ins.runTP:
                remoteStatsPath=format("/var/jopamaTest/logs/%d/TP/stats.log" % ins.gId)
                statsTargetDir=format("%s/stats/%d/TP" % (self._args.outputDir, ins.gId))
                subprocess.check_call('mkdir -p %s' % statsTargetDir, shell=True)
                subprocess.check_call('docker-machine scp %s:%s %s' % (self._dmNames[ins.hostId], remoteStatsPath, statsTargetDir), shell=True)
            if self._args.alllogs:
                if ins.runTP:
                    tpLogsPath=format("/var/jopamaTest/logs/%d/TP" % ins.gId)
                    tpTargetDirPath=format("%s/%d" % (self._args.outputDir, ins.gId))
                    subprocess.check_call('mkdir -p %s' % tpTargetDirPath, shell=True)
                    subprocess.check_call('docker-machine scp -r %s:%s %s' % (self._dmNames[ins.hostId], tpLogsPath, tpTargetDirPath), shell=True)
                tvLogsPath="/var/jopamaTest/logs/TV"
                tvTargetDirPath=self._args.outputDir
                subprocess.check_call('docker-machine scp -r %s:%s %s' % (self._dmNames[0], tvLogsPath, tvTargetDirPath), shell=True)

    def cleanup(self):
        for hostId in range(len(self._dmNames)):
            lines = self.runDockerCmd(hostId, "docker ps -aq").splitlines()
            for line in lines:
                self.runDockerCmd(hostId, "docker kill %s; true" % (line))
                self.runDockerCmd(hostId, "docker rm %s" % (line))
            self._execOnHostId(hostId, "sudo rm -fr /var/jopamaTest/*")<commit_msg>DockerMachineDockerRunner cleanup speeded up - by running single shell command over ssh instead of multiple commands.<commit_after>import argparse
import doctest
import sys
import subprocess
import time
import os
import os.path
import random
from datetime import datetime

def getDockerRunnerTypeAndConfigStr(dockerRunnerArg):
    """
    >>> getDockerRunnerTypeAndConfigStr('drType:drConfig')
    ('drType', 'drConfig')
    """
    idx = dockerRunnerArg.find(":")
    assert idx != -1
    drType = dockerRunnerArg[:idx]
    drConfig = dockerRunnerArg[idx + 1:]
    return (drType, drConfig)

class DockerRunner(object):
    def __init__(self, configStr, args):
        self._configStr = configStr
        self._args = args
        self._dist = None 

    """
    Get ips for hosts (ordered - with respect to hostId)
    """
    def getHostIps(self):
        raise NotImplementedError

    """
    Prepare environment
    """
    def prepare(self, dist):
        raise NotImplementedError

    """
    Run process on hostId
    Returns processId
    """
    def runDockerCmd(self, hostId, cmd): 
        raise NotImplementedError

    """
    Cleanup environment
    """
    def cleanup(self):
        raise NotImplementedError

class NativeDockerRunner(DockerRunner):
    def __init__(self, configStr, args):
        super(NativeDockerRunner, self).__init__(configStr, args)
        self._numHosts = int(self._configStr)

    def getHostIps(self):
        return ['127.0.0.1'] * self._numHosts

    def prepare(self, dist):
        self._dist = dist
        self.cleanup()
        for ins in self._dist:
            zkStorageDir=format("/var/jopamaTest/storage/%d/ZOOKEEPER" % ins.gId)
            subprocess.check_call('mkdir -p %s' % zkStorageDir, shell=True)
            zkLogsDir=format("/var/jopamaTest/logs/%d/ZOOKEEPER" % ins.gId)
            subprocess.check_call('mkdir -p %s' % zkLogsDir, shell=True)
            if ins.runTP:
                tpLogsDir=format("/var/jopamaTest/logs/%d/TP" % ins.gId)
                subprocess.check_call('mkdir -p %s' % tpLogsDir, shell=True)
            if ins.runTC:
                tcLogsDir=format("/var/jopamaTest/logs/%d/TC" % ins.gId)
                subprocess.check_call('mkdir -p %s' % tcLogsDir, shell=True)
        ccLogsDir=format("/var/jopamaTest/logs/CC")
        tvLogsDir=format("/var/jopamaTest/logs/TV")
        for dirToCreate in [ccLogsDir, tvLogsDir]:
            subprocess.check_call('mkdir -p %s' % dirToCreate, shell=True)

    def runDockerCmd(self, hostId, cmd):
        print('NDR::runDockerCmd, hostId: %d cmd: %s' % (hostId, cmd, ))
        return subprocess.check_output(cmd, shell=True)

    def saveResults(self):
        for ins in self._dist:
            if ins.runTP:
                remoteStatsPath=format("/var/jopamaTest/logs/%d/TP/stats.log" % ins.gId)
                statsTargetDir=format("%s/stats/%d/TP" % (self._args.outputDir, ins.gId))
                subprocess.check_call('mkdir -p %s' % statsTargetDir, shell=True)
                subprocess.check_call('cp %s %s' % (remoteStatsPath, statsTargetDir), shell=True)
            if self._args.alllogs:
                if ins.runTP:
                    tpLogsPath=format("/var/jopamaTest/logs/%d/TP" % ins.gId)
                    tpTargetDirPath=format("%s/%d" % (self._args.outputDir, ins.gId))
                    subprocess.check_call('mkdir -p %s' % tpTargetDirPath, shell=True)
                    subprocess.check_call('cp -r %s %s' % (tpLogsPath, tpTargetDirPath), shell=True)
                tvLogsPath="/var/jopamaTest/logs/TV"
                tvTargetDirPath=self._args.outputDir
                subprocess.check_call('cp -r %s %s' % (tvLogsPath, tvTargetDirPath), shell=True)

    def cleanup(self):
        subprocess.check_call("docker ps -aq | while read line; do docker kill $line; docker rm $line; done", shell=True)
        subprocess.check_call("sudo rm -fr /var/jopamaTest/*", shell=True)

class DockerMachineDockerRunner(DockerRunner):
    def __init__(self, configStr, args):
        super(DockerMachineDockerRunner, self).__init__(configStr, args)
        args = self._configStr.split(';')
        assert len(args) == 2
        self._ifName = args[0]
        self._dmNames = args[1].split(',')
        self._dmIps = []
        for dmName in self._dmNames:
            ip = subprocess.check_output('docker-machine ssh %s ip -4 -o addr show dev %s | cut -d" " -f7 | cut -d"/" -f1' % (dmName, self._ifName), shell=True)
            self._dmIps.append(ip.strip())

    def getHostIps(self):
        return self._dmIps

    def _execOnHostId(self, hostId, cmd):
        print('DMDR::_execOnHostId, hostId: %d cmd: %s' % (hostId, cmd, ))
        return subprocess.check_output("docker-machine ssh %s sh -c \"\\\"%s\\\"\"" % (self._dmNames[hostId], cmd), shell=True)

    def prepare(self, dist):
        print("prepare")
        self._dist = dist
        print("cleanup")
        self.cleanup()
        print("hostIdsAndDirs init")
        hostIdsAndDirs = []
        for ins in self._dist:
            zkStorageDir=format("/var/jopamaTest/storage/%d/ZOOKEEPER" % ins.gId)
            hostIdsAndDirs.append((ins.hostId, zkStorageDir))
            zkLogsDir=format("/var/jopamaTest/logs/%d/ZOOKEEPER" % ins.gId)
            hostIdsAndDirs.append((ins.hostId, zkLogsDir))
            if ins.runTP:
                tpLogsDir=format("/var/jopamaTest/logs/%d/TP" % ins.gId)
                hostIdsAndDirs.append((ins.hostId, tpLogsDir))
            if ins.runTC:
                tcLogsDir=format("/var/jopamaTest/logs/%d/TC" % ins.gId)
                hostIdsAndDirs.append((ins.hostId, tcLogsDir))
        ccLogsDir=format("/var/jopamaTest/logs/CC")
        hostIdsAndDirs.append((0, ccLogsDir)) 
        tvLogsDir=format("/var/jopamaTest/logs/TV")
        hostIdsAndDirs.append((0, tvLogsDir)) 
        for hostId, dirPath in hostIdsAndDirs:
            self._execOnHostId(hostId, "mkdir -p %s" % (dirPath, )) 

    def runDockerCmd(self, hostId, cmd):
        print('DMDR::runDockerCmd, hostId: %d cmd: %s' % (hostId, cmd, ))
        return subprocess.check_output("eval $(docker-machine env %s); %s; eval $(docker-machine env -u);" % (self._dmNames[hostId], cmd), shell=True)

    def saveResults(self):
        for ins in self._dist:
            if ins.runTP:
                remoteStatsPath=format("/var/jopamaTest/logs/%d/TP/stats.log" % ins.gId)
                statsTargetDir=format("%s/stats/%d/TP" % (self._args.outputDir, ins.gId))
                subprocess.check_call('mkdir -p %s' % statsTargetDir, shell=True)
                subprocess.check_call('docker-machine scp %s:%s %s' % (self._dmNames[ins.hostId], remoteStatsPath, statsTargetDir), shell=True)
            if self._args.alllogs:
                if ins.runTP:
                    tpLogsPath=format("/var/jopamaTest/logs/%d/TP" % ins.gId)
                    tpTargetDirPath=format("%s/%d" % (self._args.outputDir, ins.gId))
                    subprocess.check_call('mkdir -p %s' % tpTargetDirPath, shell=True)
                    subprocess.check_call('docker-machine scp -r %s:%s %s' % (self._dmNames[ins.hostId], tpLogsPath, tpTargetDirPath), shell=True)
                tvLogsPath="/var/jopamaTest/logs/TV"
                tvTargetDirPath=self._args.outputDir
                subprocess.check_call('docker-machine scp -r %s:%s %s' % (self._dmNames[0], tvLogsPath, tvTargetDirPath), shell=True)

    def cleanup(self):
        for hostId in range(len(self._dmNames)):
            self._execOnHostId(hostId, "sudo docker ps -aq | while read line; do sudo docker kill \\\\\\$line; sudo docker rm \\\\\\$line; done")
            self._execOnHostId(hostId, "sudo rm -fr /var/jopamaTest/*")
{'fraction_non_alphanumeric': 0.13375672347871168, 'fraction_numerical': 0.0022033568790097856, 'mean_word_length': 2.4608656649472977, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5152480', 'n_tokens_mistral': 5202, 'n_tokens_neox': 5005, 'n_words': 977}