<commit_before>#!/usr/bin/python

import sys
import subprocess
import os
import stat
from config import ClusterConfig
from run import ClusterRun, ClusterRunConcatCommands, ClusterRunFailed, ClusterRunSummaryCode
from scp import ClusterSCP

class ClusterBuild:
    def __init__(self, config):
        self.config = config
        self.patchmail_file = '.commits.patch'
        self.patch_file = '.changes.patch'

    def cd_to_code(self):
        return "cd " + self.config.code_dir

    def cd_to_build(self):
        return "cd build/cmake"

    def cd_to_scripts(self):
        return "cd scripts"

    def destroy(self):
        destroy_cmd = "rm -rf " + self.config.code_dir
        retcodes = ClusterRun(self.config, destroy_cmd)
        return ClusterRunSummaryCode(retcodes)

    def checkout(self):
        checkout_cmd = "git clone <EMAIL>:cbr.git " + self.config.code_dir
        retcodes = ClusterRun(self.config, checkout_cmd)
        return ClusterRunSummaryCode(retcodes)

    def update(self):
        cd_cmd = self.cd_to_code()
        pull_cmd = "git pull origin"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, pull_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def apply_patch(self, patch_file):
        ClusterSCP(self.config, [patch_file, "remote:"+self.config.code_dir+"/"+patch_file])
        cd_cmd = self.cd_to_code()
        patch_cmd = "patch -p1 < " + patch_file
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, patch_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def apply_patch_mail(self, patch_file):
        ClusterSCP(self.config, [patch_file, "remote:"+self.config.code_dir+"/"+patch_file])
        cd_cmd = self.cd_to_code()
        patch_cmd = "git am " + patch_file
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, patch_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def reset_to_head(self):
        cd_cmd = self.cd_to_code()
        reset_cmd = "git reset --hard HEAD"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, reset_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def reset_to_origin_head(self):
        cd_cmd = self.cd_to_code()
        reset_cmd = "git reset --hard origin/HEAD"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, reset_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def git_clean(self):
        cd_cmd = self.cd_to_code()
        # note: we put clean_garbage before clean so the latter won't complain about these directories
        clean_garbage_cmd = "rm -rf .dotest" # FIXME there's probably other types of garbage
        clean_cmd = "git clean -f"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, clean_garbage_cmd, clean_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def dependencies(self, which = None):
        cd_cmd = self.cd_to_code()
        build_cmd = "./install-deps.sh"
        if which != None:
            for dep in which:
                build_cmd += " " + dep
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, build_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def update_dependencies(self, which = None):
        cd_cmd = self.cd_to_code()
        update_cmd = "./install-deps.sh update"
        if which != None:
            for dep in which:
                update_cmd += " " + dep
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, update_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def ccache_args(self):
        if (not self.config.ccache):
            return ''

        # FIXME caching this somewhere would probably be wise...
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands(["ls /usr/bin/ccache /usr/bin/g++ /usr/bin/gcc &> /dev/null"]))
        # FIXME we should do this per-node, but cluster_run doesn't support that yet for per-node runs
        if (ClusterRunSummaryCode(retcodes) == 0):
            # We have all the pieces we need
            return 'CC="/usr/bin/ccache /usr/bin/gcc" CXX="/usr/bin/ccache /usr/bin/g++"'

        print 'Warning: Running without ccache!'
        return ""

    def build(self, build_type = 'Default', with_timestamp = True):
        cd_code_cmd = self.cd_to_code()
        cd_build_cmd = self.cd_to_build()
        build_cmd = "%s cmake -DCMAKE_BUILD_TYPE=%s -DCBR_TIMESTAMP_PACKETS=%s ." % (self.ccache_args(), build_type, str(with_timestamp))
        make_cmd = "make -j2"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_code_cmd, cd_build_cmd, build_cmd, make_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def clean(self):
        cd_code_cmd = self.cd_to_code()
        cd_build_cmd = self.cd_to_build()
        clean_cmd = "make clean && rm CMakeCache.txt"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_code_cmd, cd_build_cmd, clean_cmd]))
        return ClusterRunSummaryCode(retcodes)

    # generates a patchset based on changes made to tree locally
    def create_patchset(self):
        # first generate a patchmail
        commits_patch_file = open(self.patchmail_file, 'w')
        formatpatch_ret = subprocess.call(['git', 'format-patch', '--stdout', 'origin/master'], 0, None, None, commits_patch_file)
        if (formatpatch_ret != 0):
            return formatpatch_ret

        # then generate a diff
        changes_patch_file = open(self.patch_file, 'w')
        diff_ret = subprocess.call(['git', 'diff'], 0, None, None, changes_patch_file)
        changes_patch_file.close()
        return diff_ret

    # applies a patchset to all nodes
    def apply_patchset(self):
        # just use revert_patchset to make sure we're in a clean state
        revert_ret = self.revert_patchset()
        if (revert_ret != 0):
            return revert_ret

        # Setup all possible necessary commands, then filter out based on files
        file_cmds = [ ('git am', self.patchmail_file),
                      ('patch -p1 <', self.patch_file)
                      ]
        to_do = [ (cmd,file) for (cmd,file) in file_cmds if os.stat(file)[stat.ST_SIZE] > 0 ]

        # Copy files over
        scp_args = [ file for (cmd,file) in to_do ]<commit_msg>Fix missing patch file close in cluster/build.<commit_after>#!/usr/bin/python

import sys
import subprocess
import os
import stat
from config import ClusterConfig
from run import ClusterRun, ClusterRunConcatCommands, ClusterRunFailed, ClusterRunSummaryCode
from scp import ClusterSCP

class ClusterBuild:
    def __init__(self, config):
        self.config = config
        self.patchmail_file = '.commits.patch'
        self.patch_file = '.changes.patch'

    def cd_to_code(self):
        return "cd " + self.config.code_dir

    def cd_to_build(self):
        return "cd build/cmake"

    def cd_to_scripts(self):
        return "cd scripts"

    def destroy(self):
        destroy_cmd = "rm -rf " + self.config.code_dir
        retcodes = ClusterRun(self.config, destroy_cmd)
        return ClusterRunSummaryCode(retcodes)

    def checkout(self):
        checkout_cmd = "git clone <EMAIL>:cbr.git " + self.config.code_dir
        retcodes = ClusterRun(self.config, checkout_cmd)
        return ClusterRunSummaryCode(retcodes)

    def update(self):
        cd_cmd = self.cd_to_code()
        pull_cmd = "git pull origin"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, pull_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def apply_patch(self, patch_file):
        ClusterSCP(self.config, [patch_file, "remote:"+self.config.code_dir+"/"+patch_file])
        cd_cmd = self.cd_to_code()
        patch_cmd = "patch -p1 < " + patch_file
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, patch_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def apply_patch_mail(self, patch_file):
        ClusterSCP(self.config, [patch_file, "remote:"+self.config.code_dir+"/"+patch_file])
        cd_cmd = self.cd_to_code()
        patch_cmd = "git am " + patch_file
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, patch_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def reset_to_head(self):
        cd_cmd = self.cd_to_code()
        reset_cmd = "git reset --hard HEAD"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, reset_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def reset_to_origin_head(self):
        cd_cmd = self.cd_to_code()
        reset_cmd = "git reset --hard origin/HEAD"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, reset_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def git_clean(self):
        cd_cmd = self.cd_to_code()
        # note: we put clean_garbage before clean so the latter won't complain about these directories
        clean_garbage_cmd = "rm -rf .dotest" # FIXME there's probably other types of garbage
        clean_cmd = "git clean -f"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, clean_garbage_cmd, clean_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def dependencies(self, which = None):
        cd_cmd = self.cd_to_code()
        build_cmd = "./install-deps.sh"
        if which != None:
            for dep in which:
                build_cmd += " " + dep
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, build_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def update_dependencies(self, which = None):
        cd_cmd = self.cd_to_code()
        update_cmd = "./install-deps.sh update"
        if which != None:
            for dep in which:
                update_cmd += " " + dep
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_cmd, update_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def ccache_args(self):
        if (not self.config.ccache):
            return ''

        # FIXME caching this somewhere would probably be wise...
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands(["ls /usr/bin/ccache /usr/bin/g++ /usr/bin/gcc &> /dev/null"]))
        # FIXME we should do this per-node, but cluster_run doesn't support that yet for per-node runs
        if (ClusterRunSummaryCode(retcodes) == 0):
            # We have all the pieces we need
            return 'CC="/usr/bin/ccache /usr/bin/gcc" CXX="/usr/bin/ccache /usr/bin/g++"'

        print 'Warning: Running without ccache!'
        return ""

    def build(self, build_type = 'Default', with_timestamp = True):
        cd_code_cmd = self.cd_to_code()
        cd_build_cmd = self.cd_to_build()
        build_cmd = "%s cmake -DCMAKE_BUILD_TYPE=%s -DCBR_TIMESTAMP_PACKETS=%s ." % (self.ccache_args(), build_type, str(with_timestamp))
        make_cmd = "make -j2"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_code_cmd, cd_build_cmd, build_cmd, make_cmd]))
        return ClusterRunSummaryCode(retcodes)

    def clean(self):
        cd_code_cmd = self.cd_to_code()
        cd_build_cmd = self.cd_to_build()
        clean_cmd = "make clean && rm CMakeCache.txt"
        retcodes = ClusterRun(self.config, ClusterRunConcatCommands([cd_code_cmd, cd_build_cmd, clean_cmd]))
        return ClusterRunSummaryCode(retcodes)

    # generates a patchset based on changes made to tree locally
    def create_patchset(self):
        # first generate a patchmail
        commits_patch_file = open(self.patchmail_file, 'w')
        formatpatch_ret = subprocess.call(['git', 'format-patch', '--stdout', 'origin/master'], 0, None, None, commits_patch_file)
        commits_patch_file.close()
        if (formatpatch_ret != 0):
            return formatpatch_ret

        # then generate a diff
        changes_patch_file = open(self.patch_file, 'w')
        diff_ret = subprocess.call(['git', 'diff'], 0, None, None, changes_patch_file)
        changes_patch_file.close()
        return diff_ret

    # applies a patchset to all nodes
    def apply_patchset(self):
        # just use revert_patchset to make sure we're in a clean state
        revert_ret = self.revert_patchset()
        if (revert_ret != 0):
            return revert_ret

        # Setup all possible necessary commands, then filter out based on files
        file_cmds = [ ('git am', self.patchmail_file),
                      ('patch -p1 <', self.patch_file)
                      ]
        to_do = [ (cmd,file) for (cmd,file) in file_cmds if os.stat(file)[stat.ST_SIZE] > 0 ]

        # Copy files over
        scp_args = [ file for (cmd,file) in to_do ]
{'fraction_non_alphanumeric': 0.10195552751931872, 'fraction_numerical': 0.0014193344898281027, 'mean_word_length': 2.872671755725191, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '482157', 'n_tokens_mistral': 4002, 'n_tokens_neox': 3837, 'n_words': 919}