<commit_before>from . import Error
from . import logging_helper
from sanetime import sanetime
from traceback import format_exc

class Controller(object):
    def __init__(self, *arms):
        self.arms = arms
        self.log = logging_helper.get_log('cynq.controller')

    def cynq(self):
        self.started_at = sanetime()
        for arm in self.arms: arm.controller = self
        self.cynqable_arms = [arm for arm in self.arms if arm._pre_cynq()]
        if not self.cynqable_arms: return
        try:
            self._cynq_apis()
            self._cynq_local()
            self._cynq_apis()
            self._cynq_local()
            self._cynq_snapshot()
        except StandardError as err:
            self.log.error("giving up on entire cynq | err=%s" % format_exc(err))
        for arm in self.cynqable_arms: arm._post_cynq()
        self.log.info("completed | %s" % ' | '.join(['%s:[api:%s local:%s snapshot:%s]' % (a.spec.name, a.api.changes_tstr, a.local.changes_tstr, a.snapshot.changes_tstr) for a in self.arms]))
        self.log.info("timings | %s | total: %.2f" % (' | '.join(['%s.%s:%s:%.2f' % (a.spec.name, s.type_, t[0], t[1]) for a in self.arms for s in a.stores for t in s.timings if t[1] > 0.1]), (sanetime().ms-self.started_at.ms)/1000.0))
        return self

    def _cynq_apis(self):
        for arm in list(self.cynqable_arms):
            try:
                if len(self.cynqable_arms)>1: arm.local._clear_cache()
                arm._cynq_api()
            except StandardError as err:
                self.log.error("bailing on an arm cynq that threw up an exception during api cynq: err=%s" % format_exc(err))
                self.cynqable_arms.remove(arm)
                if not self.cynqable_arms: raise Error("no more arms left")

    def _cynq_local(self):
        for arm in list(self.cynqable_arms):
            try:
                if len(self.cynqable_arms)>1: arm.local._clear_cache()
                arm._cynq_local()
            except StandardError as err:
                self.log.error("bailing on an arm cynq that threw up an exception during snapshot cynq: err=%s" % format_exc(err))
                self.cynqable_arms.remove(arm)
                if not self.cynqable_arms: raise Error("no more arms left")

    def _cynq_snapshot(self):
        for arm in list(self.cynqable_arms):
            try:
                if len(self.cynqable_arms)>1: arm.local._clear_cache()
                arm._cynq_snapshot()
            except StandardError as err:
                self.log.error("bailing on an arm cynq that threw up an exception during snapshot cynq: err=%s" % format_exc(err))
                self.cynqable_arms.remove(arm)
                if not self.cynqable_arms: raise Error("no more arms left")

<commit_msg>some more reporting on cynq status<commit_after>from . import Error
from . import logging_helper
from sanetime import sanetime
from traceback import format_exc

class Controller(object):
    def __init__(self, *arms):
        self.arms = arms
        self.log = logging_helper.get_log('cynq.controller')

    def cynq(self):
        self.started_at = sanetime()
        for arm in self.arms: arm.controller = self
        self.cynqable_arms = [arm for arm in self.arms if arm._pre_cynq()]
        if not self.cynqable_arms: return
        self.starting_arms_count = len(self.cynqable_arms)
        try:
            self._cynq_apis()
            self._cynq_local()
            self._cynq_apis()
            self._cynq_local()
            self._cynq_snapshot()
        except StandardError as err:
            self.log.error("giving up on entire cynq | err=%s" % format_exc(err))
        for arm in self.cynqable_arms: arm._post_cynq()
        self.log.info("completed | %s" % ' | '.join(['%s:[api:%s local:%s snapshot:%s]' % (a.spec.name, a.api.changes_tstr, a.local.changes_tstr, a.snapshot.changes_tstr) for a in self.arms]))
        self.log.info("timings | %s | total: %.2f" % (' | '.join(['%s.%s:%s:%.2f' % (a.spec.name, s.type_, t[0], t[1]) for a in self.arms for s in a.stores for t in s.timings if t[1] > 0.1]), (sanetime().ms-self.started_at.ms)/1000.0))
        return self

    def _cynq_apis(self):
        for arm in list(self.cynqable_arms):
            try:
                if len(self.cynqable_arms)>1: arm.local._clear_cache()
                arm._cynq_api()
            except StandardError as err:
                self.log.error("bailing on an arm cynq that threw up an exception during api cynq: err=%s" % format_exc(err))
                self.cynqable_arms.remove(arm)
                if not self.cynqable_arms: raise Error("no more arms left")

    def _cynq_local(self):
        for arm in list(self.cynqable_arms):
            try:
                if len(self.cynqable_arms)>1: arm.local._clear_cache()
                arm._cynq_local()
            except StandardError as err:
                self.log.error("bailing on an arm cynq that threw up an exception during snapshot cynq: err=%s" % format_exc(err))
                self.cynqable_arms.remove(arm)
                if not self.cynqable_arms: raise Error("no more arms left")

    def _cynq_snapshot(self):
        for arm in list(self.cynqable_arms):
            try:
                if len(self.cynqable_arms)>1: arm.local._clear_cache()
                arm._cynq_snapshot()
            except StandardError as err:
                self.log.error("bailing on an arm cynq that threw up an exception during snapshot cynq: err=%s" % format_exc(err))
                self.cynqable_arms.remove(arm)
                if not self.cynqable_arms: raise Error("no more arms left")

    def _get_complete_success(self):
        return len(self.cynqable_arms) == len(self.starting_arms_count) and self.started_at
    complete_success = property(_get_complete_success)


{'fraction_non_alphanumeric': 0.11189533482527113, 'fraction_numerical': 0.0051644000688586674, 'mean_word_length': 2.278781038374718, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '872237', 'n_tokens_mistral': 1940, 'n_tokens_neox': 1867, 'n_words': 464}