<commit_before>"""Command to process multiple repos."""
# =============================================================================
# CONTENTS
# -----------------------------------------------------------------------------
# abdcmd_processrepos
#
# Public Classes:
#   DelayedRetrySleepOperation
#    .do
#   RefreshCachesOperation
#    .do
#   ResetFileException
#   FileCheckOperation
#    .do
#
# Public Functions:
#   getFromfilePrefixChars
#   setupParser
#   tryHandleSpecialFiles
#   process
#
# -----------------------------------------------------------------------------
# (this contents block is generated, edits will be lost)
# =============================================================================

from __future__ import absolute_import

import argparse
import contextlib
import datetime
import functools
import itertools
import logging
import os
import sys
import time

import phlsys_scheduleunreliables
import phlsys_statusline
import phlsys_tryloop
import phlurl_watcher

import abdi_processargs
import abdt_arcydreporter
import abdt_shareddictoutput


def getFromfilePrefixChars():
    return None


def setupParser(parser):
    abdi_processargs.setup_parser(parser)
    parser.add_argument(
        '--repo-configs',
        metavar="N",
        action='append',
        nargs='+',
        type=str,
        help="files to load configuration from, prefix with @")
    parser.add_argument(
        '--status-path',
        metavar="PATH",
        type=str,
        required=True,
        help="file to write status to")
    parser.add_argument(
        '--kill-file',
        metavar="NAME",
        type=str,
        help="filename to watch for, will stop operations safely if the file "
             "is detected.")
    parser.add_argument(
        '--pause-file',
        metavar="NAME",
        type=str,
        help="filename to watch for, will pause operations while the file is "
             "detected.")
    parser.add_argument(
        '--reset-file',
        metavar="NAME",
        type=str,
        help="filename to watch for, will reset operations if the file is "
             "detected and remove the file.")
    parser.add_argument(
        '--sleep-secs',
        metavar="TIME",
        type=int,
        default=60,
        help="time to wait between runs through the list")
    parser.add_argument(
        '--no-loop',
        action='store_true',
        help="supply this argument to only process each repo once then exit")


class DelayedRetrySleepOperation(object):

    def __init__(self, out, secs, reporter):
        self._out = out
        self._secs = secs
        self._reporter = reporter

    def do(self):
        sleep_remaining = self._secs
        self._reporter.start_sleep(sleep_remaining)
        while sleep_remaining > 0:
            self._out.display("sleep (" + str(sleep_remaining) + " seconds) ")
            self._reporter.update_sleep(sleep_remaining)
            time.sleep(1)
            sleep_remaining -= 1
        self._reporter.finish_sleep()
        return True


class RefreshCachesOperation(object):

    def __init__(self, conduits, url_watcher, reporter):
        super(RefreshCachesOperation, self).__init__()
        self._conduits = conduits
        self._url_watcher = url_watcher
        self._reporter = reporter

    def do(self):
        self._reporter.start_cache_refresh()

        # prepare delays in the event of trouble when refreshing
        # TODO: perhaps this policy should be decided higher-up
        delays = [
            datetime.timedelta(seconds=1),
            datetime.timedelta(seconds=1),
            datetime.timedelta(seconds=1),
            datetime.timedelta(seconds=1),
            datetime.timedelta(seconds=1),
            datetime.timedelta(seconds=10),
            datetime.timedelta(seconds=10),
            datetime.timedelta(seconds=10),
            datetime.timedelta(seconds=10),
            datetime.timedelta(seconds=10),
            datetime.timedelta(seconds=10),
            datetime.timedelta(minutes=1),
            datetime.timedelta(minutes=1),
            datetime.timedelta(minutes=1),
            datetime.timedelta(minutes=1),
            datetime.timedelta(minutes=1),
            datetime.timedelta(minutes=10),
            datetime.timedelta(minutes=10),
            datetime.timedelta(minutes=10),
            datetime.timedelta(minutes=10),
            datetime.timedelta(minutes=10),
            datetime.timedelta(minutes=10),
        ]

        forever = itertools.repeat(datetime.timedelta(minutes=10))
        delays = itertools.chain(delays, forever)

        # log.error if we get an exception when fetching
        def on_tryloop_exception_phab(e, delay):
            self._reporter.on_tryloop_exception(e, delay)
            self._reporter.log_system_exception('conduit-refresh', '', e)
            logging.error(str(e) + "\nwill wait " + str(delay))

        with self._reporter.tag_timer_context('refresh conduit cache'):
            for key in self._conduits:
                conduit = self._conduits[key]
                phlsys_tryloop.try_loop_delay(
                    conduit.refresh_cache_on_cycle,
                    delays,
                    onException=on_tryloop_exception_phab)

        # log.error if we get an exception when fetching
        def on_tryloop_exception_git(e, delay):
            self._reporter.on_tryloop_exception(e, delay)
            self._reporter.log_system_exception('git-snoop', '', e)
            logging.error(str(e) + "\nwill wait " + str(delay))

        with self._reporter.tag_timer_context('refresh git watcher'):
            phlsys_tryloop.try_loop_delay(
                self._url_watcher.refresh,
                delays,
                onException=on_tryloop_exception_git)

        self._reporter.finish_cache_refresh()
        return True


class ResetFileException(Exception):
<commit_msg>Arcyd: refactor tryloops in 'processrepos'<commit_after>"""Command to process multiple repos."""
# =============================================================================
# CONTENTS
# -----------------------------------------------------------------------------
# abdcmd_processrepos
#
# Public Classes:
#   DelayedRetrySleepOperation
#    .do
#   RefreshCachesOperation
#    .do
#   ResetFileException
#   FileCheckOperation
#    .do
#
# Public Functions:
#   getFromfilePrefixChars
#   setupParser
#   tryHandleSpecialFiles
#   process
#
# -----------------------------------------------------------------------------
# (this contents block is generated, edits will be lost)
# =============================================================================

from __future__ import absolute_import

import argparse
import contextlib
import functools
import logging
import os
import sys
import time

import phlsys_scheduleunreliables
import phlsys_statusline
import phlsys_tryloop
import phlurl_watcher

import abdi_processargs
import abdt_arcydreporter
import abdt_shareddictoutput


def getFromfilePrefixChars():
    return None


def setupParser(parser):
    abdi_processargs.setup_parser(parser)
    parser.add_argument(
        '--repo-configs',
        metavar="N",
        action='append',
        nargs='+',
        type=str,
        help="files to load configuration from, prefix with @")
    parser.add_argument(
        '--status-path',
        metavar="PATH",
        type=str,
        required=True,
        help="file to write status to")
    parser.add_argument(
        '--kill-file',
        metavar="NAME",
        type=str,
        help="filename to watch for, will stop operations safely if the file "
             "is detected.")
    parser.add_argument(
        '--pause-file',
        metavar="NAME",
        type=str,
        help="filename to watch for, will pause operations while the file is "
             "detected.")
    parser.add_argument(
        '--reset-file',
        metavar="NAME",
        type=str,
        help="filename to watch for, will reset operations if the file is "
             "detected and remove the file.")
    parser.add_argument(
        '--sleep-secs',
        metavar="TIME",
        type=int,
        default=60,
        help="time to wait between runs through the list")
    parser.add_argument(
        '--no-loop',
        action='store_true',
        help="supply this argument to only process each repo once then exit")


class DelayedRetrySleepOperation(object):

    def __init__(self, out, secs, reporter):
        self._out = out
        self._secs = secs
        self._reporter = reporter

    def do(self):
        sleep_remaining = self._secs
        self._reporter.start_sleep(sleep_remaining)
        while sleep_remaining > 0:
            self._out.display("sleep (" + str(sleep_remaining) + " seconds) ")
            self._reporter.update_sleep(sleep_remaining)
            time.sleep(1)
            sleep_remaining -= 1
        self._reporter.finish_sleep()
        return True


def _critical_tryloop(f, identifier, detail, reporter):
    def on_tryloop_exception(e, delay):
        reporter.on_tryloop_exception(e, delay)
        reporter.log_system_exception(identifier, detail, e)
        logging.error(str(e) + "\nwill wait " + str(delay))

    delays = phlsys_tryloop.make_default_endless_retry()

    phlsys_tryloop.try_loop_delay(
        f, delays, onException=on_tryloop_exception)


class RefreshCachesOperation(object):

    def __init__(self, conduits, url_watcher, reporter):
        super(RefreshCachesOperation, self).__init__()
        self._conduits = conduits
        self._url_watcher = url_watcher
        self._reporter = reporter

    def do(self):
        self._reporter.start_cache_refresh()

        with self._reporter.tag_timer_context('refresh conduit cache'):
            for key in self._conduits:
                conduit = self._conduits[key]
                _critical_tryloop(
                    conduit.refresh_cache_on_cycle,
                    "conduit-refresh",
                    conduit.describe(),
                    self._reporter)

        with self._reporter.tag_timer_context('refresh git watcher'):
            _critical_tryloop(
                self._url_watcher.refresh, 'git-snoop', '', self._reporter)

        self._reporter.finish_cache_refresh()
        return True


class ResetFileException(Exception):

{'fraction_non_alphanumeric': 0.14664088931851135, 'fraction_numerical': 0.00444659255679072, 'mean_word_length': 2.2875754687003496, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1528915', 'n_tokens_mistral': 2924, 'n_tokens_neox': 2803, 'n_words': 613}