<commit_before># Copyright 2010 <NAME> <<EMAIL>>
# See LICENSE for licensing information.

# Active test scheduler and circuit manager for TorBEL.

# We come from the __future__.
from __future__ import with_statement

import time
import resource
import threading
from collections import deque
from torbel import config
from torbel.logger import *
log = get_logger("torbel")

class TestScheduler:
    """ Abstract base class for all test schedulers. """
    controller = None
    name = "Abstract"
    def __init__(self, controller, max_pending_circuits = 10):
        self.controller = controller
        self.terminated = False
        ## Circuit dictionaries.
        # Established circuits under test.
        self.circuits = {}
        # Circuits in the process of being built.
        self.pending_circuits = {}
        self.pending_circuit_lock = threading.RLock()
        self.pending_circuit_cond = threading.Condition(self.pending_circuit_lock)
        # Circuit failure metrics and bookkeeping.
        self.circuit_failures = deque()
        self.circuit_fail_count = 0
        self.circuit_retry_success_count = 0
        
        self.max_pending_circuits = max_pending_circuits
        # Base max running circuits on the total number of file descriptors
        # we can have open (hard limit returned by getrlimit) and the maximum
        # number of file descriptors per circuit, adjusting for possible pending
        # circuits, TorCtl connection, stdin/out, and other files.
        max_files = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
            
        circuit_limit = max_files / len(controller.test_ports) - max_pending_circuits
        self.max_running_circuits = min(config.max_built_circuits, circuit_limit)
        
    def next(self):
        """ Return a set of routers to be tested. May block until enough routers
        are deemed ready by the scheduler. """
        raise ValueError("You must implement next()!")

    def stop(self):
        """ Stop the scheduler. """
        with self.pending_circuit_cond:
            self.pending_circuit_cond.notify()
            self.terminated = True

    def circ_pending(self, circ_id, router):
        with self.pending_circuit_lock:
            self.pending_circuits[circ_id] = router

    def circ_built(self, event):
        circ_id = event.circ_id
        with self.pending_circuit_cond:
            if circ_id in self.pending_circuits:
                router = self.pending_circuits[circ_id]
                del self.pending_circuits[circ_id]
                # Notify scheduler thread that we have
                # completed building a circuit and we could
                # need to pre-build more.
                self.pending_circuit_cond.notify()
            else:
                return

            # If we succeeded in building this router on retry,
            # reset its failure count to give it a clean slate.
            if router.retry:
                self.circuit_retry_success_count += 1
                router.retry = False
                router.circuit_failures = 0
                log.debug("Retry for %s successful (%d/%d succesful, %.2f%%)!",
                          router.nickname, self.circuit_retry_success_count,
                          self.circuit_fail_count + self.circuit_retry_success_count,
                          100 * float(self.circuit_retry_success_count) / \
                              (self.circuit_fail_count + self.circuit_retry_success_count))
                
            log.log(VERBOSE1, "Successfully built circuit %d for %s.",
                    circ_id, router.nickname)
            self.circuits[circ_id] = router
            self.controller.connect_test(router)

    def circ_closed(self, event):
        circ_id = event.circ_id
        with self.pending_circuit_cond:
            if circ_id in self.circuits:
                log.log(VERBOSE1, "Closed circuit %d (%s).", circ_id,
                        self.circuits[circ_id].nickname)
                del self.circuits[circ_id]
            elif circ_id in self.pending_circuits:
                # Pending circuit closed before being built (can this happen?)
                log.debug("Pending circuit closed (%d)?", circ_id)<commit_msg>Log number of failures before retry success.<commit_after># Copyright 2010 <NAME> <hbock<EMAIL>>
# See LICENSE for licensing information.

# Active test scheduler and circuit manager for TorBEL.

# We come from the __future__.
from __future__ import with_statement

import time
import resource
import threading
from collections import deque
from torbel import config
from torbel.logger import *
log = get_logger("torbel")

class TestScheduler:
    """ Abstract base class for all test schedulers. """
    controller = None
    name = "Abstract"
    def __init__(self, controller, max_pending_circuits = 10):
        self.controller = controller
        self.terminated = False
        ## Circuit dictionaries.
        # Established circuits under test.
        self.circuits = {}
        # Circuits in the process of being built.
        self.pending_circuits = {}
        self.pending_circuit_lock = threading.RLock()
        self.pending_circuit_cond = threading.Condition(self.pending_circuit_lock)
        # Circuit failure metrics and bookkeeping.
        self.circuit_failures = deque()
        self.circuit_fail_count = 0
        self.circuit_retry_success_count = 0
        
        self.max_pending_circuits = max_pending_circuits
        # Base max running circuits on the total number of file descriptors
        # we can have open (hard limit returned by getrlimit) and the maximum
        # number of file descriptors per circuit, adjusting for possible pending
        # circuits, TorCtl connection, stdin/out, and other files.
        max_files = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
            
        circuit_limit = max_files / len(controller.test_ports) - max_pending_circuits
        self.max_running_circuits = min(config.max_built_circuits, circuit_limit)
        
    def next(self):
        """ Return a set of routers to be tested. May block until enough routers
        are deemed ready by the scheduler. """
        raise ValueError("You must implement next()!")

    def stop(self):
        """ Stop the scheduler. """
        with self.pending_circuit_cond:
            self.pending_circuit_cond.notify()
            self.terminated = True

    def circ_pending(self, circ_id, router):
        with self.pending_circuit_lock:
            self.pending_circuits[circ_id] = router

    def circ_built(self, event):
        circ_id = event.circ_id
        with self.pending_circuit_cond:
            if circ_id in self.pending_circuits:
                router = self.pending_circuits[circ_id]
                del self.pending_circuits[circ_id]
                # Notify scheduler thread that we have
                # completed building a circuit and we could
                # need to pre-build more.
                self.pending_circuit_cond.notify()
            else:
                return

            # If we succeeded in building this router on retry,
            # reset its failure count to give it a clean slate.
            if router.retry:
                self.circuit_retry_success_count += 1
                router.retry = False
                log.debug("Retry for %s successful after %d failures (%d/%d %.2f%%)!",
                          router.nickname, router.circuit_failures,
                          self.circuit_retry_success_count,
                          self.circuit_fail_count + self.circuit_retry_success_count,
                          100 * float(self.circuit_retry_success_count) / \
                              (self.circuit_fail_count + self.circuit_retry_success_count))
            router.circuit_failures = 0

                
            log.log(VERBOSE1, "Successfully built circuit %d for %s.",
                    circ_id, router.nickname)
            self.circuits[circ_id] = router
            self.controller.connect_test(router)

    def circ_closed(self, event):
        circ_id = event.circ_id
        with self.pending_circuit_cond:
            if circ_id in self.circuits:
                log.log(VERBOSE1, "Closed circuit %d (%s).", circ_id,
                        self.circuits[circ_id].nickname)
                del self.circuits[circ_id]
            elif circ_id in self.pending_circuits:
                # Pending circuit closed before being built (can this happen?)
                log.debug("Pending circuit closed (%d)?", circ_id)
{'fraction_non_alphanumeric': 0.07132018209408195, 'fraction_numerical': 0.0039687171705381115, 'mean_word_length': 2.1350164654226127, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2520457', 'n_tokens_mistral': 2380, 'n_tokens_neox': 2232, 'n_words': 648}