<commit_before>class FrontierTester(object):

    def __init__(self, frontier, graph_manager, max_next_requests=0):
        self.frontier = frontier
        self.graph_manager = graph_manager
        self.max_next_requests = max_next_requests
        self.sequence = []

    def run(self, add_all_pages=False):
        if not self.frontier.auto_start:
            self.frontier.start()
        if not add_all_pages:
            self._add_seeds()
        else:
            self._add_all()
        while True:
            requests = self._run_iteration()
            self.sequence += requests
            if not requests:
                break
        self.frontier.stop()

    def _add_seeds(self):
        self.frontier.add_seeds([self._make_request(seed.url) for seed in self.graph_manager.seeds])

    def _add_all(self):
        for page in self.graph_manager.pages:
            if page.is_seed:
                self.frontier.add_seeds([self._make_request(page.url)])
            if not page.has_errors:
                for link in page.links:
                    self.frontier.add_seeds([self._make_request(link.url)])
    def _make_request(self, url):
        return self.frontier.request_model(url=url)

    def _make_response(self, url, status_code, request):
        return self.frontier.response_model(url=url, status_code=status_code, request=request)

    def _run_iteration(self):<commit_msg>PEP-8 fixes<commit_after>class FrontierTester(object):

    def __init__(self, frontier, graph_manager, max_next_requests=0):
        self.frontier = frontier
        self.graph_manager = graph_manager
        self.max_next_requests = max_next_requests
        self.sequence = []

    def run(self, add_all_pages=False):
        if not self.frontier.auto_start:
            self.frontier.start()
        if not add_all_pages:
            self._add_seeds()
        else:
            self._add_all()
        while True:
            requests = self._run_iteration()
            self.sequence += requests
            if not requests:
                break
        self.frontier.stop()

    def _add_seeds(self):
        self.frontier.add_seeds([self._make_request(seed.url) for seed in self.graph_manager.seeds])

    def _add_all(self):
        for page in self.graph_manager.pages:
            if page.is_seed:
                self.frontier.add_seeds([self._make_request(page.url)])
            if not page.has_errors:
                for link in page.links:
                    self.frontier.add_seeds([self._make_request(link.url)])

    def _make_request(self, url):
        return self.frontier.request_model(url=url)

    def _make_response(self, url, status_code, request):
        return self.frontier.response_model(url=url, status_code=status_code, request=request)

    def _run_iteration(self):
{'fraction_non_alphanumeric': 0.0929818311364446, 'fraction_numerical': 0.0010687566797292483, 'mean_word_length': 2.303529411764706, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6933380', 'n_tokens_mistral': 861, 'n_tokens_neox': 832, 'n_words': 156}