<commit_before>"""Harvest process for the FossJobs Source."""

import logging
from collections import defaultdict

from remotes.models import periodically, Source
from .fossjobs import FossJobs

logger = logging.getLogger(__name__)


class Harvester(object):
    """Simple Harvester to gather FossJobs posts."""

    source = Source.objects.get(code='fossjobs').first()

    def __init__(self):
        """Init the fossjobs harvester."""
        self.status_info = defaultdict(int)

    @periodically(source, period='hourly')
    def harvest(self):
        """Gather some Posts from hackernews."""
        fossjobs = FossJobs(self.source)
        for post in fossjobs.jobs():
            if post.exists():
                logger.debug('Alread processed this item %s, skipping the rest.', post)
                break
            self.status_info['count-rss'] += 1
            self.status_info['total'] += 1
            yield post
        logger.info("FossJobs harvester status: %s", dict(self.status_info))

    def status(self):
        """Return the current status of this harvester."""
        return self.status_info
<commit_msg>don't use first on a get<commit_after>"""Harvest process for the FossJobs Source."""

import logging
from collections import defaultdict

from remotes.models import periodically, Source
from .fossjobs import FossJobs

logger = logging.getLogger(__name__)


class Harvester(object):
    """Simple Harvester to gather FossJobs posts."""

    source = Source.objects.get(code='fossjobs')

    def __init__(self):
        """Init the fossjobs harvester."""
        self.status_info = defaultdict(int)

    @periodically(source, period='hourly')
    def harvest(self):
        """Gather some Posts from hackernews."""
        fossjobs = FossJobs(self.source)
        for post in fossjobs.jobs():
            if post.exists():
                logger.debug('Alread processed this item %s, skipping the rest.', post)
                break
            self.status_info['count-rss'] += 1
            self.status_info['total'] += 1
            yield post
        logger.info("FossJobs harvester status: %s", dict(self.status_info))

    def status(self):
        """Return the current status of this harvester."""
        return self.status_info

{'fraction_non_alphanumeric': 0.11116012351124835, 'fraction_numerical': 0.00176444640494045, 'mean_word_length': 2.798994974874372, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1210997', 'n_tokens_mistral': 658, 'n_tokens_neox': 610, 'n_words': 185}