<commit_before>import pendulum
from furl import furl

import logging
from share import Harvester

logger = logging.getLogger(__name__)


class PeerJHarvester(Harvester):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.base_url = 'https://peerj.com/articles/index.json'
        self.base_preprint_url = 'https://peerj.com/preprints/index.json'

    def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum):

        return self.fetch_records(self.base_url, start_date, end_date)

    def fetch_records(self, url, start_date, end_date):

        logger.info('Making request to {}'.format(url))
        for url in self.build_url():
            if self.base_preprint_url in url:
                yield from self.fetch_page(url, start_date, end_date, 'preprint-')
            yield from self.fetch_page(url, start_date, end_date)

        logger.info("PeerJ has been harvested")

    def fetch_page(self, url, start_date, end_date, preprint=''):
        while url:
            records = self.requests.get(url).json()

            for record in records['_items']:
                if pendulum.parse(record['date']) < start_date:
                    return

                if pendulum.parse(record['date']) > end_date:
                    continue

                yield (preprint + record['identifiers']['peerj'], record)

            if 'next' in record['_links']:
                url = record['_links']['next']['href']
            else:
                url = None

    def build_url(self):
        return [furl(self.base_url).set(query_params={
            'journal': 'peerj',
        }).url,
            furl(self.base_url).set(query_params={
                'journal': 'cs',
            }).url,
            furl(self.base_preprint_url).set(query_params={
                'journal': 'peerj',
            }).url
        ]<commit_msg>[SHARE-494][Fix] Correct PeerJ harvester<commit_after>import pendulum
from furl import furl

import logging
from share import Harvester

logger = logging.getLogger(__name__)


class PeerJHarvester(Harvester):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.base_url = 'https://peerj.com/articles/index.json'
        self.base_preprint_url = 'https://peerj.com/preprints/index.json'

    def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum):
        return self.fetch_records(self.base_url, start_date, end_date)

    def fetch_records(self, url, start_date, end_date):
        for url in self.build_url():
            if self.base_preprint_url in url:
                yield from self.fetch_page(url, start_date, end_date, 'preprint-')
            else:
                yield from self.fetch_page(url, start_date, end_date)

        logger.info("PeerJ has been harvested")

    def fetch_page(self, url, start_date, end_date, preprint=''):
        while url:
            logger.info('Making request to %s', url)
            records = self.requests.get(url).json()

            for record in records['_items']:
                if pendulum.parse(record['date']) < start_date:
                    logger.debug('%s is before %s, returning', record['date'], start_date)
                    return

                if pendulum.parse(record['date']) > end_date:
                    logger.debug('%s is after %s, skipping', record['date'], end_date)
                    continue

                yield (preprint + record['identifiers']['peerj'], record)

            if 'next' in record['_links']:
                url = record['_links']['next']['href']
            else:
                url = None

    def build_url(self):
        return [
            furl(self.base_url).set(query_params={'journal': 'peerj'}).url,
            furl(self.base_url).set(query_params={'journal': 'cs'}).url,
            furl(self.base_preprint_url).set(query_params={'journal': 'peerj'}).url,
        ]
{'fraction_non_alphanumeric': 0.12680334092634776, 'fraction_numerical': 0.0007593014426727411, 'mean_word_length': 2.307112970711297, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4310870', 'n_tokens_mistral': 1221, 'n_tokens_neox': 1148, 'n_words': 249}