<commit_before>import logging
import urlparse

import requests

from harvester.ext.crawler.base import CrawlerPluginBase

logger = logging.getLogger(__name__)

# http://www.comune.trento.it/api/opendata/v1/content/class/open_data/offset/10/limit/5


class ComunWebCrawler(CrawlerPluginBase):
    """
    Crawler for "ComunWeb"-powered websites.
    """

    options = []

    def fetch_data(self, storage):
        logger.info("Fetching data from comunweb")

        for i, dataset in enumerate(self._scan_datasets()):
            logger.info(
                u'Storing dataset {seq}: id={id!r} "{title}"'
                .format(
                    seq=i+1,
                    id=dataset['objectId'],
                    title=dataset['objectName']))

            metadata_url = dataset['link']
            metadata = requests.get(metadata_url).json()
            dataset['full_metadata'] = metadata

            # Store it
            storage.documents['dataset'][dataset['objectId']] = dataset

    def _scan_datasets(self):
        offset, limit = 0, 50
        start_url = urlparse.urljoin(
            self.url, '/api/opendata/v1/content/class/open_data')

        while True:
            page_url = '{0}/offset/{1}/limit/{2}'.format(
                start_url, offset, limit)
            response = requests.get(page_url)<commit_msg>ComunWeb crawler now downloading data for **all** types (not only open_data)<commit_after>import logging
import urlparse

import requests

from harvester.ext.crawler.base import CrawlerPluginBase

logger = logging.getLogger(__name__)


class ComunWebCrawler(CrawlerPluginBase):
    """
    Crawler for "ComunWeb"-powered websites.
    """

    options = []

    def fetch_data(self, storage):
        logger.info("Fetching data from comunweb")

        classes = self._list_object_classes()
        for clsinfo in classes:
            logger.info(u"Found class {0} ({1}): {2}"
                        .format(clsinfo['identifier'],
                                clsinfo['name'],
                                clsinfo['link']))

            obj_type = clsinfo['identifier']
            for i, obj in enumerate(self._scan_pages(clsinfo['link'])):
                obj_id = obj['objectId']
                logger.debug(
                    u'Storing "{type}" object #{seq} (id={id!r}): "{title}"'
                    .format(
                        seq=i+1,
                        type=obj_type,
                        id=obj_id,
                        title=obj['objectName']))

                metadata_url = obj['link']
                metadata = requests.get(metadata_url).json()
                obj['full_metadata'] = metadata

                # Store it
                storage.documents[obj_type][obj_id] = obj

    def _list_object_classes(self):
        response = requests.get(urlparse.urljoin(
            self.url, '/api/opendata/v1/content/classList'))
        assert response.ok
        return response.json()['classes']

    def _scan_datasets(self):
        start_url = urlparse.urljoin(
            self.url, '/api/opendata/v1/content/class/open_data')
        return self._scan_pages(start_url)

    def _scan_pages(self, start_url):
        offset, limit = 0, 50

        while True:
            page_url = '{0}/offset/{1}/limit/{2}'.format(
                start_url, offset, limit)
            response = requests.get(page_url)
{'fraction_non_alphanumeric': 0.11844431349440189, 'fraction_numerical': 0.007071302298173247, 'mean_word_length': 1.934312878133103, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6519175', 'n_tokens_mistral': 997, 'n_tokens_neox': 944, 'n_words': 185}