<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

import html2text

from udata.models import License, Resource
from . import BaseBackend, register
from ..exceptions import HarvestException, HarvestSkipException


log = logging.getLogger(__name__)


@register
class OdsHarvester(BaseBackend):
    name = 'ods'
    display_name = 'OpenDataSoft'
    verify_ssl = False

    LICENSES = {
        "Open Database License (ODbL)": "odc-odbl",
        "Licence Ouverte (Etalab)": "fr-lo",
        "Licence ouverte / Open Licence": "fr-lo",
        "CC BY-SA": "cc-by-sa",
        "Public Domain": "other-pd"
    }

    FORMATS = {
        'csv': ('CSV', 'csv', 'text/csv'),
        'geojson': ('GeoJSON', 'json', 'application/vnd.geo+json'),
        'json': ('JSON', 'json', 'application/json'),
        'shp': ('Shapefile', 'shp', None),
    }

    @property
    def api_url(self):
        return "{0}/api/datasets/1.0/search/".format(self.source.url)

    def _get_download_url(self, dataset_id, format):
        return ("%s/explore/dataset/%s/download"
                "?format=%s&timezone=Europe/Berlin"
                "&use_labels_for_header=true") % (self.source.url,
                                                  dataset_id,
                                                  format)

    def _get_explore_url(self, dataset_id):
        return "%s/explore/dataset/%s/" % (self.source.url, dataset_id)

    def _get_export_url(self, dataset_id):
        return "%s/explore/dataset/%s/?tab=export" % (self.source.url,
                                                      dataset_id)

    def initialize(self):
        count = 0
        nhits = None

        def should_fetch():
            if nhits is None:
                return True
            max_value = min(nhits, self.max_items) if self.max_items else nhits
            return count < max_value

        while should_fetch():
            response = self.get(self.api_url,
                                params={"start": count, "rows": 50})
            response.raise_for_status()
            data = response.json()
            nhits = data["nhits"]
            for dataset in data["datasets"]:
                count += 1
                self.add_item(dataset["datasetid"], dataset=dataset)

    def process(self, item):
        ods_dataset = item.kwargs["dataset"]
        dataset_id = ods_dataset["datasetid"]
        ods_metadata = ods_dataset["metas"]

        if not ods_dataset.get('has_records'):
            msg = 'Dataset {datasetid} has no record'.format(**ods_dataset)
            raise HarvestSkipException(msg)

        dataset = self.get_dataset(item.remote_id)

        dataset.title = ods_metadata['title']
        dataset.frequency = "unknown"
        description = ods_metadata.get("description", '').strip()
        description = html2text.html2text(description.strip('\n').strip(),
                                          bodywidth=0)
        dataset.description = description
        dataset.private = False

        tags = set()
        if "keyword" in ods_metadata:
            if isinstance(ods_metadata['keyword'], list):<commit_msg>Ensure no tailing spaces or new lines on description<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

import html2text

from udata.models import License, Resource
from . import BaseBackend, register
from ..exceptions import HarvestException, HarvestSkipException


log = logging.getLogger(__name__)


@register
class OdsHarvester(BaseBackend):
    name = 'ods'
    display_name = 'OpenDataSoft'
    verify_ssl = False

    LICENSES = {
        "Open Database License (ODbL)": "odc-odbl",
        "Licence Ouverte (Etalab)": "fr-lo",
        "Licence ouverte / Open Licence": "fr-lo",
        "CC BY-SA": "cc-by-sa",
        "Public Domain": "other-pd"
    }

    FORMATS = {
        'csv': ('CSV', 'csv', 'text/csv'),
        'geojson': ('GeoJSON', 'json', 'application/vnd.geo+json'),
        'json': ('JSON', 'json', 'application/json'),
        'shp': ('Shapefile', 'shp', None),
    }

    @property
    def api_url(self):
        return "{0}/api/datasets/1.0/search/".format(self.source.url)

    def _get_download_url(self, dataset_id, format):
        return ("%s/explore/dataset/%s/download"
                "?format=%s&timezone=Europe/Berlin"
                "&use_labels_for_header=true") % (self.source.url,
                                                  dataset_id,
                                                  format)

    def _get_explore_url(self, dataset_id):
        return "%s/explore/dataset/%s/" % (self.source.url, dataset_id)

    def _get_export_url(self, dataset_id):
        return "%s/explore/dataset/%s/?tab=export" % (self.source.url,
                                                      dataset_id)

    def initialize(self):
        count = 0
        nhits = None

        def should_fetch():
            if nhits is None:
                return True
            max_value = min(nhits, self.max_items) if self.max_items else nhits
            return count < max_value

        while should_fetch():
            response = self.get(self.api_url,
                                params={"start": count, "rows": 50})
            response.raise_for_status()
            data = response.json()
            nhits = data["nhits"]
            for dataset in data["datasets"]:
                count += 1
                self.add_item(dataset["datasetid"], dataset=dataset)

    def process(self, item):
        ods_dataset = item.kwargs["dataset"]
        dataset_id = ods_dataset["datasetid"]
        ods_metadata = ods_dataset["metas"]

        if not ods_dataset.get('has_records'):
            msg = 'Dataset {datasetid} has no record'.format(**ods_dataset)
            raise HarvestSkipException(msg)

        dataset = self.get_dataset(item.remote_id)

        dataset.title = ods_metadata['title']
        dataset.frequency = "unknown"
        description = ods_metadata.get("description", '').strip()
        description = html2text.html2text(description.strip('\n').strip(),
                                          bodywidth=0)
        dataset.description = description.strip().strip('\n').strip()
        dataset.private = False

        tags = set()
        if "keyword" in ods_metadata:
            if isinstance(ods_metadata['keyword'], list):
{'fraction_non_alphanumeric': 0.12439443663072355, 'fraction_numerical': 0.003750586029067042, 'mean_word_length': 2.003284842796809, 'pattern_counts': {'":': 14, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4797926', 'n_tokens_mistral': 1886, 'n_tokens_neox': 1799, 'n_words': 395}