<commit_before>import json
import urllib
import urllib2
import httplib
import datetime
import socket
import datetime

from ckanext.harvest.harvesters import CKANHarvester
from ckan import model
import ckan.lib.munge as munge

log = __import__('logging').getLogger(__name__)

MIMETYPE_FORMATS = {
    'text/html': 'HTML',
    'text/csv': 'CSV',
    'text/xml': 'XML',
    'application/pdf': 'PDF',
    'application/zip': 'ZIP',
    'application/rdf+xml': 'RDF',
    'application/json': 'JSON',
    'application/vnd.ms-excel': 'XLS',
    'application/vnd.google-earth.kml+xml': 'KML',
    'application/msword': 'DOC',
}


class DKANHarvester(CKANHarvester):

    ckan_revision_api_works = False

    def info(self):
        return {
            'name': 'dkan',
            'title': 'DKAN HARVESTER',
            'description': 'Harvests remote DKAN instances',
            'form_config_interface': 'Text'
        }

    def _get_search_api_offset(self):
        return '%s/current_package_list_with_resources' % self._get_action_api_offset()

    def _get_all_packages(self, base_url, harvest_job):
        # Request all remote packages
        url = base_url + '/api/3/action/package_list'
        log.debug('Getting all DKAN packages: %s', url)
        try:
            content = self._get_content(url)
        except Exception, e:
            self._save_gather_error('Unable to get content for URL: %s - %s'
                                    % (url, e), harvest_job)
            return None

        packages = json.loads(content)['result']

        return packages

    def _get_package(self, base_url, harvest_object):
        url = base_url + '/api/3/action/package_show/' + harvest_object.guid
        log.debug('Getting DKAN package: %s', url)

        # Get contents
        try:
            content = self._get_content(url)
        except Exception, e:
            self._save_object_error(
                'Unable to get content for package: %s - %r' % (url, e),
                harvest_object)
            return None, None

        package = json.loads(content)['result'][0]
        return url, json.dumps(package)

    def _search_for_datasets(self, remote_ckan_base_url, fq_terms=None):
        '''Does a dataset search on a remote CKAN and returns the results.

        Deals with paging to return all the results, not just the first page.
        '''
        base_search_url = remote_ckan_base_url + self._get_search_api_offset()
        params = {'limit': '100', 'offset': '0'}
        # There is the worry that datasets will be changed whilst we are paging
        # through them.
        # * In SOLR 4.7 there is a cursor, but not using that yet
        #   because few CKANs are running that version yet.
        # * However we sort, then new names added or removed before the current
        #   page would cause existing names on the next page to be missed or
        #   double counted.
        # * Another approach might be to sort by metadata_modified and always
        #   ask for changes since (and including) the date of the last item of
        #   the day before. However if the entire page is of the exact same
        #   time, then you end up in an infinite loop asking for the same page.
        # * We choose a balanced approach of sorting by ID, which means
        #   datasets are only missed if some are removed, which is far less
        #   likely than any being added. If some are missed then it is assumed
        #   they will harvested the next time anyway. When datasets are added,
        #   we are at risk of seeing datasets twice in the paging, so we detect
        #   and remove any duplicates.

        pkg_dicts = []
        pkg_ids = set()
        previous_content = None
        while True:
            url = base_search_url + '?' + urllib.urlencode(params)
            log.debug('Searching for CKAN datasets: %s', url)
            try:
                content = self._get_content(url)
            except ContentFetchError, e:
                raise SearchError(
                    'Error sending request to search remote '
                    'CKAN instance %s using URL %r. Error: %s' %
                    (remote_ckan_base_url, url, e))

            if previous_content and content == previous_content:
                raise SearchError('The paging doesn\'t seem to work. URL: %s' %
                                  url)
            try:
                response_dict = json.loads(content)
            except ValueError:
                raise SearchError('Response from remote CKAN was not JSON: %r'
                                  % content)
            try:
                pkg_dicts_page = response_dict.get('result', [])
            except ValueError:
                raise SearchError('Response JSON did not contain '
                                  'result/results: %r' % response_dict)

            if len(pkg_dicts_page) == 0:
                break
            # Weed out any datasets found on previous pages (should datasets be
            # changing while we page)

            if type(pkg_dicts_page[0]) == list:
                pkg_dicts_page = pkg_dicts_page[0]

            ids_in_page = set(p['id'] for p in pkg_dicts_page)
            duplicate_ids = ids_in_page & pkg_ids
            if duplicate_ids:
                pkg_dicts_page = [convert_dkan_package_to_ckan(p) for p in pkg_dicts_page if p['id'] not in duplicate_ids]
            pkg_ids |= ids_in_page

            pkg_dicts.extend(pkg_dicts_page)

            params['offset'] = str(int(params['offset']) + int(params['limit']))

        return pkg_dicts

    def convert_dkan_package_to_ckan(self, package):<commit_msg>Ajuste de fechas y campos a formatos de CKAN<commit_after>import json
import urllib
import urllib2
import httplib
import datetime
import socket
import datetime

from ckanext.harvest.harvesters import CKANHarvester
from ckan import model
import ckan.lib.munge as munge

log = __import__('logging').getLogger(__name__)

MIMETYPE_FORMATS = {
    'text/html': 'HTML',
    'text/csv': 'CSV',
    'text/xml': 'XML',
    'application/pdf': 'PDF',
    'application/zip': 'ZIP',
    'application/rdf+xml': 'RDF',
    'application/json': 'JSON',
    'application/vnd.ms-excel': 'XLS',
    'application/vnd.google-earth.kml+xml': 'KML',
    'application/msword': 'DOC',
}


class DKANHarvester(CKANHarvester):

    ckan_revision_api_works = False

    def info(self):
        return {
            'name': 'dkan',
            'title': 'DKAN HARVESTER',
            'description': 'Harvests remote DKAN instances',
            'form_config_interface': 'Text'
        }

    def _get_search_api_offset(self):
        return '%s/current_package_list_with_resources' % self._get_action_api_offset()

    def _get_all_packages(self, base_url, harvest_job):
        # Request all remote packages
        url = base_url + '/api/3/action/package_list'
        log.debug('Getting all DKAN packages: %s', url)
        try:
            content = self._get_content(url)
        except Exception, e:
            self._save_gather_error('Unable to get content for URL: %s - %s'
                                    % (url, e), harvest_job)
            return None

        packages = json.loads(content)['result']

        return packages

    def _get_package(self, base_url, harvest_object):
        url = base_url + '/api/3/action/package_show/' + harvest_object.guid
        log.debug('Getting DKAN package: %s', url)

        # Get contents
        try:
            content = self._get_content(url)
        except Exception, e:
            self._save_object_error(
                'Unable to get content for package: %s - %r' % (url, e),
                harvest_object)
            return None, None

        package = json.loads(content)['result'][0]
        return url, json.dumps(package)

    def _search_for_datasets(self, remote_ckan_base_url, fq_terms=None):
        '''Does a dataset search on a remote CKAN and returns the results.

        Deals with paging to return all the results, not just the first page.
        '''
        base_search_url = remote_ckan_base_url + self._get_search_api_offset()
        params = {'limit': '100', 'offset': '0'}
        # There is the worry that datasets will be changed whilst we are paging
        # through them.
        # * In SOLR 4.7 there is a cursor, but not using that yet
        #   because few CKANs are running that version yet.
        # * However we sort, then new names added or removed before the current
        #   page would cause existing names on the next page to be missed or
        #   double counted.
        # * Another approach might be to sort by metadata_modified and always
        #   ask for changes since (and including) the date of the last item of
        #   the day before. However if the entire page is of the exact same
        #   time, then you end up in an infinite loop asking for the same page.
        # * We choose a balanced approach of sorting by ID, which means
        #   datasets are only missed if some are removed, which is far less
        #   likely than any being added. If some are missed then it is assumed
        #   they will harvested the next time anyway. When datasets are added,
        #   we are at risk of seeing datasets twice in the paging, so we detect
        #   and remove any duplicates.

        pkg_dicts = []
        pkg_ids = set()
        previous_content = None
        while True:
            url = base_search_url + '?' + urllib.urlencode(params)
            log.debug('Searching for CKAN datasets: %s', url)
            try:
                content = self._get_content(url)
            except ContentFetchError, e:
                raise SearchError(
                    'Error sending request to search remote '
                    'CKAN instance %s using URL %r. Error: %s' %
                    (remote_ckan_base_url, url, e))

            if previous_content and content == previous_content:
                raise SearchError('The paging doesn\'t seem to work. URL: %s' %
                                  url)
            try:
                response_dict = json.loads(content)
            except ValueError:
                raise SearchError('Response from remote CKAN was not JSON: %r'
                                  % content)
            try:
                pkg_dicts_page = response_dict.get('result', [])
            except ValueError:
                raise SearchError('Response JSON did not contain '
                                  'result/results: %r' % response_dict)

            if len(pkg_dicts_page) == 0:
                break
            # Weed out any datasets found on previous pages (should datasets be
            # changing while we page)

            if type(pkg_dicts_page[0]) == list:
                pkg_dicts_page = pkg_dicts_page[0]

            ids_in_page = set(p['id'] for p in pkg_dicts_page)
            duplicate_ids = ids_in_page & pkg_ids
            if duplicate_ids:
                pkg_dicts_page = [self.convert_dkan_package_to_ckan(p) for p in pkg_dicts_page if p['id'] not in duplicate_ids]
            pkg_ids |= ids_in_page

            pkg_dicts.extend(pkg_dicts_page)

            params['offset'] = str(int(params['offset']) + int(params['limit']))

        return pkg_dicts

    def convert_dkan_package_to_ckan(self, package):
{'fraction_non_alphanumeric': 0.08547535211267605, 'fraction_numerical': 0.002288732394366197, 'mean_word_length': 2.0491143317230276, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '398680', 'n_tokens_mistral': 3297, 'n_tokens_neox': 3167, 'n_words': 1027}