<commit_before>#coding: utf-8

import xlrd
import os
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from uuid import NAMESPACE_OID, uuid4, uuid5
import tempfile

from ckan.lib.base import c
from ckan import model
from ckan.model import Session, Package
from ckan.logic import ValidationError, NotFound, get_action, action
from ckan.lib.helpers import json
from ckanext.harvest.harvesters.base import munge_tag
from ckan.lib.munge import munge_title_to_name

from ckanext.harvest.model import HarvestJob, HarvestObject, HarvestGatherError, \
                                    HarvestObjectError
from ckanext.harvest.harvesters import HarvesterBase

from pylons import config

import logging
log = logging.getLogger(__name__)

class SFAHarvester(HarvesterBase):
    '''
    The harvester for the SFA
    '''

    BUCKET_NAME = config.get('ckanext.sfa.s3_bucket')
    METADATA_FILE_NAME = u'OGD@Bund Metadaten BAR.xlsx'
    DEPARTMENT_BASE = u'ch.bar.'
    FILES_BASE_URL = 'http://' + BUCKET_NAME + '.s3.amazonaws.com'

    # Define the keys in the CKAN .ini file
    AWS_ACCESS_KEY = config.get('ckanext.sfa.s3_key')
    AWS_SECRET_KEY = config.get('ckanext.sfa.s3_token')

    ORGANIZATION = {
        'de': {
            'name': u'Schweizerisches Bundesarchiv',
            'description': u'Das Dienstleistungs- und Kompetenzzentrum des Bundes für nachhaltiges Informationsmanagement.',
            'website': u'http://www.bar.admin.ch/'
        },
        'fr': {
            'name': u'Archives fédérales suisses',
            'description': u"Le centre de prestations et de compétences de la Confédération pour une gestion durable de l'information."
        },
        'it': {
            'name': u'Archivio federale svizzero',
            'description': u'Il centro di servizio e competenza della Confederazione per la gestione a lungo termine delle informazioni.'
        },
        'en': {
            'name': u'Swiss Federal Archives',
            'description': u"The Confederation's service and competence centre for lasting information management."
        }
    }
    LANG_CODES = ['de', 'fr', 'it', 'en']

    config = {
        'user': u'harvest'
    }

    def _get_s3_bucket(self):
        '''
        Create an S3 connection to the department bucket
        '''
        conn = S3Connection(self.AWS_ACCESS_KEY, self.AWS_SECRET_KEY)
        bucket = conn.get_bucket(self.BUCKET_NAME)
        return bucket


    def _fetch_metadata_file(self):
        '''
        Fetching the Excel metadata file for the SFA from the S3 Bucket and save on disk
        '''
        temp_dir = tempfile.mkdtemp()
        try:
            metadata_file = Key(self._get_s3_bucket())
            metadata_file.key = self.METADATA_FILE_NAME
            metadata_file_path = os.path.join(temp_dir, self.METADATA_FILE_NAME)
            log.debug('Saving metadata file to %s' % metadata_file_path)
            metadata_file.get_contents_to_filename(metadata_file_path)
            return metadata_file_path
        except Exception, e:
            log.exception(e)
            raise


    def _guess_format(self, file_name):
        '''
        Return the format for a given full filename
        '''
        _, file_extension = os.path.splitext(file_name.lower())
        return file_extension[1:]

    def _generate_resources_dict_array(self, dataset_id):
        '''

        '''
        try:
            resources = []
            prefix = self.DEPARTMENT_BASE + dataset_id + u'/'
            bucket_list = self._get_s3_bucket().list(prefix=prefix)
            for file in bucket_list:
                if file.key != prefix:
                    resources.append({
                        'url': self.FILES_BASE_URL + '/' + file.key,
                        'name': file.key.replace(prefix, u''),
                        'format': self._guess_format(file.key),
                        'size': self._get_s3_bucket().lookup(file.key).size
                    })
            return resources
        except Exception, e:
            log.exception(e)
            raise


    def _get_row_dict_array(self, lang_index, file_path):
        '''<commit_msg>Add the last-modified date of the metadata file to a resource's metadata.<commit_after>#coding: utf-8

import xlrd
import os
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from uuid import NAMESPACE_OID, uuid4, uuid5
import tempfile

from ckan.lib.base import c
from ckan import model
from ckan.model import Session, Package
from ckan.logic import ValidationError, NotFound, get_action, action
from ckan.lib.helpers import json
from ckanext.harvest.harvesters.base import munge_tag
from ckan.lib.munge import munge_title_to_name

from ckanext.harvest.model import HarvestJob, HarvestObject, HarvestGatherError, \
                                    HarvestObjectError
from ckanext.harvest.harvesters import HarvesterBase

from pylons import config

import logging
log = logging.getLogger(__name__)

class SFAHarvester(HarvesterBase):
    '''
    The harvester for the SFA
    '''

    BUCKET_NAME = config.get('ckanext.sfa.s3_bucket')
    METADATA_FILE_NAME = u'OGD@Bund Metadaten BAR.xlsx'
    DEPARTMENT_BASE = u'ch.bar.'
    FILES_BASE_URL = 'http://' + BUCKET_NAME + '.s3.amazonaws.com'

    # Define the keys in the CKAN .ini file
    AWS_ACCESS_KEY = config.get('ckanext.sfa.s3_key')
    AWS_SECRET_KEY = config.get('ckanext.sfa.s3_token')

    ORGANIZATION = {
        'de': {
            'name': u'Schweizerisches Bundesarchiv',
            'description': u'Das Dienstleistungs- und Kompetenzzentrum des Bundes für nachhaltiges Informationsmanagement.',
            'website': u'http://www.bar.admin.ch/'
        },
        'fr': {
            'name': u'Archives fédérales suisses',
            'description': u"Le centre de prestations et de compétences de la Confédération pour une gestion durable de l'information."
        },
        'it': {
            'name': u'Archivio federale svizzero',
            'description': u'Il centro di servizio e competenza della Confederazione per la gestione a lungo termine delle informazioni.'
        },
        'en': {
            'name': u'Swiss Federal Archives',
            'description': u"The Confederation's service and competence centre for lasting information management."
        }
    }
    LANG_CODES = ['de', 'fr', 'it', 'en']

    config = {
        'user': u'harvest'
    }

    def _get_s3_bucket(self):
        '''
        Create an S3 connection to the department bucket
        '''
        conn = S3Connection(self.AWS_ACCESS_KEY, self.AWS_SECRET_KEY)
        bucket = conn.get_bucket(self.BUCKET_NAME)
        return bucket


    def _fetch_metadata_file(self):
        '''
        Fetching the Excel metadata file for the SFA from the S3 Bucket and save on disk
        '''
        temp_dir = tempfile.mkdtemp()
        try:
            metadata_file = Key(self._get_s3_bucket())
            metadata_file.key = self.METADATA_FILE_NAME
            metadata_file_path = os.path.join(temp_dir, self.METADATA_FILE_NAME)
            log.debug('Saving metadata file to %s' % metadata_file_path)
            metadata_file.get_contents_to_filename(metadata_file_path)
            return metadata_file_path
        except Exception, e:
            log.exception(e)
            raise


    def _guess_format(self, file_name):
        '''
        Return the format for a given full filename
        '''
        _, file_extension = os.path.splitext(file_name.lower())
        return file_extension[1:]

    def _generate_resources_dict_array(self, dataset_id):
        '''

        '''
        try:
            resources = []
            prefix = self.DEPARTMENT_BASE + dataset_id + u'/'
            bucket_list = self._get_s3_bucket().list(prefix=prefix)
            for file in bucket_list:
                if file.key != prefix:
                    resources.append({
                        'url': self.FILES_BASE_URL + '/' + file.key,
                        'name': file.key.replace(prefix, u''),
                        'format': self._guess_format(file.key),
                        'size': self._get_s3_bucket().lookup(file.key).size,
                        'version': self._get_s3_bucket().lookup(file.key).last_modified
                    })
            return resources
        except Exception, e:
            log.exception(e)
            raise


    def _get_row_dict_array(self, lang_index, file_path):
        '''
{'fraction_non_alphanumeric': 0.09345904537418975, 'fraction_numerical': 0.004360636417206835, 'mean_word_length': 2.2891472868217053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '234843', 'n_tokens_mistral': 2593, 'n_tokens_neox': 2553, 'n_words': 619}