<commit_before>"""Tests for metadata """
from ckanext.geocat.utils import csw_mapping
from nose.tools import *  # noqa
import os
from datetime import datetime
import time
import unittest
from pprint import pprint

__location__ = os.path.realpath(
    os.path.join(
        os.getcwd(),
        os.path.dirname(__file__)
    )
)


class TestGeocatDcatDatasetMetadata(unittest.TestCase):
    def setUp(self):
        self.csw_map = csw_mapping.GeoMetadataMapping(
            organization_slug="swisstopo",
            geocat_perma_link="https://perma-link/",
            geocat_perma_label="some label",
            legal_basis_url="",
            default_rights="",
            valid_identifiers=['8454f7d9-e3f2-4cc7-be6d-a82196660ccd@swisstopo'],
        )
        self.geocat_identifier = '93814e81-2466-4690-b54d-c1d958f1c3b8'

    def _load_xml(self, filename):
        path = os.path.join(__location__, 'fixtures', filename)
        with open(path) as xml:
            entry = xml.read()
        return entry

    def _is_multi_lang(self, value):
        for lang in ['de', 'fr', 'it', 'en']:
            self.assertIn(lang, value)

    def test_fields(self):
        xml = self._load_xml('complete.xml')
        dataset = self.csw_map.get_metadata(xml, self.geocat_identifier)

        fields = [
            'identifier',
            'title',
            'description',
            'issued',
            'modified',
            'publishers',
            'contact_points',
            'groups',
            'language',
            'relations',
            'temporals',
            'keywords',
            'url',
            'spatial',
            'coverage',
            'accrual_periodicity',
            'see_alsos',
            'owner_org',
            'resources'
        ]

        for field in fields:
            self.assertIn(field, dataset)

        # make sure only the defined fields are on the dataset
        self.assertEquals(sorted(fields), sorted(dataset.keys()))

        for key, value in dataset.iteritems():
            pprint(value)
            self.assertIn(key, fields)

        # check multilang fields
        self._is_multi_lang(dataset.get('title'))
        self._is_multi_lang(dataset.get('description'))

    def test_fields_values(self):
        xml = self._load_xml('complete.xml')
        dataset = self.csw_map.get_metadata(xml, self.geocat_identifier)

        # identifier
        self.assertEquals('93814e81-2466-4690-b54d-c1d958f1c3b8@swisstopo', dataset.get('identifier'))

        # title
        self.assertEquals(u'L\xe4rmbelastung durch Eisenbahnverkehr Nacht', dataset['title']['de'])
        self.assertEquals('Exposition au bruit du trafic ferroviaire, nuit', dataset['title']['fr'])
        self.assertEquals('Esposizione al rumore del traffico ferroviario, notte', dataset['title']['it'])
        self.assertEquals('Nighttime railway noise exposure', dataset['title']['en'])

        # description
        self.assertIn(u'Die Karte zeigt, welcher L\xe4rmbelastung', dataset['description']['de'])
        self.assertIn('', dataset['description']['fr'])
        self.assertIn('', dataset['description']['it'])
        self.assertIn('', dataset['description']['en'])

        # dates
        date_string = '2011-12-31' # revision date from XML
        d = datetime.strptime(date_string, '%Y-%m-%d')
        self.assertEquals(int(time.mktime(d.timetuple())), dataset['issued'])
        self.assertEquals(int(time.mktime(d.timetuple())), dataset['modified'])

        # publishers
        self.assertTrue(hasattr(dataset['publishers'], '__iter__'))
        self.assertEquals(1, len(dataset['publishers']))
        for publisher in dataset['publishers']:
            self.assertEquals(u'Bundesamt f\xfcr Umwelt', publisher['label'])

        # contact points
        self.assertTrue(hasattr(dataset['contact_points'], '__iter__'))
        self.assertEquals(1, len(dataset['contact_points']))
        for contact_point in dataset['contact_points']:
            self.assertEquals('<EMAIL>', contact_point['name'])
            self.assertEquals('<EMAIL>', contact_point['email'])

        # groups
        groups = ['territory', 'geography']
        for group in dataset.get('groups'):
            self.assertIn(group['name'], groups)

        # language
        self.assertEquals(set(['de', 'fr', 'it', 'en']), set(dataset.get('language')))

        # relations
        self.assertTrue(hasattr(dataset['relations'], '__iter__'))
        self.assertEquals(2, len(dataset['relations']))
        for relation in dataset['relations']:
            self.assertIsNotNone(relation['label'])
            self.assertIsNotNone(relation['url'])

        # resources<commit_msg>adapt test of metadata mapping<commit_after>"""Tests for metadata """
from ckanext.geocat.utils import csw_mapping
from nose.tools import *  # noqa
import os
import json
from datetime import datetime
import time
import unittest
from pprint import pprint

__location__ = os.path.realpath(
    os.path.join(
        os.getcwd(),
        os.path.dirname(__file__)
    )
)


class TestGeocatDcatDatasetMetadata(unittest.TestCase):
    def setUp(self):
        self.csw_map = csw_mapping.GeoMetadataMapping(
            organization_slug="swisstopo",
            geocat_perma_link="https://perma-link/",
            geocat_perma_label="some label",
            legal_basis_url="",
            default_rights="",
            valid_identifiers=['8454f7d9-e3f2-4cc7-be6d-a82196660ccd@swisstopo'],
        )
        self.geocat_identifier = '93814e81-2466-4690-b54d-c1d958f1c3b8'

    def _load_xml(self, filename):
        path = os.path.join(__location__, 'fixtures', filename)
        with open(path) as xml:
            entry = xml.read()
        return entry

    def _is_multi_lang(self, value):
        for lang in ['de', 'fr', 'it', 'en']:
            self.assertIn(lang, value)

    def test_fields(self):
        xml = self._load_xml('complete.xml')
        dataset = self.csw_map.get_metadata(xml, self.geocat_identifier)

        fields = [
            'identifier',
            'title',
            'description',
            'issued',
            'modified',
            'publisher',
            'contact_points',
            'groups',
            'language',
            'relations',
            'temporals',
            'keywords',
            'url',
            'spatial',
            'coverage',
            'accrual_periodicity',
            'see_alsos',
            'owner_org',
            'resources'
        ]

        for field in fields:
            self.assertIn(field, dataset)

        # make sure only the defined fields are on the dataset
        self.assertEquals(sorted(fields), sorted(dataset.keys()))

        for key, value in dataset.iteritems():
            pprint(value)
            self.assertIn(key, fields)

        # check multilang fields
        self._is_multi_lang(dataset.get('title'))
        self._is_multi_lang(dataset.get('description'))

    def test_fields_values(self):
        xml = self._load_xml('complete.xml')
        dataset = self.csw_map.get_metadata(xml, self.geocat_identifier)

        # identifier
        self.assertEquals('93814e81-2466-4690-b54d-c1d958f1c3b8@swisstopo', dataset.get('identifier'))

        # title
        self.assertEquals(u'L\xe4rmbelastung durch Eisenbahnverkehr Nacht', dataset['title']['de'])
        self.assertEquals('Exposition au bruit du trafic ferroviaire, nuit', dataset['title']['fr'])
        self.assertEquals('Esposizione al rumore del traffico ferroviario, notte', dataset['title']['it'])
        self.assertEquals('Nighttime railway noise exposure', dataset['title']['en'])

        # description
        self.assertIn(u'Die Karte zeigt, welcher L\xe4rmbelastung', dataset['description']['de'])
        self.assertIn('', dataset['description']['fr'])
        self.assertIn('', dataset['description']['it'])
        self.assertIn('', dataset['description']['en'])

        # dates
        date_string = '2011-12-31' # revision date from XML
        d = datetime.strptime(date_string, '%Y-%m-%d')
        self.assertEquals(int(time.mktime(d.timetuple())), dataset['issued'])
        self.assertEquals(int(time.mktime(d.timetuple())), dataset['modified'])

        # publisher
        publisher = json.loads(dataset['publisher'])
        self.assertTrue(isinstance(publisher, dict))
        self.assertEquals(u'Bundesamt f\xfcr Umwelt', publisher['name'])
        self.assertEquals(u'https://opendata.swiss/organization/swisstopo', publisher['url'])

        # contact points
        self.assertTrue(hasattr(dataset['contact_points'], '__iter__'))
        self.assertEquals(1, len(dataset['contact_points']))
        for contact_point in dataset['contact_points']:
            self.assertEquals('<EMAIL>', contact_point['name'])
            self.assertEquals('<EMAIL>', contact_point['email'])

        # groups
        groups = ['territory', 'geography']
        for group in dataset.get('groups'):
            self.assertIn(group['name'], groups)

        # language
        self.assertEquals(set(['de', 'fr', 'it', 'en']), set(dataset.get('language')))

        # relations
        self.assertTrue(hasattr(dataset['relations'], '__iter__'))
        self.assertEquals(2, len(dataset['relations']))
        for relation in dataset['relations']:
            self.assertIsNotNone(relation['label'])
            self.assertIsNotNone(relation['url'])

        # resources
{'fraction_non_alphanumeric': 0.13507234132432147, 'fraction_numerical': 0.01679163586439962, 'mean_word_length': 2.595292331055429, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5098485', 'n_tokens_mistral': 2861, 'n_tokens_neox': 2751, 'n_words': 501}