<commit_before># -*- coding: utf-8 -*-
import sys
import mock
from ordereddict import OrderedDict
from collections import defaultdict

import lxml.etree

import zope.component

import zeit.cms.content.sources
import zeit.cms.content.add
from zeit.cms.repository.folder import Folder
import zeit.cms.testing
import zeit.connector.mock
from zeit.content.article.testing import create_article
import zeit.content.volume.interfaces
from zeit.content.volume.browser.toc import Toc, Excluder
from zeit.content.volume.volume import Volume
import zeit.content.volume.testing


class TocFunctionalTest(zeit.content.volume.testing.FunctionalTestCase):

    def setUp(self):
        super(TocFunctionalTest, self).setUp()
        self.toc_data = OrderedDict()
        self.toc_data['Die Zeit'] = OrderedDict(
            {'Politik': [{'page': '1',
                          'title': 'title',
                          'teaser': 'tease',
                          'supertitle': 'Super'
                        }]
             }
        )
        self.toc_data['Anderer'] = OrderedDict(
            {'Dossier': [
                {'page': '1',
                 'title': 'title',
                 'teaser': 'tease',
                 'supertitle': 'Super'
                 },
                {'page': '3',
                 'title': 'title2',
                 'teaser': 'tease',
                 'supertitle': 'Super'}
            ]}
        )
        self.article_xml_template = u"""
            <article>
                <head>
                    <attribute ns="http://namespaces.zeit.de/CMS/document"
                    name="page">{page}</attribute>
                </head>
                <body>
                     <title>Titel</title>
                     <subtitle>Das soll der Teaser
                     sein</subtitle>
                </body>
            </article>
        """

    def test_list_relevant_ressort_folders_excludes_leserbriefe_and_images(
            self):
        toc = Toc(None, None)
        toc_connector = zope.component.getUtility(
            zeit.content.volume.interfaces.ITocConnector)
        self.zca.patch_utility(toc_connector,
                               zeit.connector.interfaces.IConnector)
        folders = ['images', 'leserbriefe', 'politik']
        with zeit.cms.testing.site(self.getRootFolder()):
            self.repository['ZEI'] = Folder()
            self.repository['ZEI']['2015'] = Folder()
            self.repository['ZEI']['2015']['01'] = Folder()
            for foldername in folders:
                self.repository['ZEI']['2015']['01'][foldername] = Folder()
            relevant_ressorts = toc.list_relevant_ressort_folders(
                'http://xml.zeit.de'
                '/ZEI/2015/01')
        foldernames = [folder.__name__ for folder in relevant_ressorts]
        self.assertIn('politik', foldernames)
        self.zca.reset()

    def test_create_toc_element_should_flatten_linebreaks(self):
        article_xml = self.article_xml_template.format(page='20-20')
        expected = {'page': 20,
                    'title': 'Titel',
                    'teaser': 'Das soll der Teaser sein',
                    'supertitle': ''
                    }
        article_element = lxml.etree.fromstring(article_xml)
        toc = Toc(None, None)
        result = toc._create_toc_element(article_element)
        self.assertEqual(expected, result)

    def test_csv_is_created_from_toc_data(self):
        expected = """Die Zeit\r
Politik\r
1\tSuper title tease\r
Anderer\r
Dossier\r
1\tSuper title tease\r
3\tSuper title2 tease\r
"""
        toc = Toc(None, None)
        res = toc._create_csv(self.toc_data)
        self.assertEqual(expected, res)

    def test_empty_page_node_in_xml_results_in_max_int_page_in_toc_entry(self):
        article_xml = self.article_xml_template.format(page='')
        article_element = lxml.etree.fromstring(article_xml)
        t = Toc(None, None)
        entry = t._create_toc_element(article_element)
        assert sys.maxint == entry.get('page')

    def test_product_id_mapping_has_full_name_for_zei_product_id(self):
        mapping = zeit.content.volume.interfaces.PRODUCT_MAPPING
        self.assertEqual('Die Zeit'.lower(), mapping.get('ZEI', '').lower())

    def test_sorts_entries_with_max_int_page_as_last_toc_element(self):
        toc_data = {
            'Die Zeit': {
                'Politik':
                    [
                        {'page': sys.maxint, 'title': 'title2'},<commit_msg>Change test to look for the ressort in the second cell<commit_after># -*- coding: utf-8 -*-
import sys
import mock
from ordereddict import OrderedDict
from collections import defaultdict

import lxml.etree

import zope.component

import zeit.cms.content.sources
import zeit.cms.content.add
from zeit.cms.repository.folder import Folder
import zeit.cms.testing
import zeit.connector.mock
from zeit.content.article.testing import create_article
import zeit.content.volume.interfaces
from zeit.content.volume.browser.toc import Toc, Excluder
from zeit.content.volume.volume import Volume
import zeit.content.volume.testing


class TocFunctionalTest(zeit.content.volume.testing.FunctionalTestCase):

    def setUp(self):
        super(TocFunctionalTest, self).setUp()
        self.toc_data = OrderedDict()
        self.toc_data['Die Zeit'] = OrderedDict(
            {'Politik': [{'page': '1',
                          'title': 'title',
                          'teaser': 'tease',
                          'supertitle': 'Super'
                        }]
             }
        )
        self.toc_data['Anderer'] = OrderedDict(
            {'Dossier': [
                {'page': '1',
                 'title': 'title',
                 'teaser': 'tease',
                 'supertitle': 'Super'
                 },
                {'page': '3',
                 'title': 'title2',
                 'teaser': 'tease',
                 'supertitle': 'Super'}
            ]}
        )
        self.article_xml_template = u"""
            <article>
                <head>
                    <attribute ns="http://namespaces.zeit.de/CMS/document"
                    name="page">{page}</attribute>
                </head>
                <body>
                     <title>Titel</title>
                     <subtitle>Das soll der Teaser
                     sein</subtitle>
                </body>
            </article>
        """

    def test_list_relevant_ressort_folders_excludes_leserbriefe_and_images(
            self):
        toc = Toc(None, None)
        toc_connector = zope.component.getUtility(
            zeit.content.volume.interfaces.ITocConnector)
        self.zca.patch_utility(toc_connector,
                               zeit.connector.interfaces.IConnector)
        folders = ['images', 'leserbriefe', 'politik']
        with zeit.cms.testing.site(self.getRootFolder()):
            self.repository['ZEI'] = Folder()
            self.repository['ZEI']['2015'] = Folder()
            self.repository['ZEI']['2015']['01'] = Folder()
            for foldername in folders:
                self.repository['ZEI']['2015']['01'][foldername] = Folder()
            relevant_ressorts = toc.list_relevant_ressort_folders(
                'http://xml.zeit.de'
                '/ZEI/2015/01')
        foldernames = [folder.__name__ for folder in relevant_ressorts]
        self.assertIn('politik', foldernames)
        self.zca.reset()

    def test_create_toc_element_should_flatten_linebreaks(self):
        article_xml = self.article_xml_template.format(page='20-20')
        expected = {'page': 20,
                    'title': 'Titel',
                    'teaser': 'Das soll der Teaser sein',
                    'supertitle': ''
                    }
        article_element = lxml.etree.fromstring(article_xml)
        toc = Toc(None, None)
        result = toc._create_toc_element(article_element)
        self.assertEqual(expected, result)

    def test_csv_is_created_from_toc_data(self):
        expected = """Die Zeit\r
\tPolitik\r
1\tSuper title tease\r
Anderer\r
\tDossier\r
1\tSuper title tease\r
3\tSuper title2 tease\r
"""
        toc = Toc(None, None)
        res = toc._create_csv(self.toc_data)
        self.assertEqual(expected, res)

    def test_empty_page_node_in_xml_results_in_max_int_page_in_toc_entry(self):
        article_xml = self.article_xml_template.format(page='')
        article_element = lxml.etree.fromstring(article_xml)
        t = Toc(None, None)
        entry = t._create_toc_element(article_element)
        assert sys.maxint == entry.get('page')

    def test_product_id_mapping_has_full_name_for_zei_product_id(self):
        mapping = zeit.content.volume.interfaces.PRODUCT_MAPPING
        self.assertEqual('Die Zeit'.lower(), mapping.get('ZEI', '').lower())

    def test_sorts_entries_with_max_int_page_as_last_toc_element(self):
        toc_data = {
            'Die Zeit': {
                'Politik':
                    [
                        {'page': sys.maxint, 'title': 'title2'},
{'fraction_non_alphanumeric': 0.11598365182812327, 'fraction_numerical': 0.008395007179940352, 'mean_word_length': 1.9773100953633673, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6366', 'n_tokens_mistral': 2798, 'n_tokens_neox': 2615, 'n_words': 463}