<commit_before># Copyright (C) 2018  Nihilate
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

import pytest

from roboragi.Kitsu import get_title_by_language_codes, ENGLISH_LANGUAGE_CODES, ROMAJI_LANGUAGE_CODES, JAPANESE_LANGUAGE_CODES


@pytest.mark.parametrize('language_codes,titles,expected', [
    (ENGLISH_LANGUAGE_CODES, {'en': 'English'}, 'English'),
    (ENGLISH_LANGUAGE_CODES, {'en_us': 'American English'}, 'American English'),
    (ROMAJI_LANGUAGE_CODES, {'en_jp': 'Romaji'}, 'Romaji'),
    (JAPANESE_LANGUAGE_CODES, {'ja_jp': 'Japanese'}, 'Japanese'),
    (ENGLISH_LANGUAGE_CODES, {'en': 'English', 'en_us': 'American English'}, 'English'),
    (ENGLISH_LANGUAGE_CODES, {'en': 'English', 'en_us': 'American English', 'en_jp': 'Romaji', 'ja_jp': 'Japanese'}, 'English'),
    (ROMAJI_LANGUAGE_CODES, {'en': 'English', 'en_us': 'American English', 'en_jp': 'Romaji', 'ja_jp': 'Japanese'}, 'Romaji'),
    (JAPANESE_LANGUAGE_CODES, {'en': 'English', 'en_us': 'American English', 'en_jp': 'Romaji', 'ja_jp': 'Japanese'}, 'Japanese'),
    (ENGLISH_LANGUAGE_CODES, {}, None),
    (ROMAJI_LANGUAGE_CODES, {}, None),
    (JAPANESE_LANGUAGE_CODES, {}, None),
])
def test_title_extraction(language_codes, titles, expected):
    actual = get_title_by_language_codes(titles, language_codes)
    assert actual == expected

<commit_msg>Make Kitsu title extraction tests more explicit<commit_after># Copyright (C) 2018  Nihilate
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

import pytest

from roboragi.Kitsu import (
    get_title_by_language_codes,
    ENGLISH_LANGUAGE_CODES,
    ROMAJI_LANGUAGE_CODES,
    JAPANESE_LANGUAGE_CODES
)

BRITISH_ENGLISH_TITLES = dict(en='British English Title')
AMERICAN_ENGLISH_TITLES = dict(en_us='American English Title')
ALL_ENGLISH_TITLES = dict(**BRITISH_ENGLISH_TITLES, **AMERICAN_ENGLISH_TITLES)

ROMAJI_TITLES = dict(en_jp='Rōmaji Title')
JAPANESE_TITLES = dict(ja_jp='日本のタイトル')

ALL_LANGUAGE_TITLES = dict(
    **ALL_ENGLISH_TITLES,
    **ROMAJI_TITLES,
    **JAPANESE_TITLES,
)


@pytest.mark.parametrize('language_codes,titles,expected', [
    (ENGLISH_LANGUAGE_CODES, BRITISH_ENGLISH_TITLES, BRITISH_ENGLISH_TITLES['en']),  # noqa: E501
    (ENGLISH_LANGUAGE_CODES, AMERICAN_ENGLISH_TITLES, AMERICAN_ENGLISH_TITLES['en_us']),  # noqa: E501
    (ROMAJI_LANGUAGE_CODES, ROMAJI_TITLES, ROMAJI_TITLES['en_jp']),
    (JAPANESE_LANGUAGE_CODES, JAPANESE_TITLES, JAPANESE_TITLES['ja_jp']),
])
def test_title_extraction_single_titles(language_codes, titles, expected):
    actual = get_title_by_language_codes(titles, language_codes)
    assert actual == expected


@pytest.mark.parametrize('language_codes,titles,expected', [
    (ENGLISH_LANGUAGE_CODES, ALL_LANGUAGE_TITLES, ALL_LANGUAGE_TITLES['en']),
    (ROMAJI_LANGUAGE_CODES, ALL_LANGUAGE_TITLES, ALL_LANGUAGE_TITLES['en_jp']),
    (JAPANESE_LANGUAGE_CODES, ALL_LANGUAGE_TITLES, ALL_LANGUAGE_TITLES['ja_jp']),  # noqa: E501
])
def test_title_extraction_multiple_titles(language_codes, titles, expected):
    actual = get_title_by_language_codes(titles, language_codes)
    assert actual == expected


@pytest.mark.parametrize('language_codes', [
    ENGLISH_LANGUAGE_CODES,
    ROMAJI_LANGUAGE_CODES,
    JAPANESE_LANGUAGE_CODES,
])
def test_title_extraction_no_titles(language_codes):
    assert get_title_by_language_codes({}, language_codes) is None


def test_title_extraction_prefers_british_over_american_english():
    expected = ALL_ENGLISH_TITLES['en']
    actual = get_title_by_language_codes(
        titles=ALL_ENGLISH_TITLES,
        language_codes=ENGLISH_LANGUAGE_CODES
    )
    assert actual == expected

{'fraction_non_alphanumeric': 0.10451800957734748, 'fraction_numerical': 0.003955860920258172, 'mean_word_length': 5.942196531791907, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '876789', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1728, 'n_words': 399}