<commit_before>from sphinx.errors import SphinxError
from sphinx.util import logging

import lib.yaml_writer as yaml_writer


logger = logging.getLogger(__name__)


# Following keys may be given a default in base (first) language
# and other language versions are allowed to omit them.

ACCEPTED_INDEX_DEFAULT_KEYS = [
    'description', 'contact', 'contact_phone', 'assistants',
    'start', 'end', 'static_dir', 'numerate_ignoring_modules',
    'module_types', 'exercise_types',
]
ACCEPTED_CATEGORY_DEFAULT_KEYS = [
    'status', 'points_to_pass',
]
ACCEPTED_MODULE_DEFAULT_KEYS = [
    'status', 'type', 'points_to_pass',
    'open', 'close', 'duration', 'late_close', 'late_duration', 'late_penalty',
]
ACCEPTED_CHILDREN_DEFAULT_KEYS = [
    'status', 'type', 'category', 'config', 'static_content',
    'max_submissions', 'max_points', 'points_to_pass',
    'min_group_size', 'max_group_size',
    'allow_assistant_viewing', 'allow_assistant_grading',
    'use_wide_column', 'generate_table_of_contents',
]
IDENTICAL_EXERCISE_KEYS = [
    'category', 'max_points', 'difficulty', 'max_submissions',
    'min_group_size', 'max_group_size', 'points_to_pass', 'feedback',
]
# Internal keys that may be joined in the |i18n format. Internal keys are only
# used inside the a-plus-rst-tools to pass information forward.
INTERNAL_KEYS_TO_JOIN = ['_rst_srcpath']


def join(app, indexes):
    base_lang,base = indexes[0]
    if len(indexes) < 2:
        return base
    joiner = IndexJoiner(app, base_lang, base)
    for lang,index in indexes[1:]:
        joiner.join(lang, index)
    return joiner.get_joined()


class IndexJoiner:

    def __init__(self, app, base_lang, base):
        self.app = app
        self.base_lang = base_lang
        self.joined = base
        self.joined['lang'] = [base_lang]
        self.errors = 0
        self.skip_errors = app.config.skip_language_inconsistencies

    def get_joined(self):
        if self.errors > 0:
            msg = "Language versions have previously reported {:d} inconsistencies!".format(self.errors)
            if self.app.config.skip_language_inconsistencies:
                logger.warning(msg + (
                    " The 'skip_language_inconsistencies' option is set and"
                    " therefore incomplete version of the course is compiled."
                    " This setting should only be used in development phase!"
                ))
            else:
                raise SphinxError(msg + " Fix the problems to compile the course.")
        return self.joined

    def join(self, lang2, index2):
        lang1 = self.base_lang
        index1 = self.joined
        path = []
        index = {}
        self.require_identical_dict_keys(path, lang1, index1, lang2, index2, ACCEPTED_INDEX_DEFAULT_KEYS)
        for k,v in index1.items():
            if k == 'lang':
                index[k] = v + [lang2]
            elif k == 'name':
                index[k] = join_values(lang1, v, lang2, index2.get(k, v))
            elif k == 'categories':
                p = path + ['categories']
                index[k] = self.join_categories(p, lang1, v, lang2, index2.get(k, {}))
            elif k == 'modules':
                p = path + ['modules']
                index[k] = self.join_modules(p, lang1, v, lang2, index2.get(k, []))
            elif deep_equals(v, index2.get(k, v)):
                index[k] = v
            else:
                self.raise_unequal(path, lang2, k)
        self.joined = index

    def join_categories(self, path, lang1, c1_map, lang2, c2_map):
        c_map = {}
        self.require_identical_dict_keys(path, lang1, c1_map, lang2, c2_map)
        for n,c1 in c1_map.items():
            c2 = c2_map.get(n, {})
            c_path = path + [n]
            c = {}
            self.require_identical_dict_keys(c_path, lang1, c1, lang2, c2, ACCEPTED_CATEGORY_DEFAULT_KEYS)
            for k,v in c1.items():
                if k == 'name':
                    c[k] = join_values(lang1, v, lang2, c2.get(k, v))
                elif deep_equals(v, c2.get(k, v)):
                    c[k] = v
                else:
                    self.raise_unequal(c_path, lang2, k)
            c_map[n] = c
        return c_map

    def join_modules(self, path, lang1, m1_list, lang2, m2_list):
        m_list = []
        for i,m1 in enumerate(
            self.require_identical_list_len(path, lang1, m1_list, lang2, m2_list)
        ):
            m2 = m2_list[i]
            m_path = path + [str(i + 1)]
            m = {}
            self.require_identical_dict_keys(m_path, lang1, m1, lang2, m2, ACCEPTED_MODULE_DEFAULT_KEYS)
            for k,v in m1.items():
                if k == 'key':
                    m[k] = join_keys(lang1, v, lang2, m2.get(k, v))
                elif k in ('name', 'title'):
                    m[k] = join_values(lang1, v, lang2, m2.get(k, v))
                elif k == 'children':
                    m[k] = self.join_children(m_path, lang1, v, lang2, m2.get(k, []))
                elif deep_equals(v, m2.get(k, v)):
                    m[k] = v
                else:
                    self.raise_unequal(m_path, lang2, k)
            m_list.append(m)
        return m_list

    def join_children(self, path, lang1, c1_list, lang2, c2_list):
        c_list = []
        for i,c1 in enumerate(
            self.require_identical_list_len(path, lang1, c1_list, lang2, c2_list)
        ):
            c2 = c2_list[i]
            c_path = path + [str(i + 1)]
            c = {}
            self.require_identical_dict_keys(c_path, lang1, c1, lang2, c2, ACCEPTED_CHILDREN_DEFAULT_KEYS)
            key = join_keys(lang1, c1.get('key', ''), lang2, c2.get('key', ''))
            for k,v in c1.items():
                if k == 'key':
                    c[k] = key
                elif k in ('name', 'title', 'static_content'):
                    c[k] = join_values(lang1, v, lang2, c2.get(k, v))
                elif k == 'config':
                    e1 = yaml_writer.read(yaml_writer.file_path(self.app.env, v))
                    e2 = yaml_writer.read(yaml_writer.file_path(self.app.env, c2.get(k, v)))
                    yaml_writer.write(
                        yaml_writer.file_path(self.app.env, key),
                        self.join_exercises(key, lang1, e1, lang2, e2)
                    )
                    c[k] = key + '.yaml'
                elif k == 'children':
                    c[k] = self.join_children(c_path, lang1, v, lang2, c2.get(k, []))
                elif k in INTERNAL_KEYS_TO_JOIN:
                    c[k + '|i18n'] = join_values(lang1, v, lang2, c2.get(k, v))
                elif deep_equals(v, c2.get(k, v)):
                    c[k] = v
                else:
                    self.raise_unequal(c_path, lang2, k)
            c_list.append(c)
        return c_list

    def join_exercises(self, key, lang1, c1, lang2, c2):
        path = [key]
        c = {}
        for k,v in c1.items():
            if k == 'key':
                c[k] = key
            elif k == 'url':
                override = self.app.env.config.override.get(c1.get('category'), {})
                if k in override:
                    c[k] = override[k].format(key=key)<commit_msg>Fix the exercise configure field in multilingual courses<commit_after>from sphinx.errors import SphinxError
from sphinx.util import logging

import lib.yaml_writer as yaml_writer


logger = logging.getLogger(__name__)


# Following keys may be given a default in base (first) language
# and other language versions are allowed to omit them.

ACCEPTED_INDEX_DEFAULT_KEYS = [
    'description', 'contact', 'contact_phone', 'assistants',
    'start', 'end', 'static_dir', 'numerate_ignoring_modules',
    'module_types', 'exercise_types',
]
ACCEPTED_CATEGORY_DEFAULT_KEYS = [
    'status', 'points_to_pass',
]
ACCEPTED_MODULE_DEFAULT_KEYS = [
    'status', 'type', 'points_to_pass',
    'open', 'close', 'duration', 'late_close', 'late_duration', 'late_penalty',
]
ACCEPTED_CHILDREN_DEFAULT_KEYS = [
    'status', 'type', 'category', 'config', 'static_content',
    'max_submissions', 'max_points', 'points_to_pass',
    'min_group_size', 'max_group_size',
    'allow_assistant_viewing', 'allow_assistant_grading',
    'use_wide_column', 'generate_table_of_contents',
]
IDENTICAL_EXERCISE_KEYS = [
    'category', 'max_points', 'difficulty', 'max_submissions',
    'min_group_size', 'max_group_size', 'points_to_pass', 'feedback',
]
# Internal keys that may be joined in the |i18n format. Internal keys are only
# used inside the a-plus-rst-tools to pass information forward.
INTERNAL_KEYS_TO_JOIN = ['_rst_srcpath']


def join(app, indexes):
    base_lang,base = indexes[0]
    if len(indexes) < 2:
        return base
    joiner = IndexJoiner(app, base_lang, base)
    for lang,index in indexes[1:]:
        joiner.join(lang, index)
    return joiner.get_joined()


class IndexJoiner:

    def __init__(self, app, base_lang, base):
        self.app = app
        self.base_lang = base_lang
        self.joined = base
        self.joined['lang'] = [base_lang]
        self.errors = 0
        self.skip_errors = app.config.skip_language_inconsistencies

    def get_joined(self):
        if self.errors > 0:
            msg = "Language versions have previously reported {:d} inconsistencies!".format(self.errors)
            if self.app.config.skip_language_inconsistencies:
                logger.warning(msg + (
                    " The 'skip_language_inconsistencies' option is set and"
                    " therefore incomplete version of the course is compiled."
                    " This setting should only be used in development phase!"
                ))
            else:
                raise SphinxError(msg + " Fix the problems to compile the course.")
        return self.joined

    def join(self, lang2, index2):
        lang1 = self.base_lang
        index1 = self.joined
        path = []
        index = {}
        self.require_identical_dict_keys(path, lang1, index1, lang2, index2, ACCEPTED_INDEX_DEFAULT_KEYS)
        for k,v in index1.items():
            if k == 'lang':
                index[k] = v + [lang2]
            elif k == 'name':
                index[k] = join_values(lang1, v, lang2, index2.get(k, v))
            elif k == 'categories':
                p = path + ['categories']
                index[k] = self.join_categories(p, lang1, v, lang2, index2.get(k, {}))
            elif k == 'modules':
                p = path + ['modules']
                index[k] = self.join_modules(p, lang1, v, lang2, index2.get(k, []))
            elif deep_equals(v, index2.get(k, v)):
                index[k] = v
            else:
                self.raise_unequal(path, lang2, k)
        self.joined = index

    def join_categories(self, path, lang1, c1_map, lang2, c2_map):
        c_map = {}
        self.require_identical_dict_keys(path, lang1, c1_map, lang2, c2_map)
        for n,c1 in c1_map.items():
            c2 = c2_map.get(n, {})
            c_path = path + [n]
            c = {}
            self.require_identical_dict_keys(c_path, lang1, c1, lang2, c2, ACCEPTED_CATEGORY_DEFAULT_KEYS)
            for k,v in c1.items():
                if k == 'name':
                    c[k] = join_values(lang1, v, lang2, c2.get(k, v))
                elif deep_equals(v, c2.get(k, v)):
                    c[k] = v
                else:
                    self.raise_unequal(c_path, lang2, k)
            c_map[n] = c
        return c_map

    def join_modules(self, path, lang1, m1_list, lang2, m2_list):
        m_list = []
        for i,m1 in enumerate(
            self.require_identical_list_len(path, lang1, m1_list, lang2, m2_list)
        ):
            m2 = m2_list[i]
            m_path = path + [str(i + 1)]
            m = {}
            self.require_identical_dict_keys(m_path, lang1, m1, lang2, m2, ACCEPTED_MODULE_DEFAULT_KEYS)
            for k,v in m1.items():
                if k == 'key':
                    m[k] = join_keys(lang1, v, lang2, m2.get(k, v))
                elif k in ('name', 'title'):
                    m[k] = join_values(lang1, v, lang2, m2.get(k, v))
                elif k == 'children':
                    m[k] = self.join_children(m_path, lang1, v, lang2, m2.get(k, []))
                elif deep_equals(v, m2.get(k, v)):
                    m[k] = v
                else:
                    self.raise_unequal(m_path, lang2, k)
            m_list.append(m)
        return m_list

    def join_children(self, path, lang1, c1_list, lang2, c2_list):
        c_list = []
        for i,c1 in enumerate(
            self.require_identical_list_len(path, lang1, c1_list, lang2, c2_list)
        ):
            c2 = c2_list[i]
            c_path = path + [str(i + 1)]
            c = {}
            self.require_identical_dict_keys(c_path, lang1, c1, lang2, c2, ACCEPTED_CHILDREN_DEFAULT_KEYS)
            key = join_keys(lang1, c1.get('key', ''), lang2, c2.get('key', ''))
            for k,v in c1.items():
                if k == 'key':
                    c[k] = key
                elif k in ('name', 'title', 'static_content'):
                    c[k] = join_values(lang1, v, lang2, c2.get(k, v))
                elif k == 'config':
                    e1 = yaml_writer.read(yaml_writer.file_path(self.app.env, v))
                    e2 = yaml_writer.read(yaml_writer.file_path(self.app.env, c2.get(k, v)))
                    yaml_writer.write(
                        yaml_writer.file_path(self.app.env, key),
                        self.join_exercises(key, lang1, e1, lang2, e2)
                    )
                    c[k] = key + '.yaml'
                elif k == 'children':
                    c[k] = self.join_children(c_path, lang1, v, lang2, c2.get(k, []))
                elif k in INTERNAL_KEYS_TO_JOIN:
                    c[k + '|i18n'] = join_values(lang1, v, lang2, c2.get(k, v))
                elif k == 'configure':
                    # Combine the configure files from all languages to a single dictionary.
                    # Do not add any language codes or the |i18n suffix to the keys.
                    # Check that the urls are identical.
                    if v.get('url') != c2.get(k, v).get('url'):
                        self.raise_unequal(c_path, lang2, 'configure.url')
                    else:
                        configure = c2.get(k, v).copy()
                        files = configure.get('files')
                        if files:
                            files = files.copy()
                            files.update(v.get('files', {}))
                            configure['files'] = files
                        else:
                            configure['files'] = v.get('files', {})
                        c[k] = configure
                elif deep_equals(v, c2.get(k, v)):
                    c[k] = v
                else:
                    self.raise_unequal(c_path, lang2, k)
            c_list.append(c)
        return c_list

    def join_exercises(self, key, lang1, c1, lang2, c2):
        path = [key]
        c = {}
        for k,v in c1.items():
            if k == 'key':
                c[k] = key
            elif k == 'url':
                override = self.app.env.config.override.get(c1.get('category'), {})
                if k in override:
                    c[k] = override[k].format(key=key)
{'fraction_non_alphanumeric': 0.1209284181782719, 'fraction_numerical': 0.016578896040569533, 'mean_word_length': 1.8234214390602055, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1835222', 'n_tokens_mistral': 5134, 'n_tokens_neox': 4938, 'n_words': 1146}