<commit_before>import logging

from rdmo.conditions.models import Condition
from rdmo.core.imports import (fetch_parents, get_foreign_field,
                               get_m2m_instances, set_common_fields,
                               set_lang_field, validate_instance)

from .models import Option, OptionSet

logger = logging.getLogger(__name__)


def import_optionset(element, save=False):
    try:
        optionset = OptionSet.objects.get(uri=element.get('uri'))
    except OptionSet.DoesNotExist:
        optionset = OptionSet()

    set_common_fields(optionset, element)

    optionset.order = element.get('order')
    optionset.provider_key = element.get('provider_key', '')

    conditions = get_m2m_instances(optionset, element.get('conditions'), Condition)

    if save and validate_instance(optionset):
        if optionset.id:
            logger.info('OptionSet created with uri %s.', element.get('uri'))
        else:
            logger.info('OptionSet %s updated.', element.get('uri'))

        optionset.save()
        optionset.conditions.set(conditions)
        optionset.imported = True

    return optionset


def import_option(element, parent_uri=False, save=False):
    if parent_uri is False:
        parent_uri = element.get('optionset')
<commit_msg>Fix optionset import by avoiding provider_key being null<commit_after>import logging

from rdmo.conditions.models import Condition
from rdmo.core.imports import (fetch_parents, get_foreign_field,
                               get_m2m_instances, set_common_fields,
                               set_lang_field, validate_instance)

from .models import Option, OptionSet

logger = logging.getLogger(__name__)


def import_optionset(element, save=False):
    try:
        optionset = OptionSet.objects.get(uri=element.get('uri'))
    except OptionSet.DoesNotExist:
        optionset = OptionSet()

    set_common_fields(optionset, element)

    optionset.order = element.get('order')
    optionset.provider_key = element.get('provider_key', '')
    if optionset.provider_key is None:
        optionset.provider_key = ''

    conditions = get_m2m_instances(optionset, element.get('conditions'), Condition)

    if save and validate_instance(optionset):
        if optionset.id:
            logger.info('OptionSet created with uri %s.', element.get('uri'))
        else:
            logger.info('OptionSet %s updated.', element.get('uri'))

        optionset.save()
        optionset.conditions.set(conditions)
        optionset.imported = True

    return optionset


def import_option(element, parent_uri=False, save=False):
    if parent_uri is False:
        parent_uri = element.get('optionset')

{'fraction_non_alphanumeric': 0.09715994020926756, 'fraction_numerical': 0.0014947683109118087, 'mean_word_length': 3.1248073959938365, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2324078', 'n_tokens_mistral': 765, 'n_tokens_neox': 742, 'n_words': 162}