<commit_before>import json

from haystack import indexes
from opaque_keys.edx.keys import CourseKey

from course_discovery.apps.course_metadata.models import Course, CourseRun, Program


class OrganizationsMixin:
    def format_organization(self, organization):
        return '{key}: {name}'.format(key=organization.key, name=organization.name)

    def format_organization_body(self, organization):
        # Deferred to prevent a circular import:
        # course_discovery.apps.api.serializers -> course_discovery.apps.course_metadata.search_indexes
        from course_discovery.apps.api.serializers import OrganizationSerializer

        return json.dumps(OrganizationSerializer(organization).data)

    def _prepare_organizations(self, organizations):
        return [self.format_organization(organization) for organization in organizations]

    def prepare_authoring_organizations(self, obj):
        return self._prepare_organizations(obj.authoring_organizations.all())


class BaseIndex(indexes.SearchIndex):
    model = None

    text = indexes.CharField(document=True, use_template=True)
    content_type = indexes.CharField(faceted=True)

    def prepare_content_type(self, obj):  # pylint: disable=unused-argument
        return self.model.__name__.lower()

    def get_model(self):
        return self.model

    def get_updated_field(self):  # pragma: no cover
        return 'modified'

    def index_queryset(self, using=None):
        return self.model.objects.all()


class BaseCourseIndex(OrganizationsMixin, BaseIndex):
    key = indexes.CharField(model_attr='key', stored=True)
    title = indexes.CharField(model_attr='title')
    short_description = indexes.CharField(model_attr='short_description', null=True)
    full_description = indexes.CharField(model_attr='full_description', null=True)
    subjects = indexes.MultiValueField(faceted=True)
    organizations = indexes.MultiValueField(faceted=True)
    authoring_organizations = indexes.MultiValueField(faceted=True)
    sponsoring_organizations = indexes.MultiValueField(faceted=True)
    level_type = indexes.CharField(null=True, faceted=True)
    partner = indexes.CharField(model_attr='partner__short_code', null=True, faceted=True)

    def prepare_subjects(self, obj):
        return [subject.name for subject in obj.subjects.all()]

    def prepare_organizations(self, obj):
        return self.prepare_authoring_organizations(obj) + self.prepare_sponsoring_organizations(obj)

    def prepare_authoring_organizations(self, obj):
        return self._prepare_organizations(obj.authoring_organizations.all())

    def prepare_sponsoring_organizations(self, obj):
        return self._prepare_organizations(obj.sponsoring_organizations.all())

    def prepare_level_type(self, obj):
        return obj.level_type.name if obj.level_type else None


class CourseIndex(BaseCourseIndex, indexes.Indexable):
    model = Course

    course_runs = indexes.MultiValueField()
    expected_learning_items = indexes.MultiValueField()

    prerequisites = indexes.MultiValueField(faceted=True)

    def prepare_course_runs(self, obj):
        return [course_run.key for course_run in obj.course_runs.all()]

    def prepare_expected_learning_items(self, obj):
        return [item.value for item in obj.expected_learning_items.all()]

    def prepare_prerequisites(self, obj):
        return [prerequisite.name for prerequisite in obj.prerequisites.all()]


class CourseRunIndex(BaseCourseIndex, indexes.Indexable):
    model = CourseRun

    course_key = indexes.CharField(model_attr='course__key', stored=True)
    org = indexes.CharField()
    number = indexes.CharField()
    status = indexes.CharField(model_attr='status', faceted=True)
    start = indexes.DateTimeField(model_attr='start', null=True, faceted=True)
    end = indexes.DateTimeField(model_attr='end', null=True)
    enrollment_start = indexes.DateTimeField(model_attr='enrollment_start', null=True)
    enrollment_end = indexes.DateTimeField(model_attr='enrollment_end', null=True)
    announcement = indexes.DateTimeField(model_attr='announcement', null=True)
    min_effort = indexes.IntegerField(model_attr='min_effort', null=True)
    max_effort = indexes.IntegerField(model_attr='max_effort', null=True)
    language = indexes.CharField(null=True, faceted=True)
    transcript_languages = indexes.MultiValueField(faceted=True)
    pacing_type = indexes.CharField(model_attr='pacing_type', null=True, faceted=True)
    marketing_url = indexes.CharField(null=True)
    slug = indexes.CharField(model_attr='slug', null=True)
    seat_types = indexes.MultiValueField(model_attr='seat_types', null=True, faceted=True)
    type = indexes.CharField(model_attr='type', null=True, faceted=True)
    image_url = indexes.CharField(model_attr='card_image_url', null=True)
    partner = indexes.CharField(null=True, faceted=True)
    program_types = indexes.MultiValueField()
    published = indexes.BooleanField(null=False, faceted=True)

    def prepare_partner(self, obj):
        return obj.course.partner.short_code

    def prepare_published(self, obj):
        return obj.status == CourseRun.Status.Published

    def _prepare_language(self, language):
        if language:
            # ECOM-5466: Render the macro language for all languages except Chinese
            if language.code.startswith('zh'):
                return language.name
            else:
                return language.macrolanguage

        return None

    def prepare_language(self, obj):
        return self._prepare_language(obj.language)

    def prepare_number(self, obj):
        course_run_key = CourseKey.from_string(obj.key)
        return course_run_key.course

    def prepare_org(self, obj):
        course_run_key = CourseKey.from_string(obj.key)
        return course_run_key.org

    def prepare_transcript_languages(self, obj):
        return [self._prepare_language(language) for language in obj.transcript_languages.all()]

    def prepare_marketing_url(self, obj):
        return obj.marketing_url

    def prepare_program_types(self, obj):
        return obj.program_types


class ProgramIndex(BaseIndex, indexes.Indexable, OrganizationsMixin):
    model = Program

    uuid = indexes.CharField(model_attr='uuid')
    title = indexes.CharField(model_attr='title')
    subtitle = indexes.CharField(model_attr='subtitle')
    type = indexes.CharField(model_attr='type__name', faceted=True)
    marketing_url = indexes.CharField(null=True)
    organizations = indexes.MultiValueField(faceted=True)
    authoring_organizations = indexes.MultiValueField(faceted=True)
    authoring_organization_bodies = indexes.MultiValueField()
    credit_backing_organizations = indexes.MultiValueField(faceted=True)
    card_image_url = indexes.CharField(model_attr='card_image_url', null=True)
    status = indexes.CharField(model_attr='status', faceted=True)
    partner = indexes.CharField(model_attr='partner__short_code', null=True, faceted=True)
    start = indexes.DateTimeField(model_attr='start', null=True, faceted=True)
    seat_types = indexes.MultiValueField(model_attr='seat_types', null=True, faceted=True)
    published = indexes.BooleanField(null=False, faceted=True)

    def prepare_published(self, obj):
        return obj.status == Program.Status.Active

    def prepare_organizations(self, obj):
        return self.prepare_authoring_organizations(obj) + self.prepare_credit_backing_organizations(obj)

    def prepare_authoring_organization_bodies(self, obj):
        return [self.format_organization_body(organization) for organization in obj.authoring_organizations.all()]

    def prepare_credit_backing_organizations(self, obj):
        return self._prepare_organizations(obj.credit_backing_organizations.all())

    def prepare_marketing_url(self, obj):
        return obj.marketing_url<commit_msg>expose uuids in catalog<commit_after>import json

from haystack import indexes
from opaque_keys.edx.keys import CourseKey

from course_discovery.apps.course_metadata.models import Course, CourseRun, Program


class OrganizationsMixin:
    def format_organization(self, organization):
        return '{key}: {name}'.format(key=organization.key, name=organization.name)

    def format_organization_body(self, organization):
        # Deferred to prevent a circular import:
        # course_discovery.apps.api.serializers -> course_discovery.apps.course_metadata.search_indexes
        from course_discovery.apps.api.serializers import OrganizationSerializer

        return json.dumps(OrganizationSerializer(organization).data)

    def _prepare_organizations(self, organizations):
        return [self.format_organization(organization) for organization in organizations]

    def prepare_authoring_organizations(self, obj):
        return self._prepare_organizations(obj.authoring_organizations.all())


class BaseIndex(indexes.SearchIndex):
    model = None

    text = indexes.CharField(document=True, use_template=True)
    content_type = indexes.CharField(faceted=True)

    def prepare_content_type(self, obj):  # pylint: disable=unused-argument
        return self.model.__name__.lower()

    def get_model(self):
        return self.model

    def get_updated_field(self):  # pragma: no cover
        return 'modified'

    def index_queryset(self, using=None):
        return self.model.objects.all()

    def prepare_authoring_organization_uuids(self, obj):
        return [str(organization.uuid) for organization in obj.authoring_organizations.all()]


class BaseCourseIndex(OrganizationsMixin, BaseIndex):
    key = indexes.CharField(model_attr='key', stored=True)
    title = indexes.CharField(model_attr='title')
    short_description = indexes.CharField(model_attr='short_description', null=True)
    full_description = indexes.CharField(model_attr='full_description', null=True)
    subjects = indexes.MultiValueField(faceted=True)
    organizations = indexes.MultiValueField(faceted=True)
    authoring_organizations = indexes.MultiValueField(faceted=True)
    sponsoring_organizations = indexes.MultiValueField(faceted=True)
    level_type = indexes.CharField(null=True, faceted=True)
    partner = indexes.CharField(model_attr='partner__short_code', null=True, faceted=True)

    def prepare_subjects(self, obj):
        return [subject.name for subject in obj.subjects.all()]

    def prepare_organizations(self, obj):
        return self.prepare_authoring_organizations(obj) + self.prepare_sponsoring_organizations(obj)

    def prepare_authoring_organizations(self, obj):
        return self._prepare_organizations(obj.authoring_organizations.all())

    def prepare_sponsoring_organizations(self, obj):
        return self._prepare_organizations(obj.sponsoring_organizations.all())

    def prepare_level_type(self, obj):
        return obj.level_type.name if obj.level_type else None


class CourseIndex(BaseCourseIndex, indexes.Indexable):
    model = Course

    course_runs = indexes.MultiValueField()
    expected_learning_items = indexes.MultiValueField()

    prerequisites = indexes.MultiValueField(faceted=True)

    def prepare_course_runs(self, obj):
        return [course_run.key for course_run in obj.course_runs.all()]

    def prepare_expected_learning_items(self, obj):
        return [item.value for item in obj.expected_learning_items.all()]

    def prepare_prerequisites(self, obj):
        return [prerequisite.name for prerequisite in obj.prerequisites.all()]


class CourseRunIndex(BaseCourseIndex, indexes.Indexable):
    model = CourseRun

    course_key = indexes.CharField(model_attr='course__key', stored=True)
    org = indexes.CharField()
    number = indexes.CharField()
    status = indexes.CharField(model_attr='status', faceted=True)
    start = indexes.DateTimeField(model_attr='start', null=True, faceted=True)
    end = indexes.DateTimeField(model_attr='end', null=True)
    enrollment_start = indexes.DateTimeField(model_attr='enrollment_start', null=True)
    enrollment_end = indexes.DateTimeField(model_attr='enrollment_end', null=True)
    announcement = indexes.DateTimeField(model_attr='announcement', null=True)
    min_effort = indexes.IntegerField(model_attr='min_effort', null=True)
    max_effort = indexes.IntegerField(model_attr='max_effort', null=True)
    language = indexes.CharField(null=True, faceted=True)
    transcript_languages = indexes.MultiValueField(faceted=True)
    pacing_type = indexes.CharField(model_attr='pacing_type', null=True, faceted=True)
    marketing_url = indexes.CharField(null=True)
    slug = indexes.CharField(model_attr='slug', null=True)
    seat_types = indexes.MultiValueField(model_attr='seat_types', null=True, faceted=True)
    type = indexes.CharField(model_attr='type', null=True, faceted=True)
    image_url = indexes.CharField(model_attr='card_image_url', null=True)
    partner = indexes.CharField(null=True, faceted=True)
    program_types = indexes.MultiValueField()
    published = indexes.BooleanField(null=False, faceted=True)
    authoring_organization_uuids = indexes.MultiValueField()
    staff_uuids = indexes.MultiValueField()
    subject_uuids = indexes.MultiValueField()

    def prepare_partner(self, obj):
        return obj.course.partner.short_code

    def prepare_published(self, obj):
        return obj.status == CourseRun.Status.Published

    def _prepare_language(self, language):
        if language:
            # ECOM-5466: Render the macro language for all languages except Chinese
            if language.code.startswith('zh'):
                return language.name
            else:
                return language.macrolanguage

        return None

    def prepare_language(self, obj):
        return self._prepare_language(obj.language)

    def prepare_number(self, obj):
        course_run_key = CourseKey.from_string(obj.key)
        return course_run_key.course

    def prepare_org(self, obj):
        course_run_key = CourseKey.from_string(obj.key)
        return course_run_key.org

    def prepare_transcript_languages(self, obj):
        return [self._prepare_language(language) for language in obj.transcript_languages.all()]

    def prepare_marketing_url(self, obj):
        return obj.marketing_url

    def prepare_program_types(self, obj):
        return obj.program_types

    def prepare_staff_uuids(self, obj):
        return [str(staff.uuid) for staff in obj.staff.all()]

    def prepare_subject_uuids(self, obj):
        return [str(subject.uuid) for subject in obj.subjects.all()]


class ProgramIndex(BaseIndex, indexes.Indexable, OrganizationsMixin):
    model = Program

    uuid = indexes.CharField(model_attr='uuid')
    title = indexes.CharField(model_attr='title')
    subtitle = indexes.CharField(model_attr='subtitle')
    type = indexes.CharField(model_attr='type__name', faceted=True)
    marketing_url = indexes.CharField(null=True)
    organizations = indexes.MultiValueField(faceted=True)
    authoring_organizations = indexes.MultiValueField(faceted=True)
    authoring_organization_uuids = indexes.MultiValueField()
    subject_uuids = indexes.MultiValueField()
    staff_uuids = indexes.MultiValueField()
    authoring_organization_bodies = indexes.MultiValueField()
    credit_backing_organizations = indexes.MultiValueField(faceted=True)
    card_image_url = indexes.CharField(model_attr='card_image_url', null=True)
    status = indexes.CharField(model_attr='status', faceted=True)
    partner = indexes.CharField(model_attr='partner__short_code', null=True, faceted=True)
    start = indexes.DateTimeField(model_attr='start', null=True, faceted=True)
    seat_types = indexes.MultiValueField(model_attr='seat_types', null=True, faceted=True)
    published = indexes.BooleanField(null=False, faceted=True)

    def prepare_published(self, obj):
        return obj.status == Program.Status.Active

    def prepare_organizations(self, obj):
        return self.prepare_authoring_organizations(obj) + self.prepare_credit_backing_organizations(obj)

    def prepare_authoring_organization_bodies(self, obj):
        return [self.format_organization_body(organization) for organization in obj.authoring_organizations.all()]

    def prepare_subject_uuids(self, obj):
        return [str(subject.uuid) for course in obj.courses.all() for subject in course.subjects.all()]

    def prepare_staff_uuids(self, obj):
        return [str(staff.uuid) for course_run in obj.course_runs.all() for staff in course_run.staff.all()]

    def prepare_credit_backing_organizations(self, obj):
        return self._prepare_organizations(obj.credit_backing_organizations.all())

    def prepare_marketing_url(self, obj):
        return obj.marketing_url
{'fraction_non_alphanumeric': 0.0946010797840432, 'fraction_numerical': 0.0004799040191961608, 'mean_word_length': 4.672337529772031, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1787463', 'n_tokens_mistral': 5032, 'n_tokens_neox': 4801, 'n_words': 885}