<commit_before>import datetime
from itertools import groupby

from django.db import models
from django.db.models import Case, Count, IntegerField, Max, Sum, When
from django.utils.timezone import now

from analytics_data_api.constants import country, genders
from analytics_data_api.constants.engagement_types import EngagementType
from analytics_data_api.utils import date_range


class BaseCourseModel(models.Model):
    course_id = models.CharField(db_index=True, max_length=255)
    created = models.DateTimeField(auto_now_add=True)

    class Meta:
        abstract = True


class CourseActivityWeekly(BaseCourseModel):
    """A count of unique users who performed a particular action during a week."""

    class Meta(BaseCourseModel.Meta):
        db_table = 'course_activity'
        index_together = [['course_id', 'activity_type']]
        ordering = ('interval_end', 'interval_start', 'course_id')
        get_latest_by = 'interval_end'

    interval_start = models.DateTimeField()
    interval_end = models.DateTimeField(db_index=True)
    activity_type = models.CharField(db_index=True, max_length=255, db_column='label')
    count = models.IntegerField()

    @classmethod
    def get_most_recent(cls, course_id, activity_type):
        """Activity for the week that was mostly recently computed."""
        return cls.objects.filter(course_id=course_id, activity_type=activity_type).latest('interval_end')


class BaseCourseEnrollment(BaseCourseModel):
    date = models.DateField(null=False, db_index=True)
    count = models.IntegerField(null=False)

    class Meta(BaseCourseModel.Meta):
        abstract = True
        get_latest_by = 'date'
        index_together = [('course_id', 'date',)]


class CourseEnrollmentDaily(BaseCourseEnrollment):
    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_daily'
        ordering = ('date', 'course_id')
        unique_together = [('course_id', 'date',)]


class CourseEnrollmentModeDaily(BaseCourseEnrollment):
    mode = models.CharField(max_length=255)
    cumulative_count = models.IntegerField(null=False)

    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_mode_daily'
        ordering = ('date', 'course_id', 'mode')
        unique_together = [('course_id', 'date', 'mode')]


class CourseMetaSummaryEnrollment(BaseCourseModel):
    catalog_course_title = models.CharField(db_index=True, max_length=255)
    catalog_course = models.CharField(db_index=True, max_length=255)
    start_time = models.DateTimeField()
    end_time = models.DateTimeField()
    pacing_type = models.CharField(db_index=True, max_length=255)
    availability = models.CharField(db_index=True, max_length=255)
    enrollment_mode = models.CharField(max_length=255)
    count = models.IntegerField(null=False)
    cumulative_count = models.IntegerField(null=False)
    count_change_7_days = models.IntegerField(default=0)
    passing_users = models.IntegerField(default=0)

    class Meta(BaseCourseModel.Meta):
        db_table = 'course_meta_summary_enrollment'
        ordering = ('course_id',)
        unique_together = [('course_id', 'enrollment_mode',)]


class CourseProgramMetadata(BaseCourseModel):
    program_id = models.CharField(db_index=True, max_length=255)
    program_type = models.CharField(db_index=True, max_length=255)
    program_title = models.CharField(max_length=255)

    class Meta(BaseCourseModel.Meta):
        db_table = 'course_program_metadata'
        ordering = ('program_id',)
        unique_together = [('course_id', 'program_id',)]


class CourseEnrollmentByBirthYear(BaseCourseEnrollment):
    birth_year = models.IntegerField(null=False)

    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_birth_year_daily'
        ordering = ('date', 'course_id', 'birth_year')
        unique_together = [('course_id', 'date', 'birth_year')]


class CourseEnrollmentByEducation(BaseCourseEnrollment):
    education_level = models.CharField(max_length=255, null=True)

    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_education_level_daily'
        ordering = ('date', 'course_id', 'education_level')
        unique_together = [('course_id', 'date', 'education_level')]


class CourseEnrollmentByGender(BaseCourseEnrollment):
    CLEANED_GENDERS = {
        'f': genders.FEMALE,
        'm': genders.MALE,
        'o': genders.OTHER
    }

    gender = models.CharField(max_length=255, null=True, db_column='gender')

    @property
    def cleaned_gender(self):
        """
        Returns the gender with full names and 'unknown' replacing null/None.
        """
        return self.CLEANED_GENDERS.get(self.gender, genders.UNKNOWN)

    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_gender_daily'
        ordering = ('date', 'course_id', 'gender')
        unique_together = [('course_id', 'date', 'gender')]


class BaseProblemResponseAnswerDistribution(BaseCourseModel):
    """ Base model for the answer_distribution table. """

    class Meta(BaseCourseModel.Meta):
        db_table = 'answer_distribution'
        abstract = True

    module_id = models.CharField(db_index=True, max_length=255)
    part_id = models.CharField(db_index=True, max_length=255)
    correct = models.NullBooleanField()
    value_id = models.CharField(db_index=True, max_length=255, null=True)
    answer_value = models.TextField(null=True, db_column='answer_value_text')
    variant = models.IntegerField(null=True)
    problem_display_name = models.TextField(null=True)
    question_text = models.TextField(null=True)


class ProblemResponseAnswerDistribution(BaseProblemResponseAnswerDistribution):
    """ Original model for the count of a particular answer to a response to a problem in a course. """

    class Meta(BaseProblemResponseAnswerDistribution.Meta):
        managed = False

    count = models.IntegerField()


class ProblemsAndTags(BaseCourseModel):
    """ Model for the tags_distribution table """

    class Meta(BaseCourseModel.Meta):
        db_table = 'tags_distribution'

    module_id = models.CharField(db_index=True, max_length=255)
    tag_name = models.CharField(max_length=255)
    tag_value = models.CharField(max_length=255)
    total_submissions = models.IntegerField(default=0)
    correct_submissions = models.IntegerField(default=0)


class ProblemFirstLastResponseAnswerDistribution(BaseProblemResponseAnswerDistribution):
    """ Updated model for answer_distribution table with counts of first and last attempts at problems. """

    class Meta(BaseProblemResponseAnswerDistribution.Meta):
        verbose_name = 'first_last_answer_distribution'

    first_response_count = models.IntegerField()
    last_response_count = models.IntegerField()


class CourseEnrollmentByCountry(BaseCourseEnrollment):
    country_code = models.CharField(max_length=255, null=False, db_column='country_code')

    @property
    def country(self):
        """
        Returns a Country object representing the country in this model's country_code.
        """
        return country.get_country(self.country_code)

    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_location_current'
        ordering = ('date', 'course_id', 'country_code')
        unique_together = [('course_id', 'date', 'country_code')]


class GradeDistribution(BaseCourseModel):
    """ Each row stores the count of a particular grade on a module for a given course. """

    class Meta(BaseCourseModel.Meta):
        db_table = 'grade_distribution'

    module_id = models.CharField(db_index=True, max_length=255)
    grade = models.IntegerField()
    max_grade = models.IntegerField()
    count = models.IntegerField()


class SequentialOpenDistribution(BaseCourseModel):
    """ Each row stores the count of views a particular module has had in a given course. """

    class Meta(BaseCourseModel.Meta):
        db_table = 'sequential_open_distribution'

    module_id = models.CharField(db_index=True, max_length=255)<commit_msg>fix: strip unnecessary expensive ordering<commit_after>import datetime
from itertools import groupby

from django.db import models
from django.db.models import Case, Count, IntegerField, Max, Sum, When
from django.utils.timezone import now

from analytics_data_api.constants import country, genders
from analytics_data_api.constants.engagement_types import EngagementType
from analytics_data_api.utils import date_range


class BaseCourseModel(models.Model):
    course_id = models.CharField(db_index=True, max_length=255)
    created = models.DateTimeField(auto_now_add=True)

    class Meta:
        abstract = True


class CourseActivityWeekly(BaseCourseModel):
    """A count of unique users who performed a particular action during a week."""

    class Meta(BaseCourseModel.Meta):
        db_table = 'course_activity'
        index_together = [['course_id', 'activity_type']]
        ordering = ('interval_end', 'interval_start', 'course_id')
        get_latest_by = 'interval_end'

    interval_start = models.DateTimeField()
    interval_end = models.DateTimeField(db_index=True)
    activity_type = models.CharField(db_index=True, max_length=255, db_column='label')
    count = models.IntegerField()

    @classmethod
    def get_most_recent(cls, course_id, activity_type):
        """Activity for the week that was mostly recently computed."""
        return cls.objects.filter(course_id=course_id, activity_type=activity_type).latest('interval_end')


class BaseCourseEnrollment(BaseCourseModel):
    date = models.DateField(null=False, db_index=True)
    count = models.IntegerField(null=False)

    class Meta(BaseCourseModel.Meta):
        abstract = True
        get_latest_by = 'date'
        index_together = [('course_id', 'date',)]


class CourseEnrollmentDaily(BaseCourseEnrollment):
    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_daily'
        ordering = ('date', 'course_id')
        unique_together = [('course_id', 'date',)]


class CourseEnrollmentModeDaily(BaseCourseEnrollment):
    mode = models.CharField(max_length=255)
    cumulative_count = models.IntegerField(null=False)

    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_mode_daily'
        ordering = ('date', 'course_id')
        unique_together = [('course_id', 'date', 'mode')]


class CourseMetaSummaryEnrollment(BaseCourseModel):
    catalog_course_title = models.CharField(db_index=True, max_length=255)
    catalog_course = models.CharField(db_index=True, max_length=255)
    start_time = models.DateTimeField()
    end_time = models.DateTimeField()
    pacing_type = models.CharField(db_index=True, max_length=255)
    availability = models.CharField(db_index=True, max_length=255)
    enrollment_mode = models.CharField(max_length=255)
    count = models.IntegerField(null=False)
    cumulative_count = models.IntegerField(null=False)
    count_change_7_days = models.IntegerField(default=0)
    passing_users = models.IntegerField(default=0)

    class Meta(BaseCourseModel.Meta):
        db_table = 'course_meta_summary_enrollment'
        ordering = ('course_id',)
        unique_together = [('course_id', 'enrollment_mode',)]


class CourseProgramMetadata(BaseCourseModel):
    program_id = models.CharField(db_index=True, max_length=255)
    program_type = models.CharField(db_index=True, max_length=255)
    program_title = models.CharField(max_length=255)

    class Meta(BaseCourseModel.Meta):
        db_table = 'course_program_metadata'
        ordering = ('program_id',)
        unique_together = [('course_id', 'program_id',)]


class CourseEnrollmentByBirthYear(BaseCourseEnrollment):
    birth_year = models.IntegerField(null=False)

    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_birth_year_daily'
        ordering = ('date', 'course_id')
        unique_together = [('course_id', 'date', 'birth_year')]


class CourseEnrollmentByEducation(BaseCourseEnrollment):
    education_level = models.CharField(max_length=255, null=True)

    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_education_level_daily'
        ordering = ('date', 'course_id')
        unique_together = [('course_id', 'date', 'education_level')]


class CourseEnrollmentByGender(BaseCourseEnrollment):
    CLEANED_GENDERS = {
        'f': genders.FEMALE,
        'm': genders.MALE,
        'o': genders.OTHER
    }

    gender = models.CharField(max_length=255, null=True, db_column='gender')

    @property
    def cleaned_gender(self):
        """
        Returns the gender with full names and 'unknown' replacing null/None.
        """
        return self.CLEANED_GENDERS.get(self.gender, genders.UNKNOWN)

    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_gender_daily'
        ordering = ('date', 'course_id')
        unique_together = [('course_id', 'date', 'gender')]


class BaseProblemResponseAnswerDistribution(BaseCourseModel):
    """ Base model for the answer_distribution table. """

    class Meta(BaseCourseModel.Meta):
        db_table = 'answer_distribution'
        abstract = True

    module_id = models.CharField(db_index=True, max_length=255)
    part_id = models.CharField(db_index=True, max_length=255)
    correct = models.NullBooleanField()
    value_id = models.CharField(db_index=True, max_length=255, null=True)
    answer_value = models.TextField(null=True, db_column='answer_value_text')
    variant = models.IntegerField(null=True)
    problem_display_name = models.TextField(null=True)
    question_text = models.TextField(null=True)


class ProblemResponseAnswerDistribution(BaseProblemResponseAnswerDistribution):
    """ Original model for the count of a particular answer to a response to a problem in a course. """

    class Meta(BaseProblemResponseAnswerDistribution.Meta):
        managed = False

    count = models.IntegerField()


class ProblemsAndTags(BaseCourseModel):
    """ Model for the tags_distribution table """

    class Meta(BaseCourseModel.Meta):
        db_table = 'tags_distribution'

    module_id = models.CharField(db_index=True, max_length=255)
    tag_name = models.CharField(max_length=255)
    tag_value = models.CharField(max_length=255)
    total_submissions = models.IntegerField(default=0)
    correct_submissions = models.IntegerField(default=0)


class ProblemFirstLastResponseAnswerDistribution(BaseProblemResponseAnswerDistribution):
    """ Updated model for answer_distribution table with counts of first and last attempts at problems. """

    class Meta(BaseProblemResponseAnswerDistribution.Meta):
        verbose_name = 'first_last_answer_distribution'

    first_response_count = models.IntegerField()
    last_response_count = models.IntegerField()


class CourseEnrollmentByCountry(BaseCourseEnrollment):
    country_code = models.CharField(max_length=255, null=False, db_column='country_code')

    @property
    def country(self):
        """
        Returns a Country object representing the country in this model's country_code.
        """
        return country.get_country(self.country_code)

    class Meta(BaseCourseEnrollment.Meta):
        db_table = 'course_enrollment_location_current'
        ordering = ('date', 'course_id')
        unique_together = [('course_id', 'date', 'country_code')]


class GradeDistribution(BaseCourseModel):
    """ Each row stores the count of a particular grade on a module for a given course. """

    class Meta(BaseCourseModel.Meta):
        db_table = 'grade_distribution'

    module_id = models.CharField(db_index=True, max_length=255)
    grade = models.IntegerField()
    max_grade = models.IntegerField()
    count = models.IntegerField()


class SequentialOpenDistribution(BaseCourseModel):
    """ Each row stores the count of views a particular module has had in a given course. """

    class Meta(BaseCourseModel.Meta):
        db_table = 'sequential_open_distribution'

    module_id = models.CharField(db_index=True, max_length=255)
{'fraction_non_alphanumeric': 0.0995706017798245, 'fraction_numerical': 0.008836890907959425, 'mean_word_length': 4.425388251181634, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6414643', 'n_tokens_mistral': 4931, 'n_tokens_neox': 4637, 'n_words': 948}