<commit_before>from jinja2 import Environment
from IPython.nbconvert.preprocessors import ExecutePreprocessor
from IPython.utils.traitlets import Bool, Unicode
from IPython.nbformat.current import read as read_nb
import numpy as np


class AssignmentPreprocessor(ExecutePreprocessor):

    solution = Bool(False, config=True, info_test="Whether to generate the release version, or the solutions")
    header = Unicode("", config=True, info_test="Path to header notebook")
    footer = Unicode("", config=True, info_test="Path to footer notebook")
    title = Unicode("", config=True, info_test="Title of the assignment")
    disable_toolbar = Bool(True, config=True, info_test="Whether to hide the assignment toolbar after conversion")
    hide_autograder_cells = Bool(True, config=True, info_test="Whether to hide autograder cells after conversion")

    def __init__(self, *args, **kwargs):
        super(AssignmentPreprocessor, self).__init__(*args, **kwargs)

        # create the jinja templating environment
        self.env = Environment(
            trim_blocks=True,
            lstrip_blocks=True,
            keep_trailing_newline=False)

    def _concatenate_notebooks(self, cells):
        """Concatenates the cells from the header and footer notebooks to the
        given cells.

        """
        new_cells = []

        # header
        if self.header != "":
            with open(self.header, 'r') as fh:
                header_nb = read_nb(fh, 'ipynb')
            new_cells.extend(header_nb.worksheets[0].cells)

        # body
        new_cells.extend(cells)

        # footer
        if self.footer != "":
            with open(self.footer, 'r') as fh:
                footer_nb = read_nb(fh, 'ipynb')
            new_cells.extend(footer_nb.worksheets[0].cells)

        return new_cells

    @staticmethod
    def _get_assignment_cell_type(cell):
        """Get the assignment cell type from the assignment metadata."""
        if 'assignment' not in cell.metadata:
            return '-'
        return cell.metadata['assignment']['cell_type']

    def _filter_cells(self, cells):
        """Filter out cells, depending on the value of `self.solution`:

        * always filter out 'skip' cells
        * if self.solution == True, then filter out 'release' cells
        * if self.release == True, then filter out 'solution' cells

        """
        new_cells = []
        for cell in cells:

            # get the cell type
            cell_type = self._get_assignment_cell_type(cell)

            # determine whether the cell should be included
            if cell_type == 'skip':
                continue
            elif cell_type == 'release' and self.solution:
                continue
            elif cell_type == 'solution' and not self.solution:
                continue
            else:
                new_cells.append(cell)

        return new_cells

    @staticmethod
    def _get_toc(cells):
        """Extract a table of contents from the cells, based on the headings
        in the cells."""

        # get the minimum heading level
        try:
            min_level = min(x.level for x in cells if x.cell_type == 'heading')
        except ValueError:
            min_level = 1

        toc = []
        for cell in cells:
            # skip non-heading cells
            if cell.cell_type != 'heading':
                continue

            # format the indentation and anchor link
            level = cell.level
            source = cell.source
            indent = "\t" * (level - min_level)
            link = '<a href="#{}">{}</a>'.format(
                source.replace(" ", "-"), source)
            toc.append("{}* {}".format(indent, link))

        toc = "\n".join(toc)
        return toc

    def _match_tests(self, cells):
        """Determine which tests correspond to which problems."""

        tests = {}
        rubric = {}

        last_problem = None
        last_problem_id = None

        for cell in cells:
            cell_type = self._get_assignment_cell_type(cell)

            # if it's a grading cell, then it becomes the most recent
            # problem (to assign the autograding tests to)
            if cell_type == "grade":
                last_problem = cell
                last_problem_id = last_problem.metadata['assignment']['id']

                if last_problem_id in rubric:
                    raise ValueError(
                        "problem '{}' already exists!".format(
                            last_problem_id))

                # add the problem to the rubric
                points = float(last_problem.metadata['assignment']['points'])
                rubric[last_problem_id] = {
                    'tests': [],
                    'points': points
                }

            # if it's an autograder cell, then we need to add it to
            # the list of tests for the last problem cell and record
            # it in the tests dictionary
            elif cell_type == "autograder":
                if not last_problem:
                    raise RuntimeError(
                        "autograding cell before any gradeable cells!")

                cell_id = cell.metadata['assignment']['id']
                if cell_id in tests:
                    raise ValueError(
                        "test id '{}' is used more than once".format(cell_id))

                # add the test to the tests dictionary
                tests[cell_id] = {
                    'weight': 1,
                    'problem': last_problem_id
                }

                # add the test to the list of tests for the problem
                rubric[last_problem_id]['tests'].append(cell_id)

        # now normalize the test weights so that the weights of all
        # tests for a particular problem sum to one.
        for problem in rubric:
            # the total number of points this problem is worth
            total_points = float(rubric[problem]['points'])

            # get the list of tests for this problem
            problem_tests = rubric[problem]['tests']

            # compute the normalizing constant for the weights
            normalizer = float(np.sum(
                np.array([tests[t]['weight'] for t in problem_tests])))

            # update weights and points for each test
            for test in problem_tests:
                tests[test]['weight'] /= normalizer
                tests[test]['points'] = total_points * tests[test]['weight']

        return tests, rubric

    def _preprocess_nb(self, nb, resources):
        cells = nb.worksheets[0].cells

        # add in the header and the footer notebooks
        cells = self._concatenate_notebooks(cells)

        # filter out various cells
        cells = self._filter_cells(cells)

        # get the table of contents -- important that this happens<commit_msg>Don't depend on numpy in assignment preprocessor<commit_after>from jinja2 import Environment
from IPython.nbconvert.preprocessors import ExecutePreprocessor
from IPython.utils.traitlets import Bool, Unicode
from IPython.nbformat.current import read as read_nb


class AssignmentPreprocessor(ExecutePreprocessor):

    solution = Bool(False, config=True, info_test="Whether to generate the release version, or the solutions")
    header = Unicode("", config=True, info_test="Path to header notebook")
    footer = Unicode("", config=True, info_test="Path to footer notebook")
    title = Unicode("", config=True, info_test="Title of the assignment")
    disable_toolbar = Bool(True, config=True, info_test="Whether to hide the assignment toolbar after conversion")
    hide_autograder_cells = Bool(True, config=True, info_test="Whether to hide autograder cells after conversion")

    def __init__(self, *args, **kwargs):
        super(AssignmentPreprocessor, self).__init__(*args, **kwargs)

        # create the jinja templating environment
        self.env = Environment(
            trim_blocks=True,
            lstrip_blocks=True,
            keep_trailing_newline=False)

    def _concatenate_notebooks(self, cells):
        """Concatenates the cells from the header and footer notebooks to the
        given cells.

        """
        new_cells = []

        # header
        if self.header != "":
            with open(self.header, 'r') as fh:
                header_nb = read_nb(fh, 'ipynb')
            new_cells.extend(header_nb.worksheets[0].cells)

        # body
        new_cells.extend(cells)

        # footer
        if self.footer != "":
            with open(self.footer, 'r') as fh:
                footer_nb = read_nb(fh, 'ipynb')
            new_cells.extend(footer_nb.worksheets[0].cells)

        return new_cells

    @staticmethod
    def _get_assignment_cell_type(cell):
        """Get the assignment cell type from the assignment metadata."""
        if 'assignment' not in cell.metadata:
            return '-'
        return cell.metadata['assignment']['cell_type']

    def _filter_cells(self, cells):
        """Filter out cells, depending on the value of `self.solution`:

        * always filter out 'skip' cells
        * if self.solution == True, then filter out 'release' cells
        * if self.release == True, then filter out 'solution' cells

        """
        new_cells = []
        for cell in cells:

            # get the cell type
            cell_type = self._get_assignment_cell_type(cell)

            # determine whether the cell should be included
            if cell_type == 'skip':
                continue
            elif cell_type == 'release' and self.solution:
                continue
            elif cell_type == 'solution' and not self.solution:
                continue
            else:
                new_cells.append(cell)

        return new_cells

    @staticmethod
    def _get_toc(cells):
        """Extract a table of contents from the cells, based on the headings
        in the cells."""

        # get the minimum heading level
        try:
            min_level = min(x.level for x in cells if x.cell_type == 'heading')
        except ValueError:
            min_level = 1

        toc = []
        for cell in cells:
            # skip non-heading cells
            if cell.cell_type != 'heading':
                continue

            # format the indentation and anchor link
            level = cell.level
            source = cell.source
            indent = "\t" * (level - min_level)
            link = '<a href="#{}">{}</a>'.format(
                source.replace(" ", "-"), source)
            toc.append("{}* {}".format(indent, link))

        toc = "\n".join(toc)
        return toc

    def _match_tests(self, cells):
        """Determine which tests correspond to which problems."""

        tests = {}
        rubric = {}

        last_problem = None
        last_problem_id = None

        for cell in cells:
            cell_type = self._get_assignment_cell_type(cell)

            # if it's a grading cell, then it becomes the most recent
            # problem (to assign the autograding tests to)
            if cell_type == "grade":
                last_problem = cell
                last_problem_id = last_problem.metadata['assignment']['id']

                if last_problem_id in rubric:
                    raise ValueError(
                        "problem '{}' already exists!".format(
                            last_problem_id))

                # add the problem to the rubric
                points = float(last_problem.metadata['assignment']['points'])
                rubric[last_problem_id] = {
                    'tests': [],
                    'points': points
                }

            # if it's an autograder cell, then we need to add it to
            # the list of tests for the last problem cell and record
            # it in the tests dictionary
            elif cell_type == "autograder":
                if not last_problem:
                    raise RuntimeError(
                        "autograding cell before any gradeable cells!")

                cell_id = cell.metadata['assignment']['id']
                if cell_id in tests:
                    raise ValueError(
                        "test id '{}' is used more than once".format(cell_id))

                # add the test to the tests dictionary
                tests[cell_id] = {
                    'weight': 1,
                    'problem': last_problem_id
                }

                # add the test to the list of tests for the problem
                rubric[last_problem_id]['tests'].append(cell_id)

        # now normalize the test weights so that the weights of all
        # tests for a particular problem sum to one.
        for problem in rubric:
            # the total number of points this problem is worth
            total_points = float(rubric[problem]['points'])

            # get the list of tests for this problem
            problem_tests = rubric[problem]['tests']

            # compute the normalizing constant for the weights
            normalizer = float(sum(
                [tests[t]['weight'] for t in problem_tests]))

            # update weights and points for each test
            for test in problem_tests:
                tests[test]['weight'] /= normalizer
                tests[test]['points'] = total_points * tests[test]['weight']

        return tests, rubric

    def _preprocess_nb(self, nb, resources):
        cells = nb.worksheets[0].cells

        # add in the header and the footer notebooks
        cells = self._concatenate_notebooks(cells)

        # filter out various cells
        cells = self._filter_cells(cells)

        # get the table of contents -- important that this happens
{'fraction_non_alphanumeric': 0.08738574040219378, 'fraction_numerical': 0.0008775137111517368, 'mean_word_length': 1.867687146152233, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2958962', 'n_tokens_mistral': 3717, 'n_tokens_neox': 3417, 'n_words': 1133}