<commit_before>var _ = require('lodash');
var kramed = require('kramed');

// Render a section using our custom renderer
function render(section) {
    // Copy section
    var links = section.links || {};
    section = _.toArray(section);
    section.links = links;

    return kramed.parser(section);
}

function quizQuestion(node) {
    if (node.text) {
        node.text = node.text.replace(/^([\[(])x([\])])/, "$1 $2");
    } else {
        return node.replace(/^([\[(])x([\])])/, "$1 $2");
    }
}

function section2obj(section) {
    var quiz = [], question, foundFeedback = false;
    var nonQuizNodes = section[0].type === 'paragraph' && section[1].type !== 'list_start' ? [section[0]] : [];
    var quizNodes = section.slice(0);
    quizNodes.splice(0, nonQuizNodes.length);

    for (var i = 0; i < quizNodes.length; i++) {
        var node = quizNodes[i];

        if (question && (((node.type === 'list_end' || node.type === 'blockquote_end') && i === quizNodes.length - 1)
                         || node.type === 'table' || (node.type === 'paragraph' && !foundFeedback))) {
            quiz.push({
                base: render(question.questionNodes),
                solution: render(question.solutionNodes),
                feedback: render(question.feedbackNodes)
            });
        }

        if (node.type === 'table' || (node.type === 'paragraph' && !foundFeedback)) {
            question = { questionNodes: [], solutionNodes: [], feedbackNodes: [] };
        }<commit_msg>Improve robustness by skipping "space" nodes<commit_after>var _ = require('lodash');
var kramed = require('kramed');

// Render a section using our custom renderer
function render(section) {
    // Copy section
    var links = section.links || {};
    section = _.toArray(section);
    section.links = links;

    return kramed.parser(section);
}

function quizQuestion(node) {
    if (node.text) {
        node.text = node.text.replace(/^([\[(])x([\])])/, "$1 $2");
    } else {
        return node.replace(/^([\[(])x([\])])/, "$1 $2");
    }
}

function section2obj(section) {
    // Skip space nodes for robustness
    var nodes = _.filter(section, function(node) {
        return node && node.type !== 'space';
    });

    var quiz = [], question, foundFeedback = false;
    var nonQuizNodes = nodes[0].type === 'paragraph' && nodes[1].type !== 'list_start' ? [nodes[0]] : [];
    var quizNodes = nodes.slice(0);
    quizNodes.splice(0, nonQuizNodes.length);

    for (var i = 0; i < quizNodes.length; i++) {
        var node = quizNodes[i];

        if (question && (((node.type === 'list_end' || node.type === 'blockquote_end') && i === quizNodes.length - 1)
                         || node.type === 'table' || (node.type === 'paragraph' && !foundFeedback))) {
            quiz.push({
                base: render(question.questionNodes),
                solution: render(question.solutionNodes),
                feedback: render(question.feedbackNodes)
            });
        }

        if (node.type === 'table' || (node.type === 'paragraph' && !foundFeedback)) {
            question = { questionNodes: [], solutionNodes: [], feedbackNodes: [] };
        }
{'fraction_non_alphanumeric': 0.18592171717171718, 'fraction_numerical': 0.007575757575757576, 'mean_word_length': 2.5447427293064875, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1412088', 'n_tokens_mistral': 980, 'n_tokens_neox': 937, 'n_words': 214}