<commit_before>const ERR = require('async-stacktrace');
const async = require('async');
const path = require('path');
const Docker = require('dockerode');
const os = require('os');
const EventEmitter = require('events');
const fs = require('fs-extra');
const { exec } = require('child_process');
const byline = require('byline');

const logger = require('./logger');
const externalGraderCommon = require('./externalGraderCommon');
const config = require('./config');
const sqldb = require('./sqldb');
const sqlLoader = require('./sql-loader');

const sql = sqlLoader.loadSqlEquiv(__filename);

class Grader {
    handleGradingRequest(grading_job, submission, variant, question, course) {
        const emitter = new EventEmitter();

        const dir = getDevJobDirectory(grading_job.id);
        const hostDir = getDevHostJobDirectory(grading_job.id);
        const timeout = question.external_grading_timeout || config.externalGradingDefaultTimeout;

        const docker = new Docker();

        const results = {
            startTime: new Date().toISOString(),
        };

        let output = '';
        async.waterfall([
            (callback) => {
                docker.ping((err) => {
                    if (ERR(err, callback)) return;
                    callback(null);
                });
            },
            (callback) => {
                externalGraderCommon.buildDirectory(dir, submission, variant, question, course, (err) => {
                    if (ERR(err, callback)) return;
                    callback(null);
                });
            },
            (callback) => {
                exec(`chmod +x ${path.join(dir, question.external_grading_entrypoint.slice(6))}`, (err) => {
                    if (err) {
                        logger.error('Could not make file executable; continuing execution anyways');
                    }
                    callback(null);
                });
            },
            (callback) => {
                emitter.emit('submit');
                callback(null);
            },
            (callback) => {
                docker.pull(question.external_grading_image, (err, stream) => {
                    if (err) {
                        logger.warn(`Error pulling "${question.external_grading_image}" image; attempting to fall back to cached version.`);
                        logger.warn(err);
                        return callback(null);
                    }

                    docker.modem.followProgress(stream, (err) => {
                        if (ERR(err, callback)) return;
                        callback(null);
                    }, (output) => {
                        logger.info(output);
                    });
                });
            },
            (callback) => {
                docker.createContainer({
                    Image: question.external_grading_image,
                    AttachStdout: true,
                    AttachStderr: true,
                    Tty: true,
                    HostConfig: {
                        Binds: [
                            `${hostDir}:/grade`,
                        ],
                    },
                    Entrypoint: question.external_grading_entrypoint.split(' '),
                }, (err, container) => {
                    if (ERR(err, callback)) return;
                    callback(null, container);
                });
            },
            (container, callback) => {
                container.attach({
                    stream: true,
                    stdout: true,
                    stderr: true,
                }, (err, stream) => {
                    if (ERR(err, callback)) return;
                    const out = byline(stream);
                    out.on('data', (line) => {
                        const newline = `container> ${line.toString('utf8')}`;
                        logger.info(newline);
                        output += (newline + '\n');
                    });<commit_msg>Keep empty lines while logging<commit_after>const ERR = require('async-stacktrace');
const async = require('async');
const path = require('path');
const Docker = require('dockerode');
const os = require('os');
const EventEmitter = require('events');
const fs = require('fs-extra');
const { exec } = require('child_process');
const byline = require('byline');

const logger = require('./logger');
const externalGraderCommon = require('./externalGraderCommon');
const config = require('./config');
const sqldb = require('./sqldb');
const sqlLoader = require('./sql-loader');

const sql = sqlLoader.loadSqlEquiv(__filename);

class Grader {
    handleGradingRequest(grading_job, submission, variant, question, course) {
        const emitter = new EventEmitter();

        const dir = getDevJobDirectory(grading_job.id);
        const hostDir = getDevHostJobDirectory(grading_job.id);
        const timeout = question.external_grading_timeout || config.externalGradingDefaultTimeout;

        const docker = new Docker();

        const results = {
            startTime: new Date().toISOString(),
        };

        let output = '';
        async.waterfall([
            (callback) => {
                docker.ping((err) => {
                    if (ERR(err, callback)) return;
                    callback(null);
                });
            },
            (callback) => {
                externalGraderCommon.buildDirectory(dir, submission, variant, question, course, (err) => {
                    if (ERR(err, callback)) return;
                    callback(null);
                });
            },
            (callback) => {
                exec(`chmod +x ${path.join(dir, question.external_grading_entrypoint.slice(6))}`, (err) => {
                    if (err) {
                        logger.error('Could not make file executable; continuing execution anyways');
                    }
                    callback(null);
                });
            },
            (callback) => {
                emitter.emit('submit');
                callback(null);
            },
            (callback) => {
                docker.pull(question.external_grading_image, (err, stream) => {
                    if (err) {
                        logger.warn(`Error pulling "${question.external_grading_image}" image; attempting to fall back to cached version.`);
                        logger.warn(err);
                        return callback(null);
                    }

                    docker.modem.followProgress(stream, (err) => {
                        if (ERR(err, callback)) return;
                        callback(null);
                    }, (output) => {
                        logger.info(output);
                    });
                });
            },
            (callback) => {
                docker.createContainer({
                    Image: question.external_grading_image,
                    AttachStdout: true,
                    AttachStderr: true,
                    Tty: true,
                    HostConfig: {
                        Binds: [
                            `${hostDir}:/grade`,
                        ],
                    },
                    Entrypoint: question.external_grading_entrypoint.split(' '),
                }, (err, container) => {
                    if (ERR(err, callback)) return;
                    callback(null, container);
                });
            },
            (container, callback) => {
                container.attach({
                    stream: true,
                    stdout: true,
                    stderr: true,
                }, (err, stream) => {
                    if (ERR(err, callback)) return;
                    const out = byline(stream, { keepEmptyLines: true });
                    out.on('data', (line) => {
                        const newline = `container> ${line.toString('utf8')}`;
                        logger.info(newline);
                        output += (newline + '\n');
                    });
{'fraction_non_alphanumeric': 0.1222707423580786, 'fraction_numerical': 0.0004990642545227698, 'mean_word_length': 1.3500439753737907, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6407303', 'n_tokens_mistral': 1960, 'n_tokens_neox': 1817, 'n_words': 400}