<commit_before>'use strict';

const { stringifyForLogs } = require('../utils/logs');

const MAX_SQL_LENGTH = (global.settings.logQueries && global.settings.maxQueriesLogLength) || 1024;
const TYPES = {
    QUERY: 'query',
    JOB: 'job'
};

module.exports = function log(sqlType = TYPES.QUERY) {
    return function logMiddleware(req, res, next) {
        const logObj = {
            request: {
                sql: prepareSQL(res.locals.params.sql, sqlType)
            }
        };

        res.set('X-SQLAPI-Log', stringifyForLogs(logObj));

        return next();
    };
};

module.exports.TYPES = TYPES;

function prepareSQL(sql, sqlType) {
    if (!sql || !global.settings.logQueries) {
        return null;
    }


    if (typeof sql === 'string') {
        return {
            type: sqlType,
            sql: ensureMaxQueryLength(sql)
        };
    }

    if (Array.isArray(sql)) {
        return {
            type: sqlType,
            sql: sql.map(q => ensureMaxQueryLength(q))
        };
    }

    if (sql.query && Array.isArray(sql.query)) {
        return {
            type: sqlType,
            sql: prepareBatchFallbackQuery(sql)
        };
    }
}

/**
 * Process a Batch API fallback query controlling the queries length
 * We need to create a new object avoiding original modifications
 *
 * @param {Object} sql
 */
function prepareBatchFallbackQuery(sql) {
    const fallbackQuery = {};

    if (sql.onsuccess) {
        fallbackQuery.onsuccess = ensureMaxQueryLength(sql.onsuccess);
    }

    if (sql.onerror) {
        fallbackQuery.onerror = ensureMaxQueryLength(sql.onerror);
    }

    fallbackQuery.query = sql.query.map(query => {
        const subquery = {
            query: ensureMaxQueryLength(query.query)
        };

        if (query.onsuccess) {
            subquery.onsuccess = ensureMaxQueryLength(query.onsuccess);
        }

        if (query.onerror) {
            subquery.onerror = ensureMaxQueryLength(query.onerror);
        }

        return subquery;
    });

    return fallbackQuery;
}

function ensureMaxQueryLength(sql) {
    return sql.substring(0, MAX_SQL_LENGTH);
}<commit_msg>Limit the size of X-SQLAPI-Log for multiqueries<commit_after>'use strict';

const { stringifyForLogs } = require('../utils/logs');

const MAX_SQL_LENGTH = (global.settings.logQueries && global.settings.maxQueriesLogLength) || 1024;

// This is used to set a hard limit in the header size
// While Node accepts headers of up to 8192 character, but different libraries impose other limits
// This might break the JSON structure of the log
const HEADER_HARD_LIMIT = 4096;

const TYPES = {
    QUERY: 'query',
    JOB: 'job'
};

module.exports = function log(sqlType = TYPES.QUERY) {
    return function logMiddleware(req, res, next) {
        const logObj = {
            request: {
                sql: prepareSQL(res.locals.params.sql, sqlType)
            }
        };

        res.set('X-SQLAPI-Log', stringifyForLogs(logObj).substring(0, HEADER_HARD_LIMIT));

        return next();
    };
};

module.exports.TYPES = TYPES;

function prepareSQL(sql, sqlType) {
    if (!sql || !global.settings.logQueries) {
        return null;
    }


    if (typeof sql === 'string') {
        return {
            type: sqlType,
            sql: ensureMaxQueryLength(sql)
        };
    }

    if (Array.isArray(sql)) {
        const lengthPerQuery = MAX_SQL_LENGTH / sql.length;
        return {
            type: sqlType,
            sql: sql.map(q => ensureMaxQueryLength(q, lengthPerQuery))
        };
    }

    if (sql.query && Array.isArray(sql.query)) {
        return {
            type: sqlType,
            sql: prepareBatchFallbackQuery(sql)
        };
    }
}

/**
 * Process a Batch API fallback query controlling the queries length
 * We need to create a new object avoiding original modifications
 *
 * @param {Object} sql
 */
function prepareBatchFallbackQuery(sql) {
    const fallbackQuery = {};

    if (sql.onsuccess) {
        fallbackQuery.onsuccess = ensureMaxQueryLength(sql.onsuccess);
    }

    if (sql.onerror) {
        fallbackQuery.onerror = ensureMaxQueryLength(sql.onerror);
    }

    fallbackQuery.query = sql.query.map(query => {
        const subquery = {
            query: ensureMaxQueryLength(query.query)
        };

        if (query.onsuccess) {
            subquery.onsuccess = ensureMaxQueryLength(query.onsuccess);
        }

        if (query.onerror) {
            subquery.onerror = ensureMaxQueryLength(query.onerror);
        }

        return subquery;
    });

    return fallbackQuery;
}

function ensureMaxQueryLength(sql, length = MAX_SQL_LENGTH) {
    return sql.substring(0, length);
}
{'fraction_non_alphanumeric': 0.11394302848575712, 'fraction_numerical': 0.004069393874491326, 'mean_word_length': 2.3309557774607703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1034240', 'n_tokens_mistral': 1443, 'n_tokens_neox': 1412, 'n_words': 309}