<commit_before>const _ = require('lodash');
const db = require('../../../data/db');

const CHUNK_SIZE = 100;

async function insert(table, data) {
    const result = {
        successful: 0,
        unsuccessful: 0,
        errors: []
    };

    for (const chunk of _.chunk(data, CHUNK_SIZE)) {
        try {
            await db.knex(table).insert(chunk);
            result.successful += chunk.length;
        } catch (error) {
            result.unsuccessful += chunk.length;
            result.errors.push(error);
        }
    }

    return result;
}

async function del(table, ids) {
    const result = {
        successful: 0,
        unsuccessful: 0,
        errors: []
    };

    for (const chunk of _.chunk(ids, CHUNK_SIZE)) {
        try {
            await db.knex(table).whereIn('id', chunk).del();
            result.successful += chunk.length;
        } catch (error) {
            result.unsuccessful += chunk.length;
            result.errors.push(error);
        }
    }

    return result;
}

module.exports.insert = insert;
module.exports.del = del;
<commit_msg>Improved error handling for batch inserted records<commit_after>const _ = require('lodash');
const db = require('../../../data/db');

const CHUNK_SIZE = 100;

async function insertChunkSequential(table, chunk, result) {
    for (const record of chunk) {
        try {
            await db.knex(table).insert(record);
            result.successful += 1;
        } catch (err) {
            result.errors.push(err);
            result.unsuccessfulIds.push(record.id);
            result.unsuccessful += 1;
        }
    }
}

async function insertChunk(table, chunk, result) {
    try {
        await db.knex(table).insert(chunk);
        result.successful += chunk.length;
    } catch (err) {
        await insertChunkSequential(table, chunk, result);
    }
}

async function insert(table, data) {
    const result = {
        successful: 0,
        unsuccessful: 0,
        unsuccessfulIds: [],
        errors: []
    };

    for (const chunk of _.chunk(data, CHUNK_SIZE)) {
        await insertChunk(table, chunk, result);
    }

    return result;
}

async function del(table, ids) {
    const result = {
        successful: 0,
        unsuccessful: 0,
        errors: []
    };

    for (const chunk of _.chunk(ids, CHUNK_SIZE)) {
        try {
            await db.knex(table).whereIn('id', chunk).del();
            result.successful += chunk.length;
        } catch (error) {
            result.unsuccessful += chunk.length;
            result.errors.push(error);
        }
    }

    return result;
}

module.exports.insert = insert;
module.exports.del = del;

{'fraction_non_alphanumeric': 0.13056603773584904, 'fraction_numerical': 0.0060377358490566035, 'mean_word_length': 1.992099322799097, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7311401', 'n_tokens_mistral': 794, 'n_tokens_neox': 798, 'n_words': 180}