<commit_before>let log = require('printit')({
    prefix: 'models/migrations',
    date: true
});

import Config from './kresusconfig';

// migration #1: remove weboob-log and weboob-installed from the db
async function m1() {
    let weboobLog = await Config.byName('weboob-log');
    if (weboobLog) {
        log.info('Destroying Config[weboob-log].');
        await weboobLog.destroy();
    }

    let weboobInstalled = await Config.byName('weboob-installed');
    if (weboobInstalled) {
        log.info('Destroying Config[weboob-installed].');
        await weboobInstalled.destroy();
    }
}

export async function run() {
    await m1();
}<commit_msg>Fixes #225: Ensure that categoryId and operationTypeID values within Operations are consistent;<commit_after>let log = require('printit')({
    prefix: 'models/migrations',
    date: true
});

import Config from './kresusconfig';
import Operation from './operation';
import Category from './category';
import Type from './operationtype';

// migration #1: remove weboob-log and weboob-installed from the db
async function m1() {
    let weboobLog = await Config.byName('weboob-log');
    if (weboobLog) {
        log.info('Destroying Config[weboob-log].');
        await weboobLog.destroy();
    }

    let weboobInstalled = await Config.byName('weboob-installed');
    if (weboobInstalled) {
        log.info('Destroying Config[weboob-installed].');
        await weboobInstalled.destroy();
    }
}

// migration #2: check that operations with types and categories are consistent
async function m2() {
    let ops = await Operation.all();
    let categories = await Category.all();
    let types = await Type.all();

    let typeSet = new Set;
    for (let t of types) {
        typeSet.add(t.id);
    }

    let categorySet = new Set;
    for (let c of categories) {
        categorySet.add(c.id);
    }

    let typeCount = 0;
    let categoryCount = 0;
    for (let op of ops) {
        let attr = null;

        if (typeof op.operationTypeID !== 'undefined' && !typeSet.has(op.operationTypeID)) {
            attr = attr || {};
            attr.operationTypeID = undefined;
            typeCount += 1;
        }

        if (typeof op.categoryId !== 'undefined' && !categorySet.has(op.categoryId)) {
            attr = attr || {};
            attr.categoryId = undefined;
            categoryCount += 1;
        }

        if (attr !== null) {
            await op.updateAttributes(attr);
        }
    }

    if (typeCount)
        log.info(`${typeCount} operations had an inconsistent operationTypeID value.`);
    if (categoryCount)
        log.info(`${categoryCount} operations had an inconsistent categoryId value.`);
}

export async function run() {
    await m1();
    await m2();
}
{'fraction_non_alphanumeric': 0.12899709302325582, 'fraction_numerical': 0.005813953488372093, 'mean_word_length': 2.6953020134228187, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '636289', 'n_tokens_mistral': 843, 'n_tokens_neox': 832, 'n_words': 228}