<commit_before>import * as fs from 'fs';

const PouchDB = require('pouchdb');
const replicationStream = require('pouchdb-replication-stream');
const MemoryStream = require('memorystream');


/**
 * @author <NAME>
 **/
export module Backup {

    export const FILE_NOT_EXIST = 'filenotexist';

    export async function dump(filePath: string, project: string) {

        PouchDB.plugin(replicationStream.plugin);
        PouchDB.adapter('writableStream', replicationStream.adapters.writableStream);

        let dumpedString = '';
        const stream = new MemoryStream();
        stream.on('data', (chunk: any) => {

            dumpedString +=
                chunk.toString()
                    .replace(/"data"[\s\S]+?,/g,'\"data\":\"\",')
        }); // TODO note that this regex is a too general. we want to get rid of this asap anyway, as soon as the thumbnail thing is fixed in pouchdb-replication stream.

        const db = new PouchDB(project);

        await db.dump(stream, {attachments:false});
        fs.writeFileSync(filePath, dumpedString);
    }


    export async function readDump(filePath: string, project: string) {

        if (!fs.existsSync(filePath)) throw FILE_NOT_EXIST;
        if (!fs.lstatSync(filePath).isFile()) throw FILE_NOT_EXIST;

        const db2 = new PouchDB(project); // TODO destroy before load and unit test it
        PouchDB.plugin(require('pouchdb-load'));

        await db2.load(filePath);
    }
}<commit_msg>Make sure db gets overwritten on read dump<commit_after>import * as fs from 'fs';

const PouchDB = require('pouchdb');
const replicationStream = require('pouchdb-replication-stream');
const MemoryStream = require('memorystream');


/**
 * @author <NAME>
 **/
export module Backup {

    export const FILE_NOT_EXIST = 'filenotexist';

    export async function dump(filePath: string, project: string) {

        PouchDB.plugin(replicationStream.plugin);
        PouchDB.adapter('writableStream', replicationStream.adapters.writableStream);

        let dumpedString = '';
        const stream = new MemoryStream();
        stream.on('data', (chunk: any) => {

            dumpedString +=
                chunk.toString()
                    .replace(/"data"[\s\S]+?,/g,'\"data\":\"\",')
        }); // TODO note that this regex is a too general. we want to get rid of this asap anyway, as soon as the thumbnail thing is fixed in pouchdb-replication stream.

        const db = new PouchDB(project);

        await db.dump(stream, {attachments:false});
        fs.writeFileSync(filePath, dumpedString);
    }


    export async function readDump(filePath: string, project: string) {

        if (!fs.existsSync(filePath)) throw FILE_NOT_EXIST;
        if (!fs.lstatSync(filePath).isFile()) throw FILE_NOT_EXIST;

        const db = new PouchDB(project);
        await db.destroy(); // to prevent pouchdb-load's incremental loading and force a true overwrite of the old db

        const db2 = new PouchDB(project);
        PouchDB.plugin(require('pouchdb-load'));

        await db2.load(filePath);
    }
}
{'fraction_non_alphanumeric': 0.12336814621409922, 'fraction_numerical': 0.0013054830287206266, 'mean_word_length': 2.9395886889460154, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '221365', 'n_tokens_mistral': 911, 'n_tokens_neox': 867, 'n_words': 245}