<commit_before>/// <reference path="../typings/node/node.d.ts"/>
/// <reference path="../typings/mocha/mocha.d.ts"/>
"use strict";

var expect = require('chai').expect;
var fs = require('fs');

var bbcms = require("../index");

describe('CCD parser test', function () {

    it('"CCD_20131121_ACMA10102495SLTXMWPGMandGoals" as input', function (done) {

        var istream = fs.createReadStream(__dirname + '/../../private-records/HCSC/CCD_20131121_ACMA10102495SLTXMWPGMandGoals.xml', 'utf-8');
        expect(istream).to.exist;

        istream
            .pipe(new bbcms.CcdParserStream())
            .on('data', function (data) {
                expect(data).to.exist;
                fs.writeFile(__dirname + '/artifacts/CCD_20131121_ACMA10102495SLTXMWPGMandGoals.json', JSON.stringify(data, null, '  '));

                var gold = fs.readFileSync(__dirname + '/artifacts/CCD_20131121_ACMA10102495SLTXMWPGMandGoals-gold.json', 'utf-8');
                expect(JSON.parse(gold)).to.eql(data);

            })
            .on('finish', function () {
                done();
            })
            .on('error', function (error) {
                done(error);
            });

    });

    it('"CCD_20131216_ACMA99457456BCFDIG.xml" as input', function (done) {

        var istream = fs.createReadStream(__dirname + '/../../private-records/HCSC/CCD_20131216_ACMA99457456BCFDIG.xml', 'utf-8');
        expect(istream).to.exist;

        istream
            .pipe(new bbcms.CcdParserStream())
            .on('data', function (data) {
                expect(data).to.exist;
                fs.writeFile(__dirname + '/artifacts/CCD_20131216_ACMA99457456BCFDIG.json', JSON.stringify(data, null, '  '));

                var gold = fs.readFileSync(__dirname + '/artifacts/CCD_20131216_ACMA99457456BCFDIG-gold.json', 'utf-8');
                expect(JSON.parse(gold)).to.eql(data);

            })
            .on('finish', function () {
                done();
            })
            .on('error', function (error) {
                done(error);
            });

    });

});<commit_msg>test: medicasoft files added for test<commit_after>/// <reference path="../typings/node/node.d.ts"/>
/// <reference path="../typings/mocha/mocha.d.ts"/>
"use strict";

var expect = require('chai').expect;
var fs = require('fs');

var bbcms = require("../index");

var testoOn = function (infile, outfile, goldfile, done) {
    var istream = fs.createReadStream(infile, 'utf-8');
    expect(istream).to.exist;

    istream
        .pipe(new bbcms.CcdParserStream())
        .on('data', function (data) {
            expect(data).to.exist;
            fs.writeFile(outfile, JSON.stringify(data, null, '  '));

            if (goldfile) {
                var gold = fs.readFileSync(goldfile, 'utf-8');
                expect(JSON.parse(gold)).to.eql(data);
            }

        })
        .on('finish', function () {
            done();
        })
        .on('error', function (error) {
            done(error);
        });
};

describe('CCD parser test', function () {

    it('"CCD_20131121_ACMA10102495SLTXMWPGMandGoals" as input', function (done) {

        testoOn(__dirname + '/../../private-records/HCSC/CCD_20131121_ACMA10102495SLTXMWPGMandGoals.xml',
            __dirname + '/artifacts/CCD_20131121_ACMA10102495SLTXMWPGMandGoals.json',
            __dirname + '/artifacts/CCD_20131121_ACMA10102495SLTXMWPGMandGoals-gold.json',
            done
        );

    });

    it('"CCD_20131216_ACMA99457456BCFDIG" as input', function (done) {

        testoOn(__dirname + '/../../private-records/HCSC/CCD_20131216_ACMA99457456BCFDIG.xml',
            __dirname + '/artifacts/CCD_20131216_ACMA99457456BCFDIG.json',
            __dirname + '/artifacts/CCD_20131216_ACMA99457456BCFDIG-gold.json',
            done
        );

    });
    it('"DEID02732967ZVZACA_CCD.xml" as input', function (done) {

        testoOn(__dirname + '/../../private-records/medicasoft/deidccd/DEID02732967ZVZACA_CCD.xml',
            __dirname + '/artifacts/DEID02732967ZVZACA_CCD.json',
            __dirname + '/artifacts/DEID02732967ZVZACA_CCD-gold.json',
            done
        );
    });

    it('"DEID06068444RGVUDR_CCD.xml" as input', function (done) {

        testoOn(__dirname + '/../../private-records/medicasoft/deidccd/DEID06068444RGVUDR_CCD.xml',
            __dirname + '/artifacts/DEID06068444RGVUDR_CCD.json',
            __dirname + '/artifacts/DEID06068444RGVUDR_CCD-gold.json',
            done
        );
    });

    it('"DEID10023094WRYZBV_CCD.xml" as input', function (done) {

        testoOn(__dirname + '/../../private-records/medicasoft/deidccd/DEID10023094WRYZBV_CCD.xml',
            __dirname + '/artifacts/DEID10023094WRYZBV_CCD.json',
            __dirname + '/artifacts/DEID10023094WRYZBV_CCD-gold.json',
            done
        );
    });

    it('"DEID10054139UQVPSZ_CCD.xml" as input', function (done) {

        testoOn(__dirname + '/../../private-records/medicasoft/deidccd/DEID10054139UQVPSZ_CCD.xml',
            __dirname + '/artifacts/DEID10054139UQVPSZ_CCD.json',
            __dirname + '/artifacts/DEID10054139UQVPSZ_CCD-gold.json',
            done
        );
    });

    it('"DEID10091332GJSVIQ_CCD.xml" as input', function (done) {

        testoOn(__dirname + '/../../private-records/medicasoft/deidccd/DEID10091332GJSVIQ_CCD.xml',
            __dirname + '/artifacts/DEID10091332GJSVIQ_CCD.json',
            __dirname + '/artifacts/DEID10091332GJSVIQ_CCD-gold.json',
            done
        );

    });

});
{'fraction_non_alphanumeric': 0.1511648907350551, 'fraction_numerical': 0.07621455661910782, 'mean_word_length': 2.304295942720764, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 15}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7285205', 'n_tokens_mistral': 2206, 'n_tokens_neox': 1879, 'n_words': 235}