<commit_before>// cSpell:ignore jpegs outing dirs lcode outring outrings

import { expect } from 'chai';
import { lineToWords, compileWordList, compileTrie } from './wordListCompiler';
import { normalizeWords, normalizeWordsToTrie } from './wordListCompiler';
import * as fsp from 'fs-extra';
import * as Trie from 'cspell-trie-lib';
import * as path from 'path';
import { genSequence } from 'gensequence';

describe('Validate the wordListCompiler', function() {
    it('tests splitting lines', () => {
        const line = 'AppendIterator::getArrayIterator';
        expect(lineToWords(line).filter(distinct()).toArray()).to.deep.equal([
            'append',
            'iterator',
            'get',
            'array',
        ]);
        expect(lineToWords('<NAME>').toArray()).to.deep.equal([
            '<NAME>', 'austin', 'martin'
        ]);
        expect(lineToWords('JPEGsBLOBs').filter(distinct()).toArray()).to.deep.equal(['jpegs', 'blobs']);
        expect(lineToWords('CURLs CURLing').filter(distinct()).toArray()).to.deep.equal(['curls curling', 'curls', 'curling']);
        expect(lineToWords('DNSTable Lookup').filter(distinct()).toArray()).to.deep.equal(['dns', 'table', 'lookup']);
        expect(lineToWords('OUTRing').filter(distinct()).toArray()).to.deep.equal(['outring']);
        expect(lineToWords('OUTRings').filter(distinct()).toArray()).to.deep.equal(['outrings']);
        expect(lineToWords('DIRs').filter(distinct()).toArray()).to.deep.equal(['dirs']);
        expect(lineToWords('AVGAspect').filter(distinct()).toArray()).to.deep.equal(['avg', 'aspect']);
        expect(lineToWords('New York').filter(distinct()).toArray()).to.deep.equal(['new york', 'new', 'york']);
        expect(lineToWords('Namespace DNSLookup').filter(distinct()).toArray()).to.deep.equal(['namespace', 'dns', 'lookup']);
        expect(lineToWords('well-educated').filter(distinct()).toArray()).to.deep.equal(['well', 'educated']);
        // Sadly we cannot do this one correctly
        expect(lineToWords('CURLcode').filter(distinct()).toArray()).to.deep.equal(['cur', 'lcode']);
        expect(lineToWords('kDNSServiceErr_BadSig').filter(distinct()).toArray()).to.deep.equal([
            'k',
            'dns',
            'service',
            'err',
            'bad',
            'sig',
        ]);
        expect(lineToWords('apd_get_active_symbols').filter(distinct()).toArray()).to.deep.equal([
            'apd',
            'get',
            'active',
            'symbols',
        ]);
    });

    it('test reading and normalizing a file', () => {
        const sourceName = path.join(__dirname, '..', '..', 'Samples', 'cities.txt');
        const destName = path.join(__dirname, '..', '..', 'temp', 'cities.txt');
        return compileWordList(sourceName, destName, { splitWords: true, sort: true })
        .then(() => fsp.readFile(destName, 'utf8'))
        .then(output => {
            expect(output).to.be.equal(citiesResultSorted);
        });
    });

    it('test compiling to a file without split', () => {
        const sourceName = path.join(__dirname, '..', '..', 'Samples', 'cities.txt');
        const destName = path.join(__dirname, '..', '..', 'temp', 'cities2.txt');
        return compileWordList(sourceName, destName, { splitWords: false, sort: true })
        .then(() => fsp.readFile(destName, 'utf8'))
        .then(output => {
            expect(output).to.be.equal(citiesSorted.toLowerCase());
        });
    });

    it('tests normalized to a trie', () => {
        const words = citiesResult.split('\n');
        const nWords = normalizeWords(genSequence(words)).toArray();
        const tWords = [...genSequence([normalizeWordsToTrie(genSequence(words))])
            .concatMap(node => Trie.iteratorTrieWords(node))];
        expect(tWords.sort()).to.be.deep.equal([...(new Set(nWords.sort()))]);
    });

    it('test reading and normalizing to a trie file', () => {
        const sourceName = path.join(__dirname, '..', '..', 'Samples', 'cities.txt');
        const destName = path.join(__dirname, '..', '..', 'temp', 'cities.trie');
        return compileTrie(sourceName, destName)
        .then(() => fsp.readFile(destName, 'UTF-8'))
        .then(output => output.split('\n'))
        .then(srcWords => {
            const node = Trie.importTrie(srcWords);
            const expected = citiesResult.split('\n').filter(a => !!a).sort();
            const words = [...Trie.iteratorTrieWords(node)].sort();
            expect(words).to.be.deep.equal(expected);
        });
    });
});

function distinct(): (word: string) => boolean {
    const known = new Set<String>();
    return a => known.has(a) ? false : (known.add(a), true);
}

// const cities = `\
// New York
// New Amsterdam
// Los Angeles
// San Francisco
// New Delhi
// Mexico City
// London
// Paris
// `;

const citiesSorted = `\
London
Los Angeles
Mexico City
New Amsterdam
New Delhi<commit_msg>Add test for compressed files.<commit_after>// cSpell:ignore jpegs outing dirs lcode outring outrings

import { expect } from 'chai';
import { lineToWords, compileWordList, compileTrie } from './wordListCompiler';
import { normalizeWords, normalizeWordsToTrie } from './wordListCompiler';
import * as fsp from 'fs-extra';
import * as Trie from 'cspell-trie-lib';
import * as path from 'path';
import { genSequence } from 'gensequence';
import { readFile } from 'cspell-lib';

const UTF8: BufferEncoding = 'utf8';

describe('Validate the wordListCompiler', function() {
    it('tests splitting lines', () => {
        const line = 'AppendIterator::getArrayIterator';
        expect(lineToWords(line).filter(distinct()).toArray()).to.deep.equal([
            'append',
            'iterator',
            'get',
            'array',
        ]);
        expect(lineToWords('<NAME>').toArray()).to.deep.equal([
            '<NAME>', 'austin', 'martin'
        ]);
        expect(lineToWords('JPEGsBLOBs').filter(distinct()).toArray()).to.deep.equal(['jpegs', 'blobs']);
        expect(lineToWords('CURLs CURLing').filter(distinct()).toArray()).to.deep.equal(['curls curling', 'curls', 'curling']);
        expect(lineToWords('DNSTable Lookup').filter(distinct()).toArray()).to.deep.equal(['dns', 'table', 'lookup']);
        expect(lineToWords('OUTRing').filter(distinct()).toArray()).to.deep.equal(['outring']);
        expect(lineToWords('OUTRings').filter(distinct()).toArray()).to.deep.equal(['outrings']);
        expect(lineToWords('DIRs').filter(distinct()).toArray()).to.deep.equal(['dirs']);
        expect(lineToWords('AVGAspect').filter(distinct()).toArray()).to.deep.equal(['avg', 'aspect']);
        expect(lineToWords('New York').filter(distinct()).toArray()).to.deep.equal(['new york', 'new', 'york']);
        expect(lineToWords('Namespace DNSLookup').filter(distinct()).toArray()).to.deep.equal(['namespace', 'dns', 'lookup']);
        expect(lineToWords('well-educated').filter(distinct()).toArray()).to.deep.equal(['well', 'educated']);
        // Sadly we cannot do this one correctly
        expect(lineToWords('CURLcode').filter(distinct()).toArray()).to.deep.equal(['cur', 'lcode']);
        expect(lineToWords('kDNSServiceErr_BadSig').filter(distinct()).toArray()).to.deep.equal([
            'k',
            'dns',
            'service',
            'err',
            'bad',
            'sig',
        ]);
        expect(lineToWords('apd_get_active_symbols').filter(distinct()).toArray()).to.deep.equal([
            'apd',
            'get',
            'active',
            'symbols',
        ]);
    });

    it('test reading and normalizing a file', () => {
        const sourceName = path.join(__dirname, '..', '..', 'Samples', 'cities.txt');
        const destName = path.join(__dirname, '..', '..', 'temp', 'cities.txt');
        return compileWordList(sourceName, destName, { splitWords: true, sort: true })
        .then(() => fsp.readFile(destName, 'utf8'))
        .then(output => {
            expect(output).to.be.equal(citiesResultSorted);
        });
    });

    it('test compiling to a file without split', () => {
        const sourceName = path.join(__dirname, '..', '..', 'Samples', 'cities.txt');
        const destName = path.join(__dirname, '..', '..', 'temp', 'cities2.txt');
        return compileWordList(sourceName, destName, { splitWords: false, sort: true })
        .then(() => fsp.readFile(destName, 'utf8'))
        .then(output => {
            expect(output).to.be.equal(citiesSorted.toLowerCase());
        });
    });

    it('tests normalized to a trie', () => {
        const words = citiesResult.split('\n');
        const nWords = normalizeWords(genSequence(words)).toArray();
        const tWords = [...genSequence([normalizeWordsToTrie(genSequence(words))])
            .concatMap(node => Trie.iteratorTrieWords(node))];
        expect(tWords.sort()).to.be.deep.equal([...(new Set(nWords.sort()))]);
    });

    it('test reading and normalizing to a trie file', () => {
        const sourceName = path.join(__dirname, '..', '..', 'Samples', 'cities.txt');
        const destName = path.join(__dirname, '..', '..', 'temp', 'cities.trie');
        return compileTrie(sourceName, destName)
        .then(() => fsp.readFile(destName, UTF8))
        .then(output => output.split('\n'))
        .then(srcWords => {
            const node = Trie.importTrie(srcWords);
            const expected = citiesResult.split('\n').filter(a => !!a).sort();
            const words = [...Trie.iteratorTrieWords(node)].sort();
            expect(words).to.be.deep.equal(expected);
        });
    });

    it('test reading and normalizing to a trie gz file', async () => {
        const sourceName = path.join(__dirname, '..', '..', 'Samples', 'cities.txt');
        const destName = path.join(__dirname, '..', '..', 'temp', 'cities.trie.gz');
        await compileTrie(sourceName, destName);
        const resultFile = await readFile(destName, UTF8);
        const srcWords = resultFile.split('\n');
        const node = Trie.importTrie(srcWords);
        const expected = citiesResult.split('\n').filter(a => !!a).sort();
        const words = [...Trie.iteratorTrieWords(node)].sort();
        expect(words).to.be.deep.equal(expected);
    });
});

function distinct(): (word: string) => boolean {
    const known = new Set<String>();
    return a => known.has(a) ? false : (known.add(a), true);
}

// const cities = `\
// New York
// New Amsterdam
// Los Angeles
// San Francisco
// New Delhi
// Mexico City
// London
// Paris
// `;

const citiesSorted = `\
London
Los Angeles
Mexico City
New Amsterdam
New Delhi
{'fraction_non_alphanumeric': 0.1964117091595845, 'fraction_numerical': 0.0010387157695939567, 'mean_word_length': 2.981578947368421, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7485389', 'n_tokens_mistral': 3282, 'n_tokens_neox': 3192, 'n_words': 583}