<commit_before>import X from '../../../lib/standard/extractor/extractor-bundle';
import {F, C} from '../../../lib/parsec/index';
import stream from '../../../lib/stream/index';


export default {
    setUp: function (done) {
        done();
    },

    'test spaces': function (test) {

        const line = stream.ofString('    this starts with 4 spaces');

        const x = new X();
        const combinator = x.spaces().thenLeft(F.any.rep());
        const value = combinator.parse(line).value;
        test.equals(value.length, 4);
        test.done();
    },
    'test spaces and line feed': function (test) {

        const line = stream.ofString('    this \n contains line feed');

        const x = new X();
        const combinator = x.spaces()
            .then(x.word())
            .thenRight(x.spaces())
            .thenLeft(F.any);
        const value = combinator.parse(line).value;
        test.equals(value.length, 3);
        test.done();
    },
    'test numbers': function (test) {

        const line = stream.ofString('98');

        const x = new X();
        const combinator = x.number();
        const value = combinator.parse(line).value;
        test.ok(value === 98);
        test.done();
    },
    'test digits': function (test) {

        const line = stream.ofString('98');

        const x = new X();
        const combinator = x.digits();
        const value = combinator.parse(line).value;
        test.ok(value === '98');
        test.done();
    },
    'test single word': function (test) {

        let line = stream.ofString('Parsec-');

        const x = new X();
        const combinator = x.word().thenLeft(C.char('-'));
        const value = combinator.parse(line).value[0];
        test.ok(value === 'Parsec');
        test.done();
    },

    'test single word': function (test) {

        let line = stream.ofString('10FF-hexadecimal');
        const hexadecimal = C.charIn('0123456789ABCDEF')
            .rep().map(values=>values.join(''))
            .map(x => parseInt(x, 16));

        const x = new X({letters: hexadecimal});
        const combinator = x.word()
            .thenLeft(C.char('-'))
            .thenLeft(C.string('hexadecimal'));
        const value = combinator.parse(line).value[0];
        test.equals(value, 4351);
        test.done();
    },
    'single word with bad letters should fail': function (test) {

        let line = stream.ofString('classicWord-notHexadecimal');
        const hexadecimal = C.charIn('0123456789ABCDEF')
            .rep().map(values=>values.join(''))
            .map(x => parseInt(x, 16));

        const x = new X({letters: hexadecimal});
        const combinator = x.word()
            .thenLeft(C.char('-'))
            .thenLeft(C.string('notHexadecimal'));
        const accepted = combinator.parse(line).isAccepted();
        test.ok(!accepted);
        test.done();
    },

    'test words': function (test) {
        let line = stream.ofString('The James Bond series, by writer <NAME>');

        const x = new X();
        const combinator = x.words();
        const value = combinator.parse(line).value;
        test.ok(value[1] === ' ');
        test.ok(_includes(value,'Bond'));
        test.done();
    },

    'test stringIn': function (test) {
        let line = stream.ofString('<NAME>');

        const x = new X();
        const combinator = x.stringIn(['The', 'James', 'Bond', 'series']);
        const value = combinator.parse(line).value;
        test.ok(typeof value === 'string');
        test.ok(value === 'James');
        test.done();
    },

    'test wordsIn': function (test) {
        let line = stream.ofString('James Bond by <NAME>');

        const x = new X();
        const combinator = x.wordsIn(['James', 'Bond', 'by', 'Ian', 'Fleming'], true);
        const value = combinator.parse(line).value;
        test.ok(value.length === 9);
        test.ok(_includes(value,'James'));<commit_msg>#41: Extractor Bundle: side cases tests<commit_after>import X from '../../../lib/standard/extractor/extractor-bundle';
import {F, C} from '../../../lib/parsec/index';
import stream from '../../../lib/stream/index';


export default {
    setUp: function (done) {
        done();
    },

    'test spaces': function (test) {

        const line = stream.ofString('    this starts with 4 spaces');

        const x = new X();
        const combinator = x.spaces().thenLeft(F.any.rep());
        const value = combinator.parse(line).value;
        test.equals(value.length, 4);
        test.done();
    },
    'test spaces and line feed': function (test) {

        const line = stream.ofString('    this \n contains line feed');

        const x = new X();
        const combinator = x.spaces()
            .then(x.word())
            .thenRight(x.spaces())
            .thenLeft(F.any);
        const value = combinator.parse(line).value;
        test.equals(value.length, 3);
        test.done();
    },
    'test numbers': function (test) {

        const line = stream.ofString('98');

        const x = new X();
        const combinator = x.number();
        const value = combinator.parse(line).value;
        test.ok(value === 98);
        test.done();
    },
    'test digits': function (test) {

        const line = stream.ofString('98');

        const x = new X();
        const combinator = x.digits();
        const value = combinator.parse(line).value;
        test.ok(value === '98');
        test.done();
    },
    'test single word': function (test) {

        let line = stream.ofString('Parsec-');

        const x = new X();
        const combinator = x.word().thenLeft(C.char('-'));
        const value = combinator.parse(line).value[0];
        test.ok(value === 'Parsec');
        test.done();
    },

    'test single word hexadecimal': function (test) {

        let line = stream.ofString('10FF-hexadecimal');
        const hexadecimal = C.charIn('0123456789ABCDEF')
            .rep().map(values=>values.join(''))
            .map(x => parseInt(x, 16));

        const x = new X({letters: hexadecimal});
        const combinator = x.word()
            .thenLeft(C.char('-'))
            .thenLeft(C.string('hexadecimal'));
        const value = combinator.parse(line).value[0];
        test.equals(value, 4351);
        test.done();
    },
    'single word with bad letters should fail': function (test) {

        let line = stream.ofString('classicWord-notHexadecimal');
        const hexadecimal = C.charIn('0123456789ABCDEF')
            .rep().map(values=>values.join(''))
            .map(x => parseInt(x, 16));

        const x = new X({letters: hexadecimal});
        const combinator = x.word()
            .thenLeft(C.char('-'))
            .thenLeft(C.string('notHexadecimal'));
        const accepted = combinator.parse(line).isAccepted();
        test.ok(!accepted);
        test.done();
    },

    'test words': function (test) {
        let line = stream.ofString('The James Bond series, by writer <NAME>');

        const x = new X();
        const combinator = x.words();
        const value = combinator.parse(line).value;
        test.ok(value[1] === ' ');
        test.ok(_includes(value,'Bond'));
        test.done();
    },

    'test stringIn': function (test) {
        let line = stream.ofString('<NAME>');

        const x = new X();
        const combinator = x.stringIn(['The', 'James', 'Bond', 'series']);
        const value = combinator.parse(line).value;
        test.ok(typeof value === 'string');
        test.ok(value === 'James');
        test.done();
    },

    'test stringIn one string sidecase': function (test) {
        let line = stream.ofString('James');

        const x = new X();
        const combinator = x.stringIn(['James']);
        const value = combinator.parse(line).value;
        test.ok(typeof value === 'string');
        test.ok(value === 'James');
        test.done();
    },

    'test stringIn empty sidecase': function (test) {
        let line = stream.ofString('James');

        const x = new X();
        const combinator = x.stringIn([]).then(F.eos);
        const parsing = combinator.parse(line);
        test.ok(!parsing.isAccepted());
        test.done();
    },

    'test wordsIn': function (test) {
        let line = stream.ofString('<NAME> by <NAME>');

        const x = new X();
        const combinator = x.wordsIn(['James', 'Bond', 'by', 'Ian', 'Fleming'], true);
        const value = combinator.parse(line).value;
        test.ok(value.length === 9);
        test.ok(_includes(value,'James'));
{'fraction_non_alphanumeric': 0.16986879100281163, 'fraction_numerical': 0.01077788191190253, 'mean_word_length': 2.1386029411764707, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2554985', 'n_tokens_mistral': 2664, 'n_tokens_neox': 2484, 'n_words': 613}