<commit_before>const InlineLexer = require('../../src/inlinelexer')

describe('Test InlineLexer', () => {
    const inlinelexer = new InlineLexer({})
    /* it('emoji', () => {
        expect(inlinelexer.output('test:up:fsdf'))
            .toEqual('test<img src="https://cdn.bootcss.com/emojify.js/1.0/images/basic/up.png" alt=":up:" title=":up:" class="emoji" align="absmiddle"/>fsdf')
    }) */
    it('html', () => {
        const newinlinelexer = new InlineLexer({}, {inlineHtml: true})
        expect(inlinelexer.output('test<i class="sfdsf">test'))
            .toEqual('test<i class="sfdsf">test')
    })
    it('escape', () => {
        expect(inlinelexer.output('\\[test[]'))
            .toEqual('[test[]')
    })
    it('autolink', () => {
        expect(/^<a href="(&#\w+;)+">(&#\w+;)+<\/a>$/.test(inlinelexer.output('<<EMAIL>>')))<commit_msg>del inline html and head sanitize<commit_after>const InlineLexer = require('../../src/inlinelexer')

describe('Test InlineLexer', () => {
    const inlinelexer = new InlineLexer({})
    /* it('emoji', () => {
        expect(inlinelexer.output('test:up:fsdf'))
            .toEqual('test<img src="https://cdn.bootcss.com/emojify.js/1.0/images/basic/up.png" alt=":up:" title=":up:" class="emoji" align="absmiddle"/>fsdf')
    }) */
    it('html', () => {
        expect(inlinelexer.output('test<i class="sfdsf">test'))
            .toEqual('test<i class="sfdsf">test')
    })
    it('escape', () => {
        expect(inlinelexer.output('\\[test[]'))
            .toEqual('[test[]')
    })
    it('autolink', () => {
        expect(/^<a href="(&#\w+;)+">(&#\w+;)+<\/a>$/.test(inlinelexer.output('<<EMAIL>>')))
{'fraction_non_alphanumeric': 0.25707405177603854, 'fraction_numerical': 0.002408187838651415, 'mean_word_length': 3.18639798488665, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2987990', 'n_tokens_mistral': 605, 'n_tokens_neox': 597, 'n_words': 66}