<commit_before>//
// Note: This example test is leveraging the Mocha test framework.
// Please refer to their documentation on https://mochajs.org/ for help.
//

// The module 'assert' provides assertion methods from node
import * as assert from 'assert';

// You can import and use all API from the 'vscode' module
// as well as import your extension to test it
import * as vscode from 'vscode';
import * as myExtension from '../src/extension';

// Defines a Mocha test suite to group tests of similar kind together
suite("Extension Tests", () => {

    // Defines a Mocha unit test
    test("Something 1", () => {
        assert.equal(-1, [1, 2, 3].indexOf(5));
        assert.equal(-1, [1, 2, 3].indexOf(0));
    });
});<commit_msg>Add tests for basic lexing of types, numbers and identifiers<commit_after>import * as assert from 'assert';

import { ShaderLexer, TokenType } from '../src/shaderlexer';
import { ShaderStream } from '../src/shaderstream';

suite("Parsing Tests", () => {
    {
        let typesContent = `\
int float vec2 /* a random comment */ ivec2 vec3 ivec3 vec4 ivec4 // an eof comment`;
        let stream = new ShaderStream(typesContent);
        let lexer = new ShaderLexer(stream);
        
        test("Parse Type", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Type, value: 'int' });
            assert.deepEqual(lexer.next(), { type: TokenType.Type, value: 'float' });
            assert.deepEqual(lexer.next(), { type: TokenType.Type, value: 'vec2' });
            assert.deepEqual(lexer.next(), { type: TokenType.Type, value: 'ivec2' });
            assert.deepEqual(lexer.next(), { type: TokenType.Type, value: 'vec3' });
            assert.deepEqual(lexer.next(), { type: TokenType.Type, value: 'ivec3' });
            assert.deepEqual(lexer.next(), { type: TokenType.Type, value: 'vec4' });
            assert.deepEqual(lexer.next(), { type: TokenType.Type, value: 'ivec4' });
            assert.equal(lexer.next(), undefined);
        });
    }
    
    {
        let stringsContents = `\
"a string" "a \\"string\\"" "a 'string'" /* a random comment */ /* back to back comments */ 'a string' 'a "string"' 'a \\'string\\''
/* 
    a
    multiline
    comment
*/`;
        let stream = new ShaderStream(stringsContents);
        let lexer = new ShaderLexer(stream);
        
        test("Parse String", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.String, value: 'a string' });
            assert.deepEqual(lexer.next(), { type: TokenType.String, value: 'a "string"' });
            assert.deepEqual(lexer.next(), { type: TokenType.String, value: 'a \'string\'' });
            assert.deepEqual(lexer.next(), { type: TokenType.String, value: 'a string' });
            assert.deepEqual(lexer.next(), { type: TokenType.String, value: 'a "string"' });
            assert.deepEqual(lexer.next(), { type: TokenType.String, value: 'a \'string\'' });
            assert.equal(lexer.next(), undefined);
        });
    }
    
    {
        let numbersContent = `\
1 999999999999999999999999999999999 1e6 1e-6 999999999999999.999999999999999999 1. .1 1.e6 1.e-6  .1e6 .1e-6
+1 +999999999999999999999999999999999 +1e6 +1e-6 +999999999999999.999999999999999999 +1. +.1 +1.e6 +1.e-6 +.1e6 +.1e-6
-1 -999999999999999999999999999999999 -1e6 -1e-6 -999999999999999.999999999999999999 -1. -.1 -1.e6 -1.e-6 -.1e6 -.1e-6 `;
        let stream = new ShaderStream(numbersContent);
        let lexer = new ShaderLexer(stream);
        
        test("Parse Integers", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: 1 });
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: 999999999999999999999999999999999 });
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: 1e6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: 1e-6 });
        });

        test("Parse Floats", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: 999999999999999.999999999999999999 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: 1. });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: .1 });
        });

        test("Parse Floats with Exponents", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: 1.e6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: 1.e-6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: .1e6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: .1e-6 });
        });

        test("Parse Integers with explicit Plus", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: 1 });
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: 999999999999999999999999999999999 });
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: 1e6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: 1e-6 });
        });

        test("Parse Floats with explicit Plus", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: 999999999999999.999999999999999999 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: 1. });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: .1 });
        });

        test("Parse Floats with explicit Plus with Exponents", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: 1.e6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: 1.e-6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: .1e6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: .1e-6 });
        });

        test("Parse negative Integers", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: -1 });
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: -999999999999999999999999999999999 });
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: -1e6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: -1e-6 });
        });

        test("Parse negative Floats", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: -999999999999999.999999999999999999 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: -1. });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: -.1 });
        });

        test("Parse negative Floats with Exponents", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: -1.e6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: -1.e-6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: -.1e6 });
            assert.deepEqual(lexer.next(), { type: TokenType.Float, value: -.1e-6 });
            assert.equal(lexer.next(), undefined);
        });
    }

    {
        let identifiersContent = `\
aFineVariable aFineVariable_1 a_fine_1_variable __a_fine_var__ 1_a_fine_var_`;
        let stream = new ShaderStream(identifiersContent);
        let lexer = new ShaderLexer(stream);
        
        test("Parse Identifiers", () => {
            assert.deepEqual(lexer.next(), { type: TokenType.Identifier, value: 'aFineVariable' });
            assert.deepEqual(lexer.next(), { type: TokenType.Identifier, value: 'aFineVariable_1' });
            assert.deepEqual(lexer.next(), { type: TokenType.Identifier, value: 'a_fine_1_variable' });
            assert.deepEqual(lexer.next(), { type: TokenType.Identifier, value: '__a_fine_var__' });
            assert.deepEqual(lexer.next(), { type: TokenType.Integer, value: '1' });
            assert.deepEqual(lexer.next(), { type: TokenType.Identifier, value: '_a_fine_var_' });
            assert.equal(lexer.next(), undefined);
        });
    }
});
{'fraction_non_alphanumeric': 0.16244096445438727, 'fraction_numerical': 0.06400695998011434, 'mean_word_length': 2.672752167959836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 18, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3182621', 'n_tokens_mistral': 2849, 'n_tokens_neox': 2441, 'n_words': 552}