<commit_before>import * as assert from 'assert';
import * as vscode from 'vscode';
import { parseSDL, tokenizeSDL } from '../src/sdl/sdlparse';

let backslash = "\\";

// Defines a Mocha test suite to group tests of similar kind together
suite("sdl parser", () => {
	test("tokenizer", () => {
		let tokens = tokenizeSDL(`my_tag

person "Akiko" "Johnson" height=60

person name:first-name="Akiko" name:last-name="Johnson"
/*
 * Foo
 */
my_namespace:person "Akiko" "Johnson" dimensions:height=68 {
    son "Nouhiro" "Johnson"
    daughter "Sabrina" "Johnson" location="Italy" {
        hobbies "swimming" "surfing"
        smoker false
    }
}   

------------------------------------------------------------------
// (notice the separator style comment above...)

# a log entry
#     note - this tag has two values (date_time and string) and an 
#            attribute (error)
entry 2005/11/23 10:14:23.253-GMT "Something bad happened" error=true

# a long line
mylist "something" "another" true "shoe" 2002/12/13 "rock" ${backslash}
    "morestuff" "sink" "penny" 12:15:23.425

# a long string
text "this is a long rambling line of text with a continuation ${backslash}
   and it keeps going and going..."
   
# anonymous tag examples

files {
    "/folder1/file.txt"
    "/file2.txt"
}
    
# To retrieve the files as a list of strings
#
#     List files = tag.getChild("files").getChildrenValues("content");
# 
# We us the name "content" because the files tag has two children, each of 
# which are anonymous tags (values with no name.)  These tags are assigned
# the name "content"
    
matrix {
    1 2 3
    4 5 6
}`);
		// TODO: make test more automatic instead of relying on tokenizer not changing
		// TODO: do we even need a tokenizer test? The value test should be enough
		assert.deepStrictEqual(tokens[0], { type: "identifier", range: [0, 6], name: "my_tag" });
		assert.deepStrictEqual(tokens.length, 91, "Tokenizer has changed");
	});
	test("example sdl file", () => {
		let root = parseSDL(`# a tag having only a name
420
my_tag

# three tags acting as name value pairs
first_name "Akiko"
last_name "Johnson"<commit_msg>remove token length test from sdl parser<commit_after>import * as assert from 'assert';
import * as vscode from 'vscode';
import { parseSDL, tokenizeSDL } from '../src/sdl/sdlparse';

let backslash = "\\";

// Defines a Mocha test suite to group tests of similar kind together
suite("sdl parser", () => {
	test("tokenizer", () => {
		let tokens = tokenizeSDL(`my_tag

person "Akiko" "Johnson" height=60

person name:first-name="Akiko" name:last-name="Johnson"
/*
 * Foo
 */
my_namespace:person "Akiko" "Johnson" dimensions:height=68 {
    son "Nouhiro" "Johnson"
    daughter "Sabrina" "Johnson" location="Italy" {
        hobbies "swimming" "surfing"
        smoker false
    }
}   

------------------------------------------------------------------
// (notice the separator style comment above...)

# a log entry
#     note - this tag has two values (date_time and string) and an 
#            attribute (error)
entry 2005/11/23 10:14:23.253-GMT "Something bad happened" error=true

# a long line
mylist "something" "another" true "shoe" 2002/12/13 "rock" ${backslash}
    "morestuff" "sink" "penny" 12:15:23.425

# a long string
text "this is a long rambling line of text with a continuation ${backslash}
   and it keeps going and going..."
   
# anonymous tag examples

files {
    "/folder1/file.txt"
    "/file2.txt"
}
    
# To retrieve the files as a list of strings
#
#     List files = tag.getChild("files").getChildrenValues("content");
# 
# We us the name "content" because the files tag has two children, each of 
# which are anonymous tags (values with no name.)  These tags are assigned
# the name "content"
    
matrix {
    1 2 3
    4 5 6
}`);
		// TODO: make test more automatic instead of relying on tokenizer not changing
		assert.deepStrictEqual(tokens[0], { type: "identifier", range: [0, 6], name: "my_tag" });
	});
	test("example sdl file", () => {
		let root = parseSDL(`# a tag having only a name
420
my_tag

# three tags acting as name value pairs
first_name "Akiko"
last_name "Johnson"
{'fraction_non_alphanumeric': 0.15156626506024096, 'fraction_numerical': 0.025542168674698794, 'mean_word_length': 3.0576735092864125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4587649', 'n_tokens_mistral': 1415, 'n_tokens_neox': 1280, 'n_words': 472}