<commit_before>{
	"$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json",
	"name": "nullc",
	"patterns": [
		{
			"include": "#comments"
		},
		{
			"include": "#keywords"
		},
		{
			"include": "#characters"
		},
		{
			"include": "#strings"
		},
		{
			"include": "#derived_types"
		},
		{
			"include": "#base_types"
		},
		{
			"include": "#identifiers"
		},
		{
			"include": "#numbers"
		}
	],
	"repository": {
		"comments": {
			"patterns": [{
				"name": "comment.block",
				"begin": "\\/\\*",
				"end": "\\*\\/"
			},
			{
				"name": "comment.line",
				"match": "\\/\\/.*$"
			}]
		},
		"keywords": {
			"patterns": [{
				"name": "keyword.control.nullc",
				"match": "\\b(if|else|while|for|switch|case|default|do|return|yield|break|continue)\\b"
			},
			{
				"name": "keyword.operator.new.nullc",
				"match": "\\b(new)\\b"
			},
			{
				"name": "keyword.operator.nullc",
				"match": "\\b(in)\\b"
			},
			{
				"name": "keyword.other",
				"match": "\\b(class|enum|namespace|coroutine|operator|this|import)\\b"
			},
			{
				"name": "constant.language",
				"match": "\\b(true|false|nullptr)\\b"
			}]
		},
		"characters": {
			"name": "string.quoted.single.nullc",
			"begin": "\\'",
			"end": "\\'",
			"patterns": [
				{
					"name": "constant.character.escape.nullc",
					"match": "\\\\."
				}
			]
		},
		"strings": {
			"name": "string.quoted.double.nullc",
			"begin": "\"",
			"end": "\"",
			"patterns": [
				{
					"name": "constant.character.escape.nullc",
					"match": "\\\\."
				}
			]
		},
		"derived_types": {
			"patterns": [{
				"name": "meta.type.nullc",
				"match": "(?<name>(?<id>\\b[a-zA-Z_][a-zA-Z_0-9]*\\b)?(?:\\s+ref\\s*?\\(\\s*?(?:(?:\\g<name>|\\g<id>)\\s*?(?:,\\s*?(?:\\g<name>|\\g<id>)\\s*)*)?\\s*\\)|\\s+ref|\\s*?\\[[^\\]]*\\])+)\\s+\\g<id>",
				"captures": {
					"1": { "name": "entity.name.type.nullc" }
				 }
			}]
		},
		"base_types": {
			"patterns": [{
				"name": "entity.name.type.nullc",
				"match": "\\b(void|bool|char|short|int|long|float|double|typeid|__function|__nullptr|generic|auto)\\b"
			}]
		},
		"identifiers": {
			"patterns": [{
				"name": "variable.nullc",
				"match": "\\b[a-zA-Z_][a-zA-Z_0-9]*\\b"
			}]
		},
		"numbers": {
			"patterns": [{
				"name": "constant.numeric.nullc",
				"match": "0x[0-9a-fA-F]+|[0-9]+\\.[0-9]*f?|[0-9]+l?"
			}]
		}
	},
	"scopeName": "source.nullc"
}<commit_msg>Extension update: add file types to grammar info<commit_after>{
	"$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json",
	"name": "nullc",
	"fileTypes": [
		".nc",
		".nullc"
	],
	"patterns": [
		{
			"include": "#comments"
		},
		{
			"include": "#keywords"
		},
		{
			"include": "#characters"
		},
		{
			"include": "#strings"
		},
		{
			"include": "#derived_types"
		},
		{
			"include": "#base_types"
		},
		{
			"include": "#identifiers"
		},
		{
			"include": "#numbers"
		}
	],
	"repository": {
		"comments": {
			"patterns": [{
				"name": "comment.block",
				"begin": "\\/\\*",
				"end": "\\*\\/"
			},
			{
				"name": "comment.line",
				"match": "\\/\\/.*$"
			}]
		},
		"keywords": {
			"patterns": [{
				"name": "keyword.control.nullc",
				"match": "\\b(if|else|while|for|switch|case|default|do|return|yield|break|continue)\\b"
			},
			{
				"name": "keyword.operator.new.nullc",
				"match": "\\b(new)\\b"
			},
			{
				"name": "keyword.operator.nullc",
				"match": "\\b(in)\\b"
			},
			{
				"name": "keyword.other",
				"match": "\\b(class|enum|namespace|coroutine|operator|this|import)\\b"
			},
			{
				"name": "constant.language",
				"match": "\\b(true|false|nullptr)\\b"
			}]
		},
		"characters": {
			"name": "string.quoted.single.nullc",
			"begin": "\\'",
			"end": "\\'",
			"patterns": [
				{
					"name": "constant.character.escape.nullc",
					"match": "\\\\."
				}
			]
		},
		"strings": {
			"name": "string.quoted.double.nullc",
			"begin": "\"",
			"end": "\"",
			"patterns": [
				{
					"name": "constant.character.escape.nullc",
					"match": "\\\\."
				}
			]
		},
		"derived_types": {
			"patterns": [{
				"name": "meta.type.nullc",
				"match": "(?<name>(?<id>\\b[a-zA-Z_][a-zA-Z_0-9]*\\b)?(?:\\s+ref\\s*?\\(\\s*?(?:(?:\\g<name>|\\g<id>)\\s*?(?:,\\s*?(?:\\g<name>|\\g<id>)\\s*)*)?\\s*\\)|\\s+ref|\\s*?\\[[^\\]]*\\])+)\\s+\\g<id>",
				"captures": {
					"1": { "name": "entity.name.type.nullc" }
				 }
			}]
		},
		"base_types": {
			"patterns": [{
				"name": "entity.name.type.nullc",
				"match": "\\b(void|bool|char|short|int|long|float|double|typeid|__function|__nullptr|generic|auto)\\b"
			}]
		},
		"identifiers": {
			"patterns": [{
				"name": "variable.nullc",
				"match": "\\b[a-zA-Z_][a-zA-Z_0-9]*\\b"
			}]
		},
		"numbers": {
			"patterns": [{
				"name": "constant.numeric.nullc",
				"match": "0x[0-9a-fA-F]+|[0-9]+\\.[0-9]*f?|[0-9]+l?"
			}]
		}
	},
	"scopeName": "source.nullc"
}
{'fraction_non_alphanumeric': 0.3058440236204439, 'fraction_numerical': 0.0057014864589696596, 'mean_word_length': 2.5465703971119136, 'pattern_counts': {'":': 131, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3431869', 'n_tokens_mistral': 2585, 'n_tokens_neox': 2100, 'n_words': 214}