<commit_before>package t

import (
	. "github.com/alecthomas/chroma" // nolint
	"github.com/alecthomas/chroma/lexers/internal"
)

// Terraform lexer.
var Terraform = internal.Register(MustNewLexer(
	&Config{
		Name:      "Terraform",
		Aliases:   []string{"terraform", "tf"},
		Filenames: []string{"*.tf"},
		MimeTypes: []string{"application/x-tf", "application/x-terraform"},
	},
	Rules{
		"root": {
			Include("string"),
			Include("punctuation"),
			Include("curly"),
			Include("basic"),
			Include("whitespace"),
			{`[0-9]+`, LiteralNumber, nil},
		},
		"basic": {
			{Words(`\b`, `\b`, `true`, `false`), KeywordType, nil},
			{`\s*/\*`, CommentMultiline, Push("comment")},
			{`\s*#.*\n`, CommentSingle, nil},
			{`(.*?)(\s*)(=)`, ByGroups(NameAttribute, Text, Operator), nil},
			{Words(`\b`, `\b`, `variable`, `resource`, `provider`, `provisioner`, `module`), KeywordReserved, Push("function")},
			{Words(`\b`, `\b`, `ingress`, `egress`, `listener`, `default`, `connection`, `alias`), KeywordDeclaration, nil},
			{`\$\{`, LiteralStringInterpol, Push("var_builtin")},
		},
		"function": {
			{`(\s+)(".*")(\s+)`, ByGroups(Text, LiteralString, Text), nil},
			Include("punctuation"),
			Include("curly"),
		},
		"var_builtin": {
			{`\$\{`, LiteralStringInterpol, Push()},
			{Words(`\b`, `\b`, `concat`, `file`, `join`, `lookup`, `element`), NameBuiltin, nil},
			Include("string"),
			Include("punctuation"),
			{`\s+`, Text, nil},
			{`\}`, LiteralStringInterpol, Pop(1)},
		},
		"string": {
			{`(".*")`, ByGroups(LiteralStringDouble), nil},
		},
		"punctuation": {
			{`[\[\](),.]`, Punctuation, nil},
		},
		"curly": {
			{`\{`, TextPunctuation, nil},
			{`\}`, TextPunctuation, nil},
		},
		"comment": {
			{`[^*/]`, CommentMultiline, nil},
			{`/\*`, CommentMultiline, Push()},
			{`\*/`, CommentMultiline, Pop(1)},
			{`[*/]`, CommentMultiline, nil},
		},
		"whitespace": {
			{`\n`, Text, nil},
			{`\s+`, Text, nil},
			{`\\\n`, Text, nil},
		},
	},
))<commit_msg>Updated terraform rules for Terraform 0.12+. Based on terraform documentation (https://www.terraform.io/docs/configuration/index.html). Added all builtin functions and updated syntax rules.<commit_after>package t

import (
	. "github.com/alecthomas/chroma" // nolint
	"github.com/alecthomas/chroma/lexers/internal"
)

// Terraform lexer.
var Terraform = internal.Register(MustNewLexer(
	&Config{
		Name:      "Terraform",
		Aliases:   []string{"terraform", "tf"},
		Filenames: []string{"*.tf"},
		MimeTypes: []string{"application/x-tf", "application/x-terraform"},
	},
	Rules{
		"root": {
			{`[\[\](),.{}]`, Punctuation, nil},
			{`-?[0-9]+`, LiteralNumber, nil},
			{`=>`, Punctuation, nil},
			{Words(``, `\b`, `true`, `false`), KeywordConstant, nil},
			{`/(?s)\*(((?!\*/).)*)\*/`, CommentMultiline, nil},
			{`\s*(#|//).*\n`, CommentSingle, nil},
			{`([a-zA-Z]\w*)(\s*)(=(?!>))`, ByGroups(NameAttribute, Text, Text), nil},
			{Words(`^\s*`, `\b`, `variable`, `data`, `resource`, `provider`, `provisioner`, `module`, `output`), KeywordReserved, nil},
			{Words(``, `\b`, `for`, `in`), Keyword, nil },
			{Words(``, ``, `count`, `data`, `var`, `module`, `each`), NameBuiltin, nil},
			{Words(``, `\b`, `abs`, `ceil`, `floor`, `log`, `max`, `min`, `parseint`, `pow`, `signum`, ), NameBuiltin, nil},
			{Words(``, `\b`, `chomp`, `format`, `formatlist`, `indent`, `join`, `lower`, `regex`, `regexall`, `replace`, `split`, `strrev`, `substr`, `title`, `trim`, `trimprefix`, `trimsuffix`, `trimspace`, `upper`), NameBuiltin, nil},
			{Words(`[^.]`, `\b`, `chunklist`,`coalesce`,`coalescelist`,`compact`,`concat`,`contains`,`distinct`,`element`,`flatten`,`index`,`keys`,`length`,`list`,`lookup`,`map`,`matchkeys`,`merge`,`range`,`reverse`,`setintersection`,`setproduct`,`setsubtract`,`setunion`,`slice`,`sort`,`transpose`,`values`,`zipmap`), NameBuiltin, nil},
			{Words(`[^.]`, `\b`, `base64decode`,`base64encode`,`base64gzip`,`csvdecode`,`jsondecode`,`jsonencode`,`urlencode`,`yamldecode`,`yamlencode`), NameBuiltin, nil},
			{Words(``, `\b`, `abspath`,`dirname`,`pathexpand`,`basename`,`file`,`fileexists`,`fileset`,`filebase64`,`templatefile`), NameBuiltin, nil},
			{Words(``, `\b`, `formatdate`,`timeadd`,`timestamp`), NameBuiltin, nil},
			{Words(``, `\b`, `base64sha256`,`base64sha512`,`bcrypt`,`filebase64sha256`,`filebase64sha512`,`filemd5`,`filesha1`,`filesha256`,`filesha512`,`md5`,`rsadecrypt`,`sha1`,`sha256`,`sha512`,`uuid`,`uuidv5`), NameBuiltin, nil},
			{Words(``, `\b`, `cidrhost`,`cidrnetmask`,`cidrsubnet`), NameBuiltin, nil},
			{Words(``, `\b`, `can`,`tobool`,`tolist`,`tomap`,`tonumber`,`toset`,`tostring`,`try`), NameBuiltin, nil},
			{`=(?!>)|\+|-|\*|\/|:|!|%|>|<(?!<)|>=|<=|==|!=|&&|\||\?`, Operator, nil},
			{`\n|\s+|\\\n`, Text, nil},
			{`[a-zA-Z]\w*`, NameOther, nil},
			{`"`, LiteralStringDouble, Push("string")},
			{`(?s)(<<-?)(\w+)(\n\s*(?:(?!\2).)*\s*\n\s*)(\2)`, ByGroups(Operator, Operator, String, Operator), nil},
		},
		"declaration": {
			{`(\s*)("(?:\\\\|\\"|[^"])*")(\s*)`, ByGroups(Text, NameVariable, Text), nil},
			{`\{`, Punctuation, Pop(1)},
		},
		"string": {
			{`"`, LiteralStringDouble, Pop(1)},
			{`\\\\`, LiteralStringDouble, nil},
			{`\\\\"`, LiteralStringDouble, nil},
			{`\$\{`, LiteralStringInterpol, Push("interp-inside")},
			{`\$`, LiteralStringDouble, nil},
			{`[^"\\\\$]+`, LiteralStringDouble, nil},
		},
		"interp-inside": {
			{`\}`, LiteralStringInterpol, Pop(1)},
			Include("root"),
		},

	},
))
{'fraction_non_alphanumeric': 0.29592959295929594, 'fraction_numerical': 0.010817748441510818, 'mean_word_length': 5.184807256235827, 'pattern_counts': {'":': 13, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7281090', 'n_tokens_mistral': 2395, 'n_tokens_neox': 2081, 'n_words': 330}