<commit_before>--- Turbo.lua Mustache.js fast compiler.
-- Logic-less templates with {{ }}, http://mustache.github.io/
-- Turbo.lua has a small and very fast Mustache parser built-in. Mustache
-- templates are logic-less templates, which are supposed to help you keep
-- your business logic outside of templates and inside "controllers".
--
-- For more information on Mustache, please see this:
-- http://mustache.github.io/mustache.5.html
--
-- Copyright 2011, 2012, 2013 <NAME>
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.

local util = require "turbo.util"
local escape = require "turbo.escape"
local buffer = require "turbo.structs.buffer"
local log = require "turbo.log"
local ffi = require "ffi"
local bit = require "bit"
require "turbo.3rdparty.middleclass"

local b = string.byte

local START =           "{{"
local START_NO_ESC =        "{{{"
local END =             "}}"
local END_NO_ESC =      "}}}"
local START_SEC =       '#'
local END_SEC =         '/'
local START_INV_SEC =       '^'
local COMMENT =         '!'
local ESCAPE =          '&'
local PARTIAL =         '>'

--- Mustache parser states.
local PNONE =           0x00
local PSTART =          0x01
local PSTARTNOESC =         0x02
local PSTARTSEC =       0x03
local PSTARTSECINV =        0x04
local PTABLEKEY =       0x05
local PTABLEKEYNOESC =      0x06
local PEND =            0x07
local PCOMMENT =        0x08
local PPARTIAL =        0x09
local PPARTIALNOESC =       0x10

--- Template instruction set
local MSTR =            0x01    -- Constant string
local TKEY =            0x02    -- Table key
local TKEE =            0x22    -- Table key escape
local SECS =            0x03    -- Section straight
local SECI =            0x33    -- Section inverted<commit_msg>Fixed indents in defines.<commit_after>--- Turbo.lua Mustache.js fast compiler.
-- Logic-less templates with {{ }}, http://mustache.github.io/
-- Turbo.lua has a small and very fast Mustache parser built-in. Mustache
-- templates are logic-less templates, which are supposed to help you keep
-- your business logic outside of templates and inside "controllers".
--
-- For more information on Mustache, please see this:
-- http://mustache.github.io/mustache.5.html
--
-- Copyright 2011, 2012, 2013 <NAME>
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.

local util = require "turbo.util"
local escape = require "turbo.escape"
local buffer = require "turbo.structs.buffer"
local log = require "turbo.log"
local ffi = require "ffi"
local bit = require "bit"
require "turbo.3rdparty.middleclass"

local b = string.byte

local START =           "{{"
local START_NO_ESC =    "{{{"
local END =             "}}"
local END_NO_ESC =      "}}}"
local START_SEC =       '#'
local END_SEC =         '/'
local START_INV_SEC =   '^'
local COMMENT =         '!'
local ESCAPE =          '&'
local PARTIAL =         '>'

--- Mustache parser states.
local PNONE =           0x00
local PSTART =          0x01
local PSTARTNOESC =     0x02
local PSTARTSEC =       0x03
local PSTARTSECINV =    0x04
local PTABLEKEY =       0x05
local PTABLEKEYNOESC =  0x06
local PEND =            0x07
local PCOMMENT =        0x08
local PPARTIAL =        0x09
local PPARTIALNOESC =   0x10

--- Template instruction set
local MSTR =            0x01    -- Constant string
local TKEY =            0x02    -- Table key
local TKEE =            0x22    -- Table key escape
local SECS =            0x03    -- Section straight
local SECI =            0x33    -- Section inverted
{'fraction_non_alphanumeric': 0.09696834596522515, 'fraction_numerical': 0.029424877396344182, 'mean_word_length': 2.4595219737856593, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5960226', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1298, 'n_words': 494}