<commit_before><#macro array a><#list a as i><#if i_index &gt; 0>,</#if>${i}</#list></#macro>
<#macro intarray a><#list a as i><#if i == 10>\n<#elseif i == 13>\r<#else>\u${i?hex(4)}</#if></#list></#macro>
<#if code.copyright?has_content>
${code.copyright}
</#if>
<#if package?has_content>
package ${package};
<@array ecs/>
</#if>

import java.io.FileInputStream;
import java.io.InputStream;
import java.io.IOException;

<#if unicode>
import java.io.Reader;
import java.io.InputStreamReader;
</#if>

<#if public?has_content && public?string == "true">public </#if>class ${ccclass}
{
<#list lexer.states as i>
	protected final static int ${i} = ${lexer.begins[i_index]};
</#list>

	// an internal class for lazy initiation
	private final static class cc_lexer
	{
		private static char[] accept = "<@intarray lexer.accept/>".toCharArray ();
<#if lexer.table == "ecs" || lexer.table == "compressed">
		private static char[] ecs = "<@intarray lexer.ecs/>".toCharArray ();
</#if>
<#if lexer.table == "full" || lexer.table == "ecs">
		private static char[][] next = {<#list lexer.dfa.table as i><#if i_index &gt; 0>,</#if>"<@intarray i/>".toCharArray ()</#list>};
</#if>
<#if lexer.table == "compressed">
		private static char[] base = "<@intarray lexer.dfa.base/>".toCharArray ();
		private static char[] next = "<@intarray lexer.dfa.next/>".toCharArray ();
		private static char[] check = "<@intarray lexer.dfa.check/>".toCharArray ();
	<#if lexer.dfa.default?has_content>
		private static char[] defaults = "<@intarray lexer.dfa.default/>".toCharArray ();
	</#if>
	<#if lexer.dfa.meta?has_content>
		private static char[] meta = "<@intarray lexer.dfa.meta/>".toCharArray ();
	</#if>
</#if>
	}

<#if unicode>
	private Reader _yyIs = new InputStreamReader (System.in);
	private char[] _yyBuffer;
<#else>
	private InputStream _yyIs = System.in;
	private byte[] _yyBuffer;
</#if>
	private int _yyBufferSize = 4096;
	private int _yyMatchStart;
	private int _yyBufferEnd;

	private int _yyBaseState;

	private int _yyTextStart;
	private int _yyLength;

<#if lexer.bol>
	// we need to track beginning of line (BOL) status
	private boolean _yyIsNextBOL = true;
	private boolean _yyBOL = true;
</#if>

<#if unicode>
	public void setInput (Reader reader)
	{
		_yyIs = reader;
	}

	public Reader getInput ()
	{
		return _yyIs;
	}
<#else>
	public void setInput (InputStream is)
	{
		_yyIs = is;
	}

	public InputStream getInput ()
	{
		return _yyIs;
	}
</#if>

<#if lexer.bol>
	public boolean isBOL ()
	{
		return _yyBOL;
	}
</#if>

	/**
	 * Get the current token text.
	 * <p>
	 * Avoid calling this function unless it is absolutely necessary since it creates
	 * a copy of the token string.  The string length can be found by reading _yyLength
	 * or calling yyLength () function.
	 *
	 * @return	the current text token.
	 */
	public String getText ()
	{
		if (_yyMatchStart == _yyTextStart)		// this is the case when we have EOF
			return null;
		return new String (_yyBuffer, _yyTextStart, _yyMatchStart - _yyTextStart);
	}

	/**
	 * Get the current text token's length.  Actions specified in the CookCC file
	 * can directly access the variable
	 * @return	the string token length
	 */
	public int getLength ()
	{
		return _yyLength;
	}

	/**
	 * Print the current string token to the standard output.
	 */
	public void echo ()
	{
		System.out.print (getText ());
	}

	/**
	 * Put all but n characters back to the input stream.  Be aware that calling
	 * yyLess (0) is allowed, but be sure to change the state some how to avoid
	 * an endless loop.
	 */
	protected void yyLess (int n)
	{
		if (n < 0)
			throw new IllegalArgumentException ("yyLess function requires a non-zero value.");
		if (n > (_yyMatchStart - _yyTextStart))
			throw new IndexOutOfBoundsException ("yyLess function called with a too large index value " + n + ".");
		_yyMatchStart = _yyTextStart + n;
	}

	/**
	 * Set the lexer's current state.
	 *
	 * @param	baseState
	 *			the base state index
	 */
	protected void begin (int baseState)
	{
		_yyBaseState = baseState;
	}

	// read more data from the input
	private boolean yyRefreshBuffer () throws IOException
	{
<#if unicode>
		if (_yyBuffer == null)
			_yyBuffer = new char[_yyBufferSize];
<#else>
		if (_yyBuffer == null)
			_yyBuffer = new byte[_yyBufferSize];
</#if>
		if (_yyMatchStart > 0)
		{
			if (_yyBufferEnd > _yyMatchStart)
			{
				System.arraycopy (_yyBuffer, _yyMatchStart, _yyBuffer, 0, _yyBufferEnd - _yyMatchStart);
				_yyBufferEnd -= _yyMatchStart;
				_yyMatchStart = 0;
			}
			else
			{
				_yyMatchStart = 0;
				_yyBufferEnd = 0;
			}
		}
		int readSize = _yyIs.read (_yyBuffer, _yyBufferEnd, _yyBufferSize - _yyBufferEnd);
		if (readSize > 0)
			_yyBufferEnd += readSize;
		return readSize >= 0;
	}

	/**
	 * Reset the internal buffer.
	 */
	public void resetBuffer ()
	{
		_yyMatchStart = 0;
		_yyBufferEnd = 0;
	}

	/**
	 * Set the internal buffer size.  This action can only be performed
	 * when the buffer is empty.  Having a large buffer is useful to read
	 * a whole file in to increase the performance sometimes.
	 *
	 * @param	bufferSize
	 *			the new buffer size.
	 */
	public void setBufferSize (int bufferSize)
	{
		if (_yyBufferEnd > _yyMatchStart)
			throw new IllegalArgumentException ("Cannot change lexer buffer size at this moment.");
		_yyBufferSize = bufferSize;
		_yyMatchStart = 0;
		_yyBufferEnd = 0;
		if (_yyBuffer != null && bufferSize != _yyBuffer.length)
<#if unicode>
			_yyBuffer = new char[bufferSize];
<#else>
			_yyBuffer = new byte[bufferSize];
</#if>
	}

	/**
	 * Call this function to start the scanning of the input.
	 *
	 * @return	a token or status value.
	 * @throws	IOException
	 *			in case of I/O error.
	 */
	public ${yytype} yyLex () throws IOException
	{
<#if code.lexerprolog?has_content>
	${code.lexerprolog}
</#if>

<#if lexer.table == "ecs" || lexer.table == "compressed">
		char[] cc_ecs = cc_lexer.ecs;
</#if>
<#if lexer.table == "ecs" || lexer.table == "full">
		char[][] cc_next = cc_lexer.next;
<#elseif lexer.table="compressed">
		char[] cc_next = cc_lexer.next;
		char[] cc_check = cc_lexer.check;
	<#if lexer.dfa.base?has_content>
		char[] cc_base = cc_lexer.base;
	</#if>
	<#if lexer.dfa.default?has_content>
		char[] cc_default = cc_lexer.defaults;
	</#if>
	<#if lexer.dfa.meta?has_content>
		char[] cc_meta = cc_lexer.meta;
	</#if>
</#if>
		char[] cc_accept = cc_lexer.accept;

<#if unicode>
		char[] buffer = _yyBuffer;
<#else>
		byte[] buffer = _yyBuffer;
</#if>

		while (true)
		{
			// initiate variables necessary for lookup
<#if lexer.bol>
			_yyBOL = _yyIsNextBOL;
			_yyIsNextBOL = false;
	<#if lexer.bolStates>
			int matchedState = _yyBaseState + (_yyBOL ? 1 : 0);
	<#else>
			int matchedState = _yyBaseState;
	</#if>
<#else>
			int matchedState = _yyBaseState;
</#if>

			int matchedLength = 0;

			int internalBufferEnd = _yyBufferEnd;
			int lookahead = _yyMatchStart;

<#if lexer.backup>
			int cc_backupMatchedState = matchedState;
			int cc_backupMatchedLength = 0;
</#if>

			// the DFA lookup
			while (true)
			{
				// check buffer status
				if (lookahead < internalBufferEnd)
				{
					// now okay to process the character
					int currentState;
<#if lexer.table == "full">
					currentState = cc_next[matchedState][buffer[lookahead]<#if !unicode> & 0xff</#if>];
<#elseif lexer.table == "ecs">
					currentState = cc_next[matchedState][cc_ecs[buffer[lookahead]<#if !unicode> & 0xff</#if>]];
<#else>
					int symbol = cc_ecs[buffer[lookahead]<#if !unicode> & 0xff</#if>];
					currentState = matchedState;
	<#if !lexer.dfa.default?has_content>
					if (cc_check[symbol + cc_base[matchedState]] == matchedState)
						currentState = cc_next[symbol + cc_base[matchedState]];
					else
						currentState = 0;
	<#elseif !lexer.dfa.error?has_content>
					if (cc_check[symbol + cc_base[matchedState]] == matchedState)
						currentState = cc_next[symbol + cc_base[matchedState]];
					else
						currentState = cc_default[matchedState];
	<#elseif !lexer.dfa.meta?has_content>
					while (cc_check[symbol + cc_base[currentState]] != currentState)
					{
						currentState = cc_default[currentState];
						if (currentState >= ${lexer.dfa.size})
							symbol = 0;
					}
					currentState = cc_next[symbol + cc_base[currentState]];
	<#else>
					while (cc_check[symbol + cc_base[currentState]] != currentState)
					{
						currentState = cc_default[currentState];
						if (currentState >= ${lexer.dfa.states})
							symbol = cc_meta[symbol];
					}
					currentState = cc_next[symbol + cc_base[currentState]];
	</#if>
</#if>

<#if lexer.backup>
					if (currentState == 0)
					{
						matchedState = cc_backupMatchedState;
						matchedLength = cc_backupMatchedLength;
						break;
					}
<#else>
					if (currentState == 0)
						break;
</#if>

					matchedState = currentState;
					++lookahead;
					++matchedLength;

<#if lexer.backup>
					if (cc_accept[matchedState] > 0)
					{
						cc_backupMatchedState = currentState;
						cc_backupMatchedLength = matchedLength;
					}
</#if>
				}
				else
				{
					int lookPos = lookahead - _yyMatchStart;
					boolean refresh = yyRefreshBuffer ();
					buffer = _yyBuffer;
					internalBufferEnd = _yyBufferEnd;
					lookahead = _yyMatchStart + lookPos;
					if (! refresh)
					{
						// <<EOF>>
						int currentState;
<#if lexer.table == "full">
						currentState = cc_next[matchedState][${lexer.eof}];
<#elseif lexer.table == "ecs">
						currentState = cc_next[matchedState][cc_ecs[${lexer.eof}]];
<#elseif lexer.table == "compressed">
						currentState = _yyBaseState;
</#if>
						if (currentState != 0)
							matchedState = currentState;
<#if lexer.backup>
						else
						{
							matchedState = cc_backupMatchedState;
							matchedLength = cc_backupMatchedLength;
						}
</#if>
						break;
					}
				}
			}

			_yyTextStart = _yyMatchStart;
			_yyMatchStart += matchedLength;
			_yyLength = matchedLength;

			switch (cc_accept[matchedState])
			{
<#list lexer.cases as i>
	<#if !i.internal>
		<#list i.patterns as p>
				case ${p.caseValue}:	// ${p.pattern}
				{
			<#if p.trailContext != 0 && p.trailContext != 3>
				<#if (p.trailContext % 2) == 1>
					_yyLength = ${p.trailLength};
					_yyMatchStart = _yyTextStart + ${p.trailLength};
				<#else>
					_yyLength -= ${p.trailLength};
					_yyMatchStart -= ${p.trailLength};
				</#if>
			</#if>
					${i.action}
				}
				case ${p.caseValue + lexer.caseCount + 1}: break;
		</#list>
	<#else>
		<#list i.patterns as p>
				case ${p.caseValue}:	// ${p.pattern}
				{
			<#if p.pattern == "<<EOF>>">
					return 0;			// default EOF action
			<#else>
					echo ();			// default character action
			</#if>
				}
				case ${p.caseValue + lexer.caseCount + 1}: break;
		</#list>
	</#if>
</#list>
				default:
					throw new IOException ("Internal error in ${ccclass} lexer.");
			}

<#if lexer.bol>
			// check BOL here since '\n' may be unput back into the stream buffer

			// specifically used _yyBuffer since it could be changed by user
			if (_yyMatchStart > 0 && _yyBuffer[_yyMatchStart - 1] == '\n')
				_yyIsNextBOL = true;
</#if>
		}
	}

<#if main?has_content && main?string == "true">
	/**
	 * This is a stub main function that either reads the file that user specified
	 * or from the standard input.
	 *
	 * @param	args
	 *			command line arguments.
	 *
	 * @throws	Exception
	 *			in case of any errors.
	 */
	public static void main (String[] args) throws Exception
	{
		${ccclass} tmpLexer = new ${ccclass} ();
<#if unicode>
		if (args.length > 0)
			tmpLexer.setInput (new InputStreamReader (FileInputStream (args[0])));
<#else>
		if (args.length > 0)
			tmpLexer.setInput (new FileInputStream (args[0]));
</#if>

		tmpLexer.yyLex ();
	}
</#if>

<#if code.default?has_content>
${code.default}
</#if>

/*
 * properties and statistics
 * unicode = ${unicode?string}
 * bol = ${lexer.bol?string}
 * backup = ${lexer.backup?string}
 * cases = ${lexer.caseCount}
 * table = ${lexer.table}
<#if lexer.table == "ecs" || lexer.table == "compressed">
 * ecs = ${lexer.ecsGroupCount}
</#if>
 * states = ${lexer.dfa.size}
 * max symbol value = ${lexer.maxSymbol}
 */
}
<commit_msg>add classheader section.<commit_after><#macro array a><#list a as i><#if i_index &gt; 0>,</#if>${i}</#list></#macro>
<#macro intarray a><#list a as i><#if i == 10>\n<#elseif i == 13>\r<#else>\u${i?hex(4)}</#if></#list></#macro>
<#if code.copyright?has_content>
${code.copyright}
</#if>
<#if package?has_content>
package ${package};
<@array ecs/>
</#if>

import java.io.FileInputStream;
import java.io.InputStream;
import java.io.IOException;

<#if unicode>
import java.io.Reader;
import java.io.InputStreamReader;
</#if>

<#if code.classheader?has_content>
${code.classheader}
</#if>
<#if public?has_content && public?string == "true">public </#if>class ${ccclass}
{
<#list lexer.states as i>
	protected final static int ${i} = ${lexer.begins[i_index]};
</#list>

	// an internal class for lazy initiation
	private final static class cc_lexer
	{
		private static char[] accept = "<@intarray lexer.accept/>".toCharArray ();
<#if lexer.table == "ecs" || lexer.table == "compressed">
		private static char[] ecs = "<@intarray lexer.ecs/>".toCharArray ();
</#if>
<#if lexer.table == "full" || lexer.table == "ecs">
		private static char[][] next = {<#list lexer.dfa.table as i><#if i_index &gt; 0>,</#if>"<@intarray i/>".toCharArray ()</#list>};
</#if>
<#if lexer.table == "compressed">
		private static char[] base = "<@intarray lexer.dfa.base/>".toCharArray ();
		private static char[] next = "<@intarray lexer.dfa.next/>".toCharArray ();
		private static char[] check = "<@intarray lexer.dfa.check/>".toCharArray ();
	<#if lexer.dfa.default?has_content>
		private static char[] defaults = "<@intarray lexer.dfa.default/>".toCharArray ();
	</#if>
	<#if lexer.dfa.meta?has_content>
		private static char[] meta = "<@intarray lexer.dfa.meta/>".toCharArray ();
	</#if>
</#if>
	}

<#if unicode>
	private Reader _yyIs = new InputStreamReader (System.in);
	private char[] _yyBuffer;
<#else>
	private InputStream _yyIs = System.in;
	private byte[] _yyBuffer;
</#if>
	private int _yyBufferSize = 4096;
	private int _yyMatchStart;
	private int _yyBufferEnd;

	private int _yyBaseState;

	private int _yyTextStart;
	private int _yyLength;

<#if lexer.bol>
	// we need to track beginning of line (BOL) status
	private boolean _yyIsNextBOL = true;
	private boolean _yyBOL = true;
</#if>

<#if unicode>
	public void setInput (Reader reader)
	{
		_yyIs = reader;
	}

	public Reader getInput ()
	{
		return _yyIs;
	}
<#else>
	public void setInput (InputStream is)
	{
		_yyIs = is;
	}

	public InputStream getInput ()
	{
		return _yyIs;
	}
</#if>

<#if lexer.bol>
	public boolean isBOL ()
	{
		return _yyBOL;
	}
</#if>

	/**
	 * Get the current token text.
	 * <p>
	 * Avoid calling this function unless it is absolutely necessary since it creates
	 * a copy of the token string.  The string length can be found by reading _yyLength
	 * or calling yyLength () function.
	 *
	 * @return	the current text token.
	 */
	public String getText ()
	{
		if (_yyMatchStart == _yyTextStart)		// this is the case when we have EOF
			return null;
		return new String (_yyBuffer, _yyTextStart, _yyMatchStart - _yyTextStart);
	}

	/**
	 * Get the current text token's length.  Actions specified in the CookCC file
	 * can directly access the variable
	 * @return	the string token length
	 */
	public int getLength ()
	{
		return _yyLength;
	}

	/**
	 * Print the current string token to the standard output.
	 */
	public void echo ()
	{
		System.out.print (getText ());
	}

	/**
	 * Put all but n characters back to the input stream.  Be aware that calling
	 * yyLess (0) is allowed, but be sure to change the state some how to avoid
	 * an endless loop.
	 */
	protected void yyLess (int n)
	{
		if (n < 0)
			throw new IllegalArgumentException ("yyLess function requires a non-zero value.");
		if (n > (_yyMatchStart - _yyTextStart))
			throw new IndexOutOfBoundsException ("yyLess function called with a too large index value " + n + ".");
		_yyMatchStart = _yyTextStart + n;
	}

	/**
	 * Set the lexer's current state.
	 *
	 * @param	baseState
	 *			the base state index
	 */
	protected void begin (int baseState)
	{
		_yyBaseState = baseState;
	}

	// read more data from the input
	private boolean yyRefreshBuffer () throws IOException
	{
<#if unicode>
		if (_yyBuffer == null)
			_yyBuffer = new char[_yyBufferSize];
<#else>
		if (_yyBuffer == null)
			_yyBuffer = new byte[_yyBufferSize];
</#if>
		if (_yyMatchStart > 0)
		{
			if (_yyBufferEnd > _yyMatchStart)
			{
				System.arraycopy (_yyBuffer, _yyMatchStart, _yyBuffer, 0, _yyBufferEnd - _yyMatchStart);
				_yyBufferEnd -= _yyMatchStart;
				_yyMatchStart = 0;
			}
			else
			{
				_yyMatchStart = 0;
				_yyBufferEnd = 0;
			}
		}
		int readSize = _yyIs.read (_yyBuffer, _yyBufferEnd, _yyBufferSize - _yyBufferEnd);
		if (readSize > 0)
			_yyBufferEnd += readSize;
		return readSize >= 0;
	}

	/**
	 * Reset the internal buffer.
	 */
	public void resetBuffer ()
	{
		_yyMatchStart = 0;
		_yyBufferEnd = 0;
	}

	/**
	 * Set the internal buffer size.  This action can only be performed
	 * when the buffer is empty.  Having a large buffer is useful to read
	 * a whole file in to increase the performance sometimes.
	 *
	 * @param	bufferSize
	 *			the new buffer size.
	 */
	public void setBufferSize (int bufferSize)
	{
		if (_yyBufferEnd > _yyMatchStart)
			throw new IllegalArgumentException ("Cannot change lexer buffer size at this moment.");
		_yyBufferSize = bufferSize;
		_yyMatchStart = 0;
		_yyBufferEnd = 0;
		if (_yyBuffer != null && bufferSize != _yyBuffer.length)
<#if unicode>
			_yyBuffer = new char[bufferSize];
<#else>
			_yyBuffer = new byte[bufferSize];
</#if>
	}

	/**
	 * Call this function to start the scanning of the input.
	 *
	 * @return	a token or status value.
	 * @throws	IOException
	 *			in case of I/O error.
	 */
	public ${yytype} yyLex () throws IOException
	{
<#if code.lexerprolog?has_content>
	${code.lexerprolog}
</#if>

<#if lexer.table == "ecs" || lexer.table == "compressed">
		char[] cc_ecs = cc_lexer.ecs;
</#if>
<#if lexer.table == "ecs" || lexer.table == "full">
		char[][] cc_next = cc_lexer.next;
<#elseif lexer.table="compressed">
		char[] cc_next = cc_lexer.next;
		char[] cc_check = cc_lexer.check;
	<#if lexer.dfa.base?has_content>
		char[] cc_base = cc_lexer.base;
	</#if>
	<#if lexer.dfa.default?has_content>
		char[] cc_default = cc_lexer.defaults;
	</#if>
	<#if lexer.dfa.meta?has_content>
		char[] cc_meta = cc_lexer.meta;
	</#if>
</#if>
		char[] cc_accept = cc_lexer.accept;

<#if unicode>
		char[] buffer = _yyBuffer;
<#else>
		byte[] buffer = _yyBuffer;
</#if>

		while (true)
		{
			// initiate variables necessary for lookup
<#if lexer.bol>
			_yyBOL = _yyIsNextBOL;
			_yyIsNextBOL = false;
	<#if lexer.bolStates>
			int matchedState = _yyBaseState + (_yyBOL ? 1 : 0);
	<#else>
			int matchedState = _yyBaseState;
	</#if>
<#else>
			int matchedState = _yyBaseState;
</#if>

			int matchedLength = 0;

			int internalBufferEnd = _yyBufferEnd;
			int lookahead = _yyMatchStart;

<#if lexer.backup>
			int cc_backupMatchedState = matchedState;
			int cc_backupMatchedLength = 0;
</#if>

			// the DFA lookup
			while (true)
			{
				// check buffer status
				if (lookahead < internalBufferEnd)
				{
					// now okay to process the character
					int currentState;
<#if lexer.table == "full">
					currentState = cc_next[matchedState][buffer[lookahead]<#if !unicode> & 0xff</#if>];
<#elseif lexer.table == "ecs">
					currentState = cc_next[matchedState][cc_ecs[buffer[lookahead]<#if !unicode> & 0xff</#if>]];
<#else>
					int symbol = cc_ecs[buffer[lookahead]<#if !unicode> & 0xff</#if>];
					currentState = matchedState;
	<#if !lexer.dfa.default?has_content>
					if (cc_check[symbol + cc_base[matchedState]] == matchedState)
						currentState = cc_next[symbol + cc_base[matchedState]];
					else
						currentState = 0;
	<#elseif !lexer.dfa.error?has_content>
					if (cc_check[symbol + cc_base[matchedState]] == matchedState)
						currentState = cc_next[symbol + cc_base[matchedState]];
					else
						currentState = cc_default[matchedState];
	<#elseif !lexer.dfa.meta?has_content>
					while (cc_check[symbol + cc_base[currentState]] != currentState)
					{
						currentState = cc_default[currentState];
						if (currentState >= ${lexer.dfa.size})
							symbol = 0;
					}
					currentState = cc_next[symbol + cc_base[currentState]];
	<#else>
					while (cc_check[symbol + cc_base[currentState]] != currentState)
					{
						currentState = cc_default[currentState];
						if (currentState >= ${lexer.dfa.states})
							symbol = cc_meta[symbol];
					}
					currentState = cc_next[symbol + cc_base[currentState]];
	</#if>
</#if>

<#if lexer.backup>
					if (currentState == 0)
					{
						matchedState = cc_backupMatchedState;
						matchedLength = cc_backupMatchedLength;
						break;
					}
<#else>
					if (currentState == 0)
						break;
</#if>

					matchedState = currentState;
					++lookahead;
					++matchedLength;

<#if lexer.backup>
					if (cc_accept[matchedState] > 0)
					{
						cc_backupMatchedState = currentState;
						cc_backupMatchedLength = matchedLength;
					}
</#if>
				}
				else
				{
					int lookPos = lookahead - _yyMatchStart;
					boolean refresh = yyRefreshBuffer ();
					buffer = _yyBuffer;
					internalBufferEnd = _yyBufferEnd;
					lookahead = _yyMatchStart + lookPos;
					if (! refresh)
					{
						// <<EOF>>
						int currentState;
<#if lexer.table == "full">
						currentState = cc_next[matchedState][${lexer.eof}];
<#elseif lexer.table == "ecs">
						currentState = cc_next[matchedState][cc_ecs[${lexer.eof}]];
<#elseif lexer.table == "compressed">
						currentState = _yyBaseState;
</#if>
						if (currentState != 0)
							matchedState = currentState;
<#if lexer.backup>
						else
						{
							matchedState = cc_backupMatchedState;
							matchedLength = cc_backupMatchedLength;
						}
</#if>
						break;
					}
				}
			}

			_yyTextStart = _yyMatchStart;
			_yyMatchStart += matchedLength;
			_yyLength = matchedLength;

			switch (cc_accept[matchedState])
			{
<#list lexer.cases as i>
	<#if !i.internal>
		<#list i.patterns as p>
				case ${p.caseValue}:	// ${p.pattern}
				{
			<#if p.trailContext != 0 && p.trailContext != 3>
				<#if (p.trailContext % 2) == 1>
					_yyLength = ${p.trailLength};
					_yyMatchStart = _yyTextStart + ${p.trailLength};
				<#else>
					_yyLength -= ${p.trailLength};
					_yyMatchStart -= ${p.trailLength};
				</#if>
			</#if>
					${i.action}
				}
				case ${p.caseValue + lexer.caseCount + 1}: break;
		</#list>
	<#else>
		<#list i.patterns as p>
				case ${p.caseValue}:	// ${p.pattern}
				{
			<#if p.pattern == "<<EOF>>">
					return 0;			// default EOF action
			<#else>
					echo ();			// default character action
			</#if>
				}
				case ${p.caseValue + lexer.caseCount + 1}: break;
		</#list>
	</#if>
</#list>
				default:
					throw new IOException ("Internal error in ${ccclass} lexer.");
			}

<#if lexer.bol>
			// check BOL here since '\n' may be unput back into the stream buffer

			// specifically used _yyBuffer since it could be changed by user
			if (_yyMatchStart > 0 && _yyBuffer[_yyMatchStart - 1] == '\n')
				_yyIsNextBOL = true;
</#if>
		}
	}

<#if main?has_content && main?string == "true">
	/**
	 * This is a stub main function that either reads the file that user specified
	 * or from the standard input.
	 *
	 * @param	args
	 *			command line arguments.
	 *
	 * @throws	Exception
	 *			in case of any errors.
	 */
	public static void main (String[] args) throws Exception
	{
		${ccclass} tmpLexer = new ${ccclass} ();
<#if unicode>
		if (args.length > 0)
			tmpLexer.setInput (new InputStreamReader (FileInputStream (args[0])));
<#else>
		if (args.length > 0)
			tmpLexer.setInput (new FileInputStream (args[0]));
</#if>

		tmpLexer.yyLex ();
	}
</#if>

<#if code.default?has_content>
${code.default}
</#if>

/*
 * properties and statistics
 * unicode = ${unicode?string}
 * bol = ${lexer.bol?string}
 * backup = ${lexer.backup?string}
 * cases = ${lexer.caseCount}
 * table = ${lexer.table}
<#if lexer.table == "ecs" || lexer.table == "compressed">
 * ecs = ${lexer.ecsGroupCount}
</#if>
 * states = ${lexer.dfa.size}
 * max symbol value = ${lexer.maxSymbol}
 */
}

{'fraction_non_alphanumeric': 0.14040635623369435, 'fraction_numerical': 0.003952881650723377, 'mean_word_length': 2.8589078706528372, 'pattern_counts': {'":': 0, '<': 301, '<?xml version=': 0, '>': 321, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2542518', 'n_tokens_mistral': 10311, 'n_tokens_neox': 8758, 'n_words': 2262}