<commit_before>// Scintilla source code edit control
/** @file LexLua.cxx
 ** Lexer for Lua language.
 **
 ** Written by <NAME>.
 ** Folder by <NAME>.
 **/

#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <stdarg.h>
#include <stdio.h>
#include <fcntl.h>

#include "Platform.h"

#include "PropSet.h"
#include "Accessor.h"
#include "KeyWords.h"
#include "Scintilla.h"
#include "SciLexer.h"

inline bool isLuaOperator(char ch) {
	if (isalnum(ch))
		return false;
	// '.' left out as it is used to make up numbers
	if (ch == '*' || ch == '/' || ch == '-' || ch == '+' ||
	        ch == '(' || ch == ')' || ch == '=' ||
	        ch == '{' || ch == '}' || ch == '~' ||
	        ch == '[' || ch == ']' || ch == ';' ||
	        ch == '<' || ch == '>' || ch == ',' ||
	        ch == '.' || ch == '^' || ch == '%' || ch == ':')
		return true;
	return false;
}

static void classifyWordLua(unsigned int start,
                            unsigned int end,
                            WordList	&keywords,
                            Accessor	&styler) {
	char s[100];
	bool wordIsNumber = isdigit(styler[start]) || (styler[start] == '.');

	for (unsigned int i = 0; i < end - start + 1 && i < 30; i++) {
		s[i] = styler[start + i];
		s[i + 1] = '\0';
	}

	char chAttr = SCE_LUA_IDENTIFIER;

	if (wordIsNumber)
		chAttr = SCE_LUA_NUMBER;
	else {
		if (keywords.InList(s)) {
			chAttr = SCE_LUA_WORD;
		}
	}
	styler.ColourTo(end, chAttr);
}

static void ColouriseLuaDoc(unsigned int startPos,
                            int length,
                            int initStyle,
                            WordList *keywordlists[],
                            Accessor &styler) {

	WordList &keywords = *keywordlists[0];

	styler.StartAt(startPos);
	styler.GetLine(startPos);

	int state = initStyle;
	char chPrev = ' ';
	char chNext = styler[startPos];
	unsigned int lengthDoc = startPos + length;
	bool firstChar = true;
	int literalString = 0;

	styler.StartSegment(startPos);
	for (unsigned int i = startPos; i <= lengthDoc; i++) {
		char ch = chNext;
		chNext = styler.SafeGetCharAt(i + 1);

		if (styler.IsLeadByte(ch)) {<commit_msg>Patch from Philippe to set literal string state correctly based on initial state.<commit_after>// Scintilla source code edit control
/** @file LexLua.cxx
 ** Lexer for Lua language.
 **
 ** Written by <NAME>.
 ** Folder by <NAME>.
 **/

#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <stdarg.h>
#include <stdio.h>
#include <fcntl.h>

#include "Platform.h"

#include "PropSet.h"
#include "Accessor.h"
#include "KeyWords.h"
#include "Scintilla.h"
#include "SciLexer.h"

inline bool isLuaOperator(char ch) {
	if (isalnum(ch))
		return false;
	// '.' left out as it is used to make up numbers
	if (ch == '*' || ch == '/' || ch == '-' || ch == '+' ||
	        ch == '(' || ch == ')' || ch == '=' ||
	        ch == '{' || ch == '}' || ch == '~' ||
	        ch == '[' || ch == ']' || ch == ';' ||
	        ch == '<' || ch == '>' || ch == ',' ||
	        ch == '.' || ch == '^' || ch == '%' || ch == ':')
		return true;
	return false;
}

static void classifyWordLua(unsigned int start,
                            unsigned int end,
                            WordList	&keywords,
                            Accessor	&styler) {
	char s[100];
	bool wordIsNumber = isdigit(styler[start]) || (styler[start] == '.');

	for (unsigned int i = 0; i < end - start + 1 && i < 30; i++) {
		s[i] = styler[start + i];
		s[i + 1] = '\0';
	}

	char chAttr = SCE_LUA_IDENTIFIER;

	if (wordIsNumber)
		chAttr = SCE_LUA_NUMBER;
	else {
		if (keywords.InList(s)) {
			chAttr = SCE_LUA_WORD;
		}
	}
	styler.ColourTo(end, chAttr);
}

static void ColouriseLuaDoc(unsigned int startPos,
                            int length,
                            int initStyle,
                            WordList *keywordlists[],
                            Accessor &styler) {

	WordList &keywords = *keywordlists[0];

	styler.StartAt(startPos);
	styler.GetLine(startPos);

	int state = initStyle;
	char chPrev = ' ';
	char chNext = styler[startPos];
	unsigned int lengthDoc = startPos + length;
	bool firstChar = true;

	/* Must initialize the literalString level, if we are inside such a string.
	 * Note: this isn't enough, because literal strings can be nested,
	 * we should go back to see at what level we are...
	 */
	int literalString = (initStyle == SCE_LUA_LITERALSTRING) ? 1 : 0;

	styler.StartSegment(startPos);
	for (unsigned int i = startPos; i <= lengthDoc; i++) {
		char ch = chNext;
		chNext = styler.SafeGetCharAt(i + 1);

		if (styler.IsLeadByte(ch)) {
{'fraction_non_alphanumeric': 0.17134159599913024, 'fraction_numerical': 0.005435964340073929, 'mean_word_length': 2.389830508474576, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7170280', 'n_tokens_mistral': 1699, 'n_tokens_neox': 1596, 'n_words': 426}