<commit_before>// Scintilla source code edit control
/** @file LexScriptol.cxx
 ** Lexer for Scriptol.
 **/

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdarg.h>
#include <assert.h>
#include <ctype.h>

#include "ILexer.h"
#include "Scintilla.h"
#include "SciLexer.h"

#include "WordList.h"
#include "LexAccessor.h"
#include "Accessor.h"
#include "StyleContext.h"
#include "CharacterSet.h"
#include "LexerModule.h"

#ifdef SCI_NAMESPACE
using namespace Scintilla;
#endif

static void ClassifyWordSol(unsigned int start, unsigned int end, WordList &keywords, Accessor &styler, char *prevWord)
{
    char s[100] = "";
    bool wordIsNumber = isdigit(styler[start]) != 0;
    for (unsigned int i = 0; i < end - start + 1 && i < 30; i++)
     {
           s[i] = styler[start + i];
           s[i + 1] = '\0';
     }
    char chAttr = SCE_SCRIPTOL_IDENTIFIER;
    if (0 == strcmp(prevWord, "class"))       chAttr = SCE_SCRIPTOL_CLASSNAME;
    else if (wordIsNumber)                    chAttr = SCE_SCRIPTOL_NUMBER;
    else if (keywords.InList(s))              chAttr = SCE_SCRIPTOL_KEYWORD;
    else for (unsigned int i = 0; i < end - start + 1; i++)  // test dotted idents
    {
        if (styler[start + i] == '.')
        {
            styler.ColourTo(start + i - 1, chAttr);
            styler.ColourTo(start + i, SCE_SCRIPTOL_OPERATOR);
        }
    }
    styler.ColourTo(end, chAttr);
    strcpy(prevWord, s);
}

static bool IsSolComment(Accessor &styler, int pos, int len)
{
   if(len > 0)
   {
     char c = styler[pos];
     if(c == '`') return true;
     if(len > 1)
     {
        if(c == '/')
        {
          c = styler[pos + 1];
          if(c == '/') return true;
          if(c == '*') return true;
        }
     }
   }
   return false;
}

static bool IsSolStringStart(char ch)
{
    if (ch == '\'' || ch == '"')  return true;
    return false;
}

static bool IsSolWordStart(char ch)
{
    return (iswordchar(ch) && !IsSolStringStart(ch));
}


static int GetSolStringState(Accessor &styler, int i, int *nextIndex)
{
	char ch = styler.SafeGetCharAt(i);
	char chNext = styler.SafeGetCharAt(i + 1);

        if (ch != '\"' && ch != '\'')
        {
            *nextIndex = i + 1;
            return SCE_SCRIPTOL_DEFAULT;
	}
        // ch is either single or double quotes in string
        // code below seem non-sense but is here for future extensions
	if (ch == chNext && ch == styler.SafeGetCharAt(i + 2))
        {
          *nextIndex = i + 3;
          if(ch == '\"') return SCE_SCRIPTOL_TRIPLE;
          if(ch == '\'') return SCE_SCRIPTOL_TRIPLE;
          return SCE_SCRIPTOL_STRING;
	}
        else
        {
          *nextIndex = i + 1;
          if (ch == '"') return SCE_SCRIPTOL_STRING;
          else           return SCE_SCRIPTOL_STRING;
	}
}


static void ColouriseSolDoc(unsigned int startPos, int length, int initStyle,
                            WordList *keywordlists[], Accessor &styler)
 {

	int lengthDoc = startPos + length;
        char stringType = '\"';

	if (startPos > 0)
        {
            int lineCurrent = styler.GetLine(startPos);
            if (lineCurrent > 0)
            {
              startPos = styler.LineStart(lineCurrent-1);
              if (startPos == 0) initStyle = SCE_SCRIPTOL_DEFAULT;
              else               initStyle = styler.StyleAt(startPos-1);
            }
	}

	styler.StartAt(startPos, 127);

	WordList &keywords = *keywordlists[0];

	int whingeLevel = styler.GetPropertyInt("tab.timmy.whinge.level");
	char prevWord[200];
	prevWord[0] = '\0';
        if (length == 0)  return;

	int state = initStyle & 31;

	int nextIndex = 0;
        char chPrev  = ' ';
        char chPrev2 = ' ';
        char chNext  = styler[startPos];
	styler.StartSegment(startPos);
	bool atStartLine = true;
	int spaceFlags = 0;
	for (int i = startPos; i < lengthDoc; i++)
        {

         if (atStartLine)
         {
         char chBad = static_cast<char>(64);
         char chGood = static_cast<char>(0);
         char chFlags = chGood;

         if (whingeLevel == 1)
         {
             chFlags = (spaceFlags & wsInconsistent) ? chBad : chGood;
         }
         else if (whingeLevel == 2)
         {
             chFlags = (spaceFlags & wsSpaceTab) ? chBad : chGood;
         }
         else if (whingeLevel == 3)
         {
             chFlags = (spaceFlags & wsSpace) ? chBad : chGood;
         }
         else if (whingeLevel == 4)
         {
              chFlags = (spaceFlags & wsTab) ? chBad : chGood;
         }
         styler.SetFlags(chFlags, static_cast<char>(state));
         atStartLine = false;
       }

       char ch = chNext;
       chNext = styler.SafeGetCharAt(i + 1);

       if ((ch == '\r' && chNext != '\n') || (ch == '\n') || (i == lengthDoc))
       {
          if ((state == SCE_SCRIPTOL_DEFAULT) ||
              (state == SCE_SCRIPTOL_TRIPLE) ||
              (state == SCE_SCRIPTOL_COMMENTBLOCK))
          {
              styler.ColourTo(i, state);
          }
          atStartLine = true;
        }

        if (styler.IsLeadByte(ch))
         {
             chNext = styler.SafeGetCharAt(i + 2);
             chPrev  = ' ';<commit_msg>Removing use of style byte indicator in Scriptol lexer.<commit_after>// Scintilla source code edit control
/** @file LexScriptol.cxx
 ** Lexer for Scriptol.
 **/

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdarg.h>
#include <assert.h>
#include <ctype.h>

#include "ILexer.h"
#include "Scintilla.h"
#include "SciLexer.h"

#include "WordList.h"
#include "LexAccessor.h"
#include "Accessor.h"
#include "StyleContext.h"
#include "CharacterSet.h"
#include "LexerModule.h"

#ifdef SCI_NAMESPACE
using namespace Scintilla;
#endif

static void ClassifyWordSol(unsigned int start, unsigned int end, WordList &keywords, Accessor &styler, char *prevWord)
{
    char s[100] = "";
    bool wordIsNumber = isdigit(styler[start]) != 0;
    for (unsigned int i = 0; i < end - start + 1 && i < 30; i++)
     {
           s[i] = styler[start + i];
           s[i + 1] = '\0';
     }
    char chAttr = SCE_SCRIPTOL_IDENTIFIER;
    if (0 == strcmp(prevWord, "class"))       chAttr = SCE_SCRIPTOL_CLASSNAME;
    else if (wordIsNumber)                    chAttr = SCE_SCRIPTOL_NUMBER;
    else if (keywords.InList(s))              chAttr = SCE_SCRIPTOL_KEYWORD;
    else for (unsigned int i = 0; i < end - start + 1; i++)  // test dotted idents
    {
        if (styler[start + i] == '.')
        {
            styler.ColourTo(start + i - 1, chAttr);
            styler.ColourTo(start + i, SCE_SCRIPTOL_OPERATOR);
        }
    }
    styler.ColourTo(end, chAttr);
    strcpy(prevWord, s);
}

static bool IsSolComment(Accessor &styler, int pos, int len)
{
   if(len > 0)
   {
     char c = styler[pos];
     if(c == '`') return true;
     if(len > 1)
     {
        if(c == '/')
        {
          c = styler[pos + 1];
          if(c == '/') return true;
          if(c == '*') return true;
        }
     }
   }
   return false;
}

static bool IsSolStringStart(char ch)
{
    if (ch == '\'' || ch == '"')  return true;
    return false;
}

static bool IsSolWordStart(char ch)
{
    return (iswordchar(ch) && !IsSolStringStart(ch));
}


static int GetSolStringState(Accessor &styler, int i, int *nextIndex)
{
	char ch = styler.SafeGetCharAt(i);
	char chNext = styler.SafeGetCharAt(i + 1);

        if (ch != '\"' && ch != '\'')
        {
            *nextIndex = i + 1;
            return SCE_SCRIPTOL_DEFAULT;
	}
        // ch is either single or double quotes in string
        // code below seem non-sense but is here for future extensions
	if (ch == chNext && ch == styler.SafeGetCharAt(i + 2))
        {
          *nextIndex = i + 3;
          if(ch == '\"') return SCE_SCRIPTOL_TRIPLE;
          if(ch == '\'') return SCE_SCRIPTOL_TRIPLE;
          return SCE_SCRIPTOL_STRING;
	}
        else
        {
          *nextIndex = i + 1;
          if (ch == '"') return SCE_SCRIPTOL_STRING;
          else           return SCE_SCRIPTOL_STRING;
	}
}


static void ColouriseSolDoc(unsigned int startPos, int length, int initStyle,
                            WordList *keywordlists[], Accessor &styler)
 {

	int lengthDoc = startPos + length;
        char stringType = '\"';

	if (startPos > 0)
        {
            int lineCurrent = styler.GetLine(startPos);
            if (lineCurrent > 0)
            {
              startPos = styler.LineStart(lineCurrent-1);
              if (startPos == 0) initStyle = SCE_SCRIPTOL_DEFAULT;
              else               initStyle = styler.StyleAt(startPos-1);
            }
	}

	styler.StartAt(startPos);

	WordList &keywords = *keywordlists[0];

	char prevWord[200];
	prevWord[0] = '\0';
        if (length == 0)  return;

	int state = initStyle & 31;

	int nextIndex = 0;
        char chPrev  = ' ';
        char chPrev2 = ' ';
        char chNext  = styler[startPos];
	styler.StartSegment(startPos);
	for (int i = startPos; i < lengthDoc; i++)
        {

       char ch = chNext;
       chNext = styler.SafeGetCharAt(i + 1);

       if ((ch == '\r' && chNext != '\n') || (ch == '\n') || (i == lengthDoc))
       {
          if ((state == SCE_SCRIPTOL_DEFAULT) ||
              (state == SCE_SCRIPTOL_TRIPLE) ||
              (state == SCE_SCRIPTOL_COMMENTBLOCK))
          {
              styler.ColourTo(i, state);
          }
        }

        if (styler.IsLeadByte(ch))
         {
             chNext = styler.SafeGetCharAt(i + 2);
             chPrev  = ' ';
{'fraction_non_alphanumeric': 0.13473085431370482, 'fraction_numerical': 0.009586010744759297, 'mean_word_length': 2.014925373134328, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5287554', 'n_tokens_mistral': 3385, 'n_tokens_neox': 3219, 'n_words': 774}