<commit_before>// Scintilla source code edit control
/** @file LexVHDL.cxx
 ** Lexer for VHDL
 ** Written by <NAME>,
 ** Based on:
 **  - The Verilog Lexer by <NAME>
 **  - The Fortran Lexer by <NAME>
 **  - The C++ lexer by <NAME>
 **/
// Copyright 1998-2002 by <NAME> <<EMAIL>>
// The License.txt file describes the conditions under which this software may be distributed.

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdarg.h>
#include <assert.h>
#include <ctype.h>

#include "ILexer.h"
#include "Scintilla.h"
#include "SciLexer.h"

#include "WordList.h"
#include "LexAccessor.h"
#include "Accessor.h"
#include "StyleContext.h"
#include "CharacterSet.h"
#include "LexerModule.h"

#ifdef SCI_NAMESPACE
using namespace Scintilla;
#endif

static void ColouriseVHDLDoc(
  unsigned int startPos,
  int length,
  int initStyle,
  WordList *keywordlists[],
  Accessor &styler);


/***************************************/
static inline bool IsAWordChar(const int ch) {
  return (ch < 0x80) && (isalnum(ch) || ch == '.' || ch == '_' );
}

/***************************************/
static inline bool IsAWordStart(const int ch) {
  return (ch < 0x80) && (isalnum(ch) || ch == '_');
}

/***************************************/
static inline bool IsABlank(unsigned int ch) {
    return (ch == ' ') || (ch == 0x09) || (ch == 0x0b) ;
}

/***************************************/
static void ColouriseVHDLDoc(
  unsigned int startPos,
  int length,
  int initStyle,
  WordList *keywordlists[],
  Accessor &styler)
{
  WordList &Keywords   = *keywordlists[0];
  WordList &Operators  = *keywordlists[1];
  WordList &Attributes = *keywordlists[2];
  WordList &Functions  = *keywordlists[3];
  WordList &Packages   = *keywordlists[4];
  WordList &Types      = *keywordlists[5];
  WordList &User       = *keywordlists[6];

  StyleContext sc(startPos, length, initStyle, styler);

  for (; sc.More(); sc.Forward())
  {

    // Determine if the current state should terminate.
    if (sc.state == SCE_VHDL_OPERATOR) {
      sc.SetState(SCE_VHDL_DEFAULT);
    } else if (sc.state == SCE_VHDL_NUMBER) {
      if (!IsAWordChar(sc.ch) && (sc.ch != '#')) {
        sc.SetState(SCE_VHDL_DEFAULT);
      }
    } else if (sc.state == SCE_VHDL_IDENTIFIER) {
      if (!IsAWordChar(sc.ch) || (sc.ch == '.')) {
        char s[100];
        sc.GetCurrentLowered(s, sizeof(s));
        if (Keywords.InList(s)) {
          sc.ChangeState(SCE_VHDL_KEYWORD);
        } else if (Operators.InList(s)) {
          sc.ChangeState(SCE_VHDL_STDOPERATOR);
        } else if (Attributes.InList(s)) {
          sc.ChangeState(SCE_VHDL_ATTRIBUTE);
        } else if (Functions.InList(s)) {
          sc.ChangeState(SCE_VHDL_STDFUNCTION);
        } else if (Packages.InList(s)) {
          sc.ChangeState(SCE_VHDL_STDPACKAGE);
        } else if (Types.InList(s)) {
          sc.ChangeState(SCE_VHDL_STDTYPE);
        } else if (User.InList(s)) {
          sc.ChangeState(SCE_VHDL_USERWORD);
        }
        sc.SetState(SCE_VHDL_DEFAULT);
      }
    } else if (sc.state == SCE_VHDL_COMMENT || sc.state == SCE_VHDL_COMMENTLINEBANG) {
      if (sc.atLineEnd) {
        sc.SetState(SCE_VHDL_DEFAULT);
      }
    } else if (sc.state == SCE_VHDL_STRING) {
      if (sc.ch == '\\') {
        if (sc.chNext == '\"' || sc.chNext == '\'' || sc.chNext == '\\') {
          sc.Forward();
        }
      } else if (sc.ch == '\"') {
        sc.ForwardSetState(SCE_VHDL_DEFAULT);
      } else if (sc.atLineEnd) {
        sc.ChangeState(SCE_VHDL_STRINGEOL);
        sc.ForwardSetState(SCE_VHDL_DEFAULT);
      }
    } else if (sc.state == SCE_VHDL_BLOCK_COMMENT){
      if(sc.ch == '*' && sc.chNext == '/'){
        sc.Forward();
        sc.ForwardSetState(SCE_VHDL_DEFAULT);
      }
    }

    // Determine if a new state should be entered.
    if (sc.state == SCE_VHDL_DEFAULT) {
      if (IsADigit(sc.ch) || (sc.ch == '.' && IsADigit(sc.chNext))) {
        sc.SetState(SCE_VHDL_NUMBER);
      } else if (IsAWordStart(sc.ch)) {
        sc.SetState(SCE_VHDL_IDENTIFIER);
      } else if (sc.Match('-', '-')) {
        if (sc.Match("--!"))  // Nice to have a different comment style
          sc.SetState(SCE_VHDL_COMMENTLINEBANG);
        else
          sc.SetState(SCE_VHDL_COMMENT);
      } else if (sc.Match('/', '*')){
        sc.SetState(SCE_VHDL_BLOCK_COMMENT);
      } else if (sc.ch == '\"') {
        sc.SetState(SCE_VHDL_STRING);
      } else if (isoperator(static_cast<char>(sc.ch))) {
        sc.SetState(SCE_VHDL_OPERATOR);
      }
    }
  }
  sc.Complete();
}
//=============================================================================
static bool IsCommentLine(int line, Accessor &styler) {
	int pos = styler.LineStart(line);
	int eol_pos = styler.LineStart(line + 1) - 1;
	for (int i = pos; i < eol_pos; i++) {
		char ch = styler[i];
		char chNext = styler[i+1];
		if ((ch == '-') && (chNext == '-'))
			return true;
		else if (ch != ' ' && ch != '\t')
			return false;
	}
	return false;
}
static bool IsCommentBlockStart(int line, Accessor &styler)
{
    int pos = styler.LineStart(line);<commit_msg>add code to support extended identifiers<commit_after>// Scintilla source code edit control
/** @file LexVHDL.cxx
 ** Lexer for VHDL
 ** Written by <NAME>,
 ** Based on:
 **  - The Verilog Lexer by <NAME>
 **  - The Fortran Lexer by <NAME>
 **  - The C++ lexer by <NAME>
 **/
// Copyright 1998-2002 by <NAME> <<EMAIL>>
// The License.txt file describes the conditions under which this software may be distributed.

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdarg.h>
#include <assert.h>
#include <ctype.h>

#include "ILexer.h"
#include "Scintilla.h"
#include "SciLexer.h"

#include "WordList.h"
#include "LexAccessor.h"
#include "Accessor.h"
#include "StyleContext.h"
#include "CharacterSet.h"
#include "LexerModule.h"

#ifdef SCI_NAMESPACE
using namespace Scintilla;
#endif

static void ColouriseVHDLDoc(
  unsigned int startPos,
  int length,
  int initStyle,
  WordList *keywordlists[],
  Accessor &styler);


/***************************************/
static inline bool IsAWordChar(const int ch) {
  return (ch < 0x80) && (isalnum(ch) || ch == '.' || ch == '_' );
}

/***************************************/
static inline bool IsAWordStart(const int ch) {
  return (ch < 0x80) && (isalnum(ch) || ch == '_');
}

/***************************************/
static inline bool IsABlank(unsigned int ch) {
    return (ch == ' ') || (ch == 0x09) || (ch == 0x0b) ;
}

/***************************************/
static void ColouriseVHDLDoc(
  unsigned int startPos,
  int length,
  int initStyle,
  WordList *keywordlists[],
  Accessor &styler)
{
  WordList &Keywords   = *keywordlists[0];
  WordList &Operators  = *keywordlists[1];
  WordList &Attributes = *keywordlists[2];
  WordList &Functions  = *keywordlists[3];
  WordList &Packages   = *keywordlists[4];
  WordList &Types      = *keywordlists[5];
  WordList &User       = *keywordlists[6];

  StyleContext sc(startPos, length, initStyle, styler);
  bool isExtendedId = false;    // true when parsing an extended identifier

  for (; sc.More(); sc.Forward())
  {

    // Determine if the current state should terminate.
    if (sc.state == SCE_VHDL_OPERATOR) {
      sc.SetState(SCE_VHDL_DEFAULT);
    } else if (sc.state == SCE_VHDL_NUMBER) {
      if (!IsAWordChar(sc.ch) && (sc.ch != '#')) {
        sc.SetState(SCE_VHDL_DEFAULT);
      }
    } else if (sc.state == SCE_VHDL_IDENTIFIER) {
      if (!isExtendedId && (!IsAWordChar(sc.ch) || (sc.ch == '.'))) {
        char s[100];
        sc.GetCurrentLowered(s, sizeof(s));
        if (Keywords.InList(s)) {
          sc.ChangeState(SCE_VHDL_KEYWORD);
        } else if (Operators.InList(s)) {
          sc.ChangeState(SCE_VHDL_STDOPERATOR);
        } else if (Attributes.InList(s)) {
          sc.ChangeState(SCE_VHDL_ATTRIBUTE);
        } else if (Functions.InList(s)) {
          sc.ChangeState(SCE_VHDL_STDFUNCTION);
        } else if (Packages.InList(s)) {
          sc.ChangeState(SCE_VHDL_STDPACKAGE);
        } else if (Types.InList(s)) {
          sc.ChangeState(SCE_VHDL_STDTYPE);
        } else if (User.InList(s)) {
          sc.ChangeState(SCE_VHDL_USERWORD);
        }
        sc.SetState(SCE_VHDL_DEFAULT);
      } else if (isExtendedId && ((sc.ch == '\\') || sc.atLineEnd)) {
        // extended identifiers are terminated by backslash, check for end of line in case we have invalid syntax
        isExtendedId = false;
        sc.ForwardSetState(SCE_VHDL_DEFAULT);
      }
    } else if (sc.state == SCE_VHDL_COMMENT || sc.state == SCE_VHDL_COMMENTLINEBANG) {
      if (sc.atLineEnd) {
        sc.SetState(SCE_VHDL_DEFAULT);
      }
    } else if (sc.state == SCE_VHDL_STRING) {
      if (sc.ch == '\\') {
        if (sc.chNext == '\"' || sc.chNext == '\'' || sc.chNext == '\\') {
          sc.Forward();
        }
      } else if (sc.ch == '\"') {
        sc.ForwardSetState(SCE_VHDL_DEFAULT);
      } else if (sc.atLineEnd) {
        sc.ChangeState(SCE_VHDL_STRINGEOL);
        sc.ForwardSetState(SCE_VHDL_DEFAULT);
      }
    } else if (sc.state == SCE_VHDL_BLOCK_COMMENT){
      if(sc.ch == '*' && sc.chNext == '/'){
        sc.Forward();
        sc.ForwardSetState(SCE_VHDL_DEFAULT);
      }
    }

    // Determine if a new state should be entered.
    if (sc.state == SCE_VHDL_DEFAULT) {
      if (IsADigit(sc.ch) || (sc.ch == '.' && IsADigit(sc.chNext))) {
        sc.SetState(SCE_VHDL_NUMBER);
      } else if (IsAWordStart(sc.ch)) {
        sc.SetState(SCE_VHDL_IDENTIFIER);
      } else if (sc.Match('-', '-')) {
        if (sc.Match("--!"))  // Nice to have a different comment style
          sc.SetState(SCE_VHDL_COMMENTLINEBANG);
        else
          sc.SetState(SCE_VHDL_COMMENT);
      } else if (sc.Match('/', '*')){
        sc.SetState(SCE_VHDL_BLOCK_COMMENT);
      } else if (sc.ch == '\"') {
        sc.SetState(SCE_VHDL_STRING);
      } else if (sc.ch == '\\') {
        isExtendedId = true;
        sc.SetState(SCE_VHDL_IDENTIFIER);
      } else if (isoperator(static_cast<char>(sc.ch))) {
        sc.SetState(SCE_VHDL_OPERATOR);
      }
    }
  }
  sc.Complete();
}
//=============================================================================
static bool IsCommentLine(int line, Accessor &styler) {
	int pos = styler.LineStart(line);
	int eol_pos = styler.LineStart(line + 1) - 1;
	for (int i = pos; i < eol_pos; i++) {
		char ch = styler[i];
		char chNext = styler[i+1];
		if ((ch == '-') && (chNext == '-'))
			return true;
		else if (ch != ' ' && ch != '\t')
			return false;
	}
	return false;
}
static bool IsCommentBlockStart(int line, Accessor &styler)
{
    int pos = styler.LineStart(line);
{'fraction_non_alphanumeric': 0.19595164464436324, 'fraction_numerical': 0.0059975634898322555, 'mean_word_length': 2.959925788497217, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1258913', 'n_tokens_mistral': 3954, 'n_tokens_neox': 3768, 'n_words': 785}