<commit_before>package com.intellij.lexer;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.psi.jsp.el.ELTokenType;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.xml.XmlTokenType;

public class HtmlHighlightingLexer extends BaseHtmlLexer {
  private static final Logger LOG = Logger.getInstance("#com.intellij.lexer.HtmlHighlightingLexer");

  private static final int EMBEDDED_LEXER_ON = 0x1 << BASE_STATE_SHIFT;
  private static final int EMBEDDED_LEXER_STATE_SHIFT = BASE_STATE_SHIFT + 1;

  private Lexer embeddedLexer;
  private Lexer styleLexer;
  private Lexer scriptLexer;
  private Lexer elLexer;
  private boolean hasNoEmbeddments;
  private static FileType ourStyleFileType;
  private static FileType ourScriptFileType;

  class XmlEmbeddmentHandler implements TokenHandler {
    public void handleElement(Lexer lexer) {
      if (!hasSeenStyle() && !hasSeenScript() || hasNoEmbeddments) return;
      final IElementType tokenType = lexer.getTokenType();

      if ((tokenType==XmlTokenType.XML_ATTRIBUTE_VALUE_TOKEN && hasSeenAttribute()) ||
          (tokenType==XmlTokenType.XML_DATA_CHARACTERS && hasSeenTag()) ||
          tokenType==XmlTokenType.XML_COMMENT_CHARACTERS && hasSeenTag()
          ) {
        setEmbeddedLexer();
        if (embeddedLexer!=null) {
          embeddedLexer.start(getBuffer(),HtmlHighlightingLexer.super.getTokenStart(),skipToTheEndOfTheEmbeddment());
        }
      }
    }
  }

  class ElEmbeddmentHandler implements TokenHandler {
    public void handleElement(Lexer lexer) {
      setEmbeddedLexer();
      embeddedLexer.start(getBuffer(),HtmlHighlightingLexer.super.getTokenStart(),HtmlHighlightingLexer.super.getTokenEnd());
    }
  }

  public HtmlHighlightingLexer() {
    this(new MergingLexerAdapter(new FlexAdapter(new _HtmlLexer()),TOKENS_TO_MERGE),true, false);
  }

  protected HtmlHighlightingLexer(Lexer lexer, boolean caseInsensitive, boolean withEl) {
    super(lexer,caseInsensitive);

    XmlEmbeddmentHandler value = new XmlEmbeddmentHandler();
    registerHandler(XmlTokenType.XML_ATTRIBUTE_VALUE_TOKEN,value);
    registerHandler(XmlTokenType.XML_DATA_CHARACTERS,value);
    registerHandler(XmlTokenType.XML_COMMENT_CHARACTERS,value);

    if (withEl) {
      activateElSupport();
    }
  }

  public void activateElSupport() {
    registerHandler(ELTokenType.JSP_EL_CONTENT, new ElEmbeddmentHandler());
    Lexer baseLexer = getBaseLexer();

    if (baseLexer instanceof MergingLexerAdapter) {
      baseLexer = ((MergingLexerAdapter)baseLexer).getOriginal();
      if (baseLexer instanceof FlexAdapter) {
        final FlexLexer flex = ((FlexAdapter)baseLexer).getFlex();

        if (flex instanceof ELHostLexer) {
          ((ELHostLexer)flex).setElTypes(ELTokenType.JSP_EL_CONTENT,ELTokenType.JSP_EL_CONTENT);
        }
      }
    }
  }

  public void start(char[] buffer, int startOffset, int endOffset, int initialState) {
    super.start(buffer, startOffset, endOffset, initialState);

    if ((initialState & EMBEDDED_LEXER_ON)!=0) {
      int state = initialState >> EMBEDDED_LEXER_STATE_SHIFT;
      setEmbeddedLexer();
      LOG.assertTrue(embeddedLexer!=null);
      embeddedLexer.start(buffer,startOffset,skipToTheEndOfTheEmbeddment(),state);
    } else {
      embeddedLexer = null;
    }
  }

  private void setEmbeddedLexer() {
    Lexer newLexer = null;
    if (hasSeenStyle()) {
      if (styleLexer==null) {
        styleLexer = (ourStyleFileType!=null)? ourStyleFileType.getHighlighter(null).getHighlightingLexer():null;
      }

      newLexer = styleLexer;
    } else if (hasSeenScript()) {
      if (scriptLexer==null) {
        scriptLexer = (ourScriptFileType!=null)? ourScriptFileType.getHighlighter(null).getHighlightingLexer():null;
      }
      newLexer = scriptLexer;
    } else if (super.getTokenType() == ELTokenType.JSP_EL_CONTENT) {
      if (elLexer==null) elLexer = new ELLexer();
      newLexer = elLexer;
    }

    if (newLexer!=null) {
      embeddedLexer = newLexer;
    }
  }

  public void advance() {
    if (embeddedLexer!=null) {
      embeddedLexer.advance();
      if (embeddedLexer.getTokenType()==null) {
        embeddedLexer=null;
      }
    }

    if (embeddedLexer==null) {
      super.advance();
    }
  }

  protected boolean isValidAttributeValueTokenType(final IElementType tokenType) {
    return super.isValidAttributeValueTokenType(tokenType) ||
      tokenType == ELTokenType.JSP_EL_CONTENT;
  }

  public IElementType getTokenType() {
    if (embeddedLexer!=null) {
      return embeddedLexer.getTokenType();
    } else {
      IElementType tokenType = super.getTokenType();

      // TODO: fix no DOCTYPE highlighting
      if (tokenType == null) return tokenType;

      if (tokenType==XmlTokenType.XML_NAME) {
        // we need to convert single xml_name for tag name and attribute name into to separate
        // lex types for the highlighting!
        final int state = getState() & BASE_STATE_MASK;

        if (isHtmlTagState(state)) {
          tokenType = XmlTokenType.XML_TAG_NAME;
        }
      }
      else if (tokenType == XmlTokenType.XML_WHITE_SPACE) {
        tokenType = (getState()!=0)?XmlTokenType.TAG_WHITE_SPACE:XmlTokenType.XML_REAL_WHITE_SPACE;<commit_msg>fixed issue with tag whitespace appearing before style or script<commit_after>package com.intellij.lexer;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.psi.jsp.el.ELTokenType;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.xml.XmlTokenType;

public class HtmlHighlightingLexer extends BaseHtmlLexer {
  private static final Logger LOG = Logger.getInstance("#com.intellij.lexer.HtmlHighlightingLexer");

  private static final int EMBEDDED_LEXER_ON = 0x1 << BASE_STATE_SHIFT;
  private static final int EMBEDDED_LEXER_STATE_SHIFT = BASE_STATE_SHIFT + 1;

  private Lexer embeddedLexer;
  private Lexer styleLexer;
  private Lexer scriptLexer;
  private Lexer elLexer;
  private boolean hasNoEmbeddments;
  private static FileType ourStyleFileType;
  private static FileType ourScriptFileType;

  class XmlEmbeddmentHandler implements TokenHandler {
    public void handleElement(Lexer lexer) {
      if (!hasSeenStyle() && !hasSeenScript() || hasNoEmbeddments) return;
      final IElementType tokenType = lexer.getTokenType();

      if ((tokenType==XmlTokenType.XML_ATTRIBUTE_VALUE_TOKEN && hasSeenAttribute()) ||
          (tokenType==XmlTokenType.XML_DATA_CHARACTERS && hasSeenTag()) ||
          tokenType==XmlTokenType.XML_COMMENT_CHARACTERS && hasSeenTag()
          ) {
        setEmbeddedLexer();
        if (embeddedLexer!=null) {
          embeddedLexer.start(getBuffer(),HtmlHighlightingLexer.super.getTokenStart(),skipToTheEndOfTheEmbeddment());
        }
      }
    }
  }

  class ElEmbeddmentHandler implements TokenHandler {
    public void handleElement(Lexer lexer) {
      setEmbeddedLexer();
      embeddedLexer.start(getBuffer(),HtmlHighlightingLexer.super.getTokenStart(),HtmlHighlightingLexer.super.getTokenEnd());
    }
  }

  public HtmlHighlightingLexer() {
    this(new MergingLexerAdapter(new FlexAdapter(new _HtmlLexer()),TOKENS_TO_MERGE),true, false);
  }

  protected HtmlHighlightingLexer(Lexer lexer, boolean caseInsensitive, boolean withEl) {
    super(lexer,caseInsensitive);

    XmlEmbeddmentHandler value = new XmlEmbeddmentHandler();
    registerHandler(XmlTokenType.XML_ATTRIBUTE_VALUE_TOKEN,value);
    registerHandler(XmlTokenType.XML_DATA_CHARACTERS,value);
    registerHandler(XmlTokenType.XML_COMMENT_CHARACTERS,value);

    if (withEl) {
      activateElSupport();
    }
  }

  public void activateElSupport() {
    registerHandler(ELTokenType.JSP_EL_CONTENT, new ElEmbeddmentHandler());
    Lexer baseLexer = getBaseLexer();

    if (baseLexer instanceof MergingLexerAdapter) {
      baseLexer = ((MergingLexerAdapter)baseLexer).getOriginal();
      if (baseLexer instanceof FlexAdapter) {
        final FlexLexer flex = ((FlexAdapter)baseLexer).getFlex();

        if (flex instanceof ELHostLexer) {
          ((ELHostLexer)flex).setElTypes(ELTokenType.JSP_EL_CONTENT,ELTokenType.JSP_EL_CONTENT);
        }
      }
    }
  }

  public void start(char[] buffer, int startOffset, int endOffset, int initialState) {
    super.start(buffer, startOffset, endOffset, initialState);

    if ((initialState & EMBEDDED_LEXER_ON)!=0) {
      int state = initialState >> EMBEDDED_LEXER_STATE_SHIFT;
      setEmbeddedLexer();
      LOG.assertTrue(embeddedLexer!=null);
      embeddedLexer.start(buffer,startOffset,skipToTheEndOfTheEmbeddment(),state);
    } else {
      embeddedLexer = null;
    }
  }

  private void setEmbeddedLexer() {
    Lexer newLexer = null;
    if (hasSeenStyle()) {
      if (styleLexer==null) {
        styleLexer = (ourStyleFileType!=null)? ourStyleFileType.getHighlighter(null).getHighlightingLexer():null;
      }

      newLexer = styleLexer;
    } else if (hasSeenScript()) {
      if (scriptLexer==null) {
        scriptLexer = (ourScriptFileType!=null)? ourScriptFileType.getHighlighter(null).getHighlightingLexer():null;
      }
      newLexer = scriptLexer;
    } else if (super.getTokenType() == ELTokenType.JSP_EL_CONTENT) {
      if (elLexer==null) elLexer = new ELLexer();
      newLexer = elLexer;
    }

    if (newLexer!=null) {
      embeddedLexer = newLexer;
    }
  }

  public void advance() {
    if (embeddedLexer!=null) {
      embeddedLexer.advance();
      if (embeddedLexer.getTokenType()==null) {
        embeddedLexer=null;
      }
    }

    if (embeddedLexer==null) {
      super.advance();
    }
  }

  protected boolean isValidAttributeValueTokenType(final IElementType tokenType) {
    return super.isValidAttributeValueTokenType(tokenType) ||
      tokenType == ELTokenType.JSP_EL_CONTENT;
  }

  public IElementType getTokenType() {
    if (embeddedLexer!=null) {
      return embeddedLexer.getTokenType();
    } else {
      IElementType tokenType = super.getTokenType();

      // TODO: fix no DOCTYPE highlighting
      if (tokenType == null) return tokenType;

      if (tokenType==XmlTokenType.XML_NAME) {
        // we need to convert single xml_name for tag name and attribute name into to separate
        // lex types for the highlighting!
        final int state = getState() & BASE_STATE_MASK;

        if (isHtmlTagState(state)) {
          tokenType = XmlTokenType.XML_TAG_NAME;
        }
      }
      else if (tokenType == XmlTokenType.XML_WHITE_SPACE) {
        if (hasSeenTag() && (hasSeenStyle() || hasSeenScript())) {
          int a = 1; 
        } else {
          tokenType = (getState()!=0)?XmlTokenType.TAG_WHITE_SPACE:XmlTokenType.XML_REAL_WHITE_SPACE;
        }
{'fraction_non_alphanumeric': 0.10034313270889363, 'fraction_numerical': 0.0010201242696837615, 'mean_word_length': 3.7843833185448092, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5931269', 'n_tokens_mistral': 3639, 'n_tokens_neox': 3371, 'n_words': 610}