<commit_before>package com.intellij.psi.impl.source.tree;

import com.intellij.lang.ASTFactory;
import com.intellij.lang.ASTNode;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.psi.*;
import com.intellij.psi.impl.GeneratedMarkerVisitor;
import com.intellij.psi.impl.source.PsiTypeElementImpl;
import com.intellij.psi.impl.source.SourceTreeToPsiMap;
import com.intellij.psi.tree.IElementType;
import com.intellij.util.CharTable;
import com.intellij.util.IncorrectOperationException;

public class JavaSharedImplUtil {
  private static final Logger LOG = Logger.getInstance("#com.intellij.psi.impl.source.tree.JavaSharedImplUtil");

  private JavaSharedImplUtil() {
  }

  public static PsiType getType(PsiVariable variable) {
    PsiTypeElement typeElement = variable.getTypeElement();
    PsiIdentifier nameIdentifier = variable.getNameIdentifier();
    return getType(typeElement, nameIdentifier, variable);
  }

  //context == null means no detached type should be created
  public static PsiType getType(PsiTypeElement typeElement, PsiElement anchor, PsiElement context) {
    int arrayCount = 0;
    ASTNode name = SourceTreeToPsiMap.psiElementToTree(anchor);
    for (ASTNode child = name.getTreeNext(); child != null; child = child.getTreeNext()) {
      IElementType i = child.getElementType();
      if (i == JavaTokenType.LBRACKET) {
        arrayCount++;
      }
      else if (i != JavaTokenType.RBRACKET && i != TokenType.WHITE_SPACE &&
               i != JavaTokenType.C_STYLE_COMMENT &&
               i != JavaDocElementType.DOC_COMMENT &&
               i != JavaTokenType.DOC_COMMENT &&
               i != JavaTokenType.END_OF_LINE_COMMENT) {
        break;
      }
    }
    PsiType type;
    if (context != null && typeElement instanceof PsiTypeElementImpl) {
      type = ((PsiTypeElementImpl)typeElement).getDetachedType(context);
    }
    else {
      type = typeElement.getType();
    }

    for (int i = 0; i < arrayCount; i++) {
      type = type.createArrayType();
    }
    return type;
  }

  public static void normalizeBrackets(PsiVariable variable) {
    CompositeElement variableElement = (CompositeElement)SourceTreeToPsiMap.psiElementToTree(variable);
    ASTNode type = variableElement.findChildByRole(ChildRole.TYPE);
    LOG.assertTrue(type.getTreeParent() == variableElement);
    ASTNode name = variableElement.findChildByRole(ChildRole.NAME);

    ASTNode firstBracket = null;
    ASTNode lastBracket = null;
    int arrayCount = 0;
    ASTNode element = name;
    while (true) {
      element = TreeUtil.skipElements(element.getTreeNext(), StdTokenSets.WHITE_SPACE_OR_COMMENT_BIT_SET);
      if (element == null || element.getElementType() != JavaTokenType.LBRACKET) break;
      if (firstBracket == null) firstBracket = element;
      lastBracket = element;
      arrayCount++;

      element = TreeUtil.skipElements(element.getTreeNext(), StdTokenSets.WHITE_SPACE_OR_COMMENT_BIT_SET);
      if (element == null || element.getElementType() != JavaTokenType.RBRACKET) break;
      lastBracket = element;
    }

    if (firstBracket != null) {
      element = firstBracket;
      while (true) {
        ASTNode next = element.getTreeNext();
        variableElement.removeChild(element);
        if (element == lastBracket) break;
        element = next;
      }

      CompositeElement newType = (CompositeElement)type.clone();
      final CharTable treeCharTable = SharedImplUtil.findCharTableByTree(type);
      for (int i = 0; i < arrayCount; i++) {
        CompositeElement newType1 = ASTFactory.composite(JavaElementType.TYPE);
        newType1.rawAddChildren(newType);

        newType1.rawAddChildren(ASTFactory.leaf(JavaTokenType.LBRACKET, "[", 0, 1, treeCharTable));
        newType1.rawAddChildren(ASTFactory.leaf(JavaTokenType.RBRACKET, "]", 0, 1, treeCharTable));
        newType = newType1;
        newType.acceptTree(new GeneratedMarkerVisitor());
      }
      newType.putUserData(CharTable.CHAR_TABLE_KEY, SharedImplUtil.findCharTableByTree(type));
      variableElement.replaceChild(type, newType);
    }
  }

  public static void setInitializer(PsiVariable variable, final PsiExpression initializer) throws IncorrectOperationException {
    PsiExpression oldInitializer = variable.getInitializer();
    if (oldInitializer != null) {
      oldInitializer.delete();
    }
    if (initializer == null) {
      return;
    }
    CompositeElement variableElement = (CompositeElement)variable.getNode();
    ASTNode eq = variableElement.findChildByRole(ChildRole.INITIALIZER_EQ);
    if (eq == null) {
      final CharTable charTable = SharedImplUtil.findCharTableByTree(variableElement);
      eq = Factory.createSingleLeafElement(JavaTokenType.EQ, "=", 0, 1, charTable, variable.getManager());
      variableElement.addInternal((TreeElement)eq, eq, variable.getNameIdentifier().getNode(), Boolean.FALSE);
      eq = variableElement.findChildByRole(ChildRole.INITIALIZER_EQ);
    }
    variable.addAfter(initializer, eq.getPsi());
  }
}
<commit_msg>NPE (IDEADEV-34618)<commit_after>package com.intellij.psi.impl.source.tree;

import com.intellij.lang.ASTFactory;
import com.intellij.lang.ASTNode;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.psi.*;
import com.intellij.psi.impl.GeneratedMarkerVisitor;
import com.intellij.psi.impl.source.PsiTypeElementImpl;
import com.intellij.psi.impl.source.SourceTreeToPsiMap;
import com.intellij.psi.tree.IElementType;
import com.intellij.util.CharTable;
import com.intellij.util.IncorrectOperationException;

public class JavaSharedImplUtil {
  private static final Logger LOG = Logger.getInstance("#com.intellij.psi.impl.source.tree.JavaSharedImplUtil");

  private JavaSharedImplUtil() {
  }

  public static PsiType getType(PsiVariable variable) {
    PsiTypeElement typeElement = variable.getTypeElement();
    PsiIdentifier nameIdentifier = variable.getNameIdentifier();
    return getType(typeElement, nameIdentifier, variable);
  }

  //context == null means no detached type should be created
  public static PsiType getType(PsiTypeElement typeElement, PsiElement anchor, PsiElement context) {
    int arrayCount = 0;
    ASTNode name = SourceTreeToPsiMap.psiElementToTree(anchor);
    for (ASTNode child = name.getTreeNext(); child != null; child = child.getTreeNext()) {
      IElementType i = child.getElementType();
      if (i == JavaTokenType.LBRACKET) {
        arrayCount++;
      }
      else if (i != JavaTokenType.RBRACKET && i != TokenType.WHITE_SPACE &&
               i != JavaTokenType.C_STYLE_COMMENT &&
               i != JavaDocElementType.DOC_COMMENT &&
               i != JavaTokenType.DOC_COMMENT &&
               i != JavaTokenType.END_OF_LINE_COMMENT) {
        break;
      }
    }
    PsiType type;
    if (context != null && typeElement instanceof PsiTypeElementImpl) {
      type = ((PsiTypeElementImpl)typeElement).getDetachedType(context);
    }
    else {
      type = typeElement.getType();
    }

    for (int i = 0; i < arrayCount; i++) {
      type = type.createArrayType();
    }
    return type;
  }

  public static void normalizeBrackets(PsiVariable variable) {
    CompositeElement variableElement = (CompositeElement)SourceTreeToPsiMap.psiElementToTree(variable);
    ASTNode type = variableElement.findChildByRole(ChildRole.TYPE);
    LOG.assertTrue(type.getTreeParent() == variableElement);
    ASTNode name = variableElement.findChildByRole(ChildRole.NAME);

    ASTNode firstBracket = null;
    ASTNode lastBracket = null;
    int arrayCount = 0;
    ASTNode element = name;
    while (element != null) {
      element = TreeUtil.skipElements(element.getTreeNext(), StdTokenSets.WHITE_SPACE_OR_COMMENT_BIT_SET);
      if (element == null || element.getElementType() != JavaTokenType.LBRACKET) break;
      if (firstBracket == null) firstBracket = element;
      lastBracket = element;
      arrayCount++;

      element = TreeUtil.skipElements(element.getTreeNext(), StdTokenSets.WHITE_SPACE_OR_COMMENT_BIT_SET);
      if (element == null || element.getElementType() != JavaTokenType.RBRACKET) break;
      lastBracket = element;
    }

    if (firstBracket != null) {
      element = firstBracket;
      while (true) {
        ASTNode next = element.getTreeNext();
        variableElement.removeChild(element);
        if (element == lastBracket) break;
        element = next;
      }

      CompositeElement newType = (CompositeElement)type.clone();
      final CharTable treeCharTable = SharedImplUtil.findCharTableByTree(type);
      for (int i = 0; i < arrayCount; i++) {
        CompositeElement newType1 = ASTFactory.composite(JavaElementType.TYPE);
        newType1.rawAddChildren(newType);

        newType1.rawAddChildren(ASTFactory.leaf(JavaTokenType.LBRACKET, "[", 0, 1, treeCharTable));
        newType1.rawAddChildren(ASTFactory.leaf(JavaTokenType.RBRACKET, "]", 0, 1, treeCharTable));
        newType = newType1;
        newType.acceptTree(new GeneratedMarkerVisitor());
      }
      newType.putUserData(CharTable.CHAR_TABLE_KEY, SharedImplUtil.findCharTableByTree(type));
      variableElement.replaceChild(type, newType);
    }
  }

  public static void setInitializer(PsiVariable variable, final PsiExpression initializer) throws IncorrectOperationException {
    PsiExpression oldInitializer = variable.getInitializer();
    if (oldInitializer != null) {
      oldInitializer.delete();
    }
    if (initializer == null) {
      return;
    }
    CompositeElement variableElement = (CompositeElement)variable.getNode();
    ASTNode eq = variableElement.findChildByRole(ChildRole.INITIALIZER_EQ);
    if (eq == null) {
      final CharTable charTable = SharedImplUtil.findCharTableByTree(variableElement);
      eq = Factory.createSingleLeafElement(JavaTokenType.EQ, "=", 0, 1, charTable, variable.getManager());
      variableElement.addInternal((TreeElement)eq, eq, variable.getNameIdentifier().getNode(), Boolean.FALSE);
      eq = variableElement.findChildByRole(ChildRole.INITIALIZER_EQ);
    }
    variable.addAfter(initializer, eq.getPsi());
  }
}

{'fraction_non_alphanumeric': 0.10598986788516937, 'fraction_numerical': 0.0034767060693354523, 'mean_word_length': 3.833413346135382, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6614171', 'n_tokens_mistral': 3031, 'n_tokens_neox': 2956, 'n_words': 568}