<commit_before>// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.lang.properties.xml;

import com.intellij.ide.highlighter.XmlFileType;
import com.intellij.openapi.project.DumbService;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.io.StreamUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.xml.XmlFile;
import com.intellij.util.indexing.*;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.EnumeratorStringDescriptor;
import com.intellij.util.io.IOUtil;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.text.CharArrayUtil;
import com.intellij.util.xml.NanoXmlBuilder;
import com.intellij.util.xml.NanoXmlUtil;
import net.n3.nanoxml.StdXMLReader;
import org.jetbrains.annotations.NotNull;

import java.io.*;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

/**
 * @author <NAME>
 */
public class XmlPropertiesIndex extends FileBasedIndexExtension<XmlPropertiesIndex.Key, String>
  implements DataIndexer<XmlPropertiesIndex.Key, String, FileContent>,
             KeyDescriptor<XmlPropertiesIndex.Key> {

  public final static Key MARKER_KEY = new Key();
  public static final ID<Key,String> NAME = ID.create("xmlProperties");

  private static final String HTTP_JAVA_SUN_COM_DTD_PROPERTIES_DTD = "http://java.sun.com/dtd/properties.dtd";

  @NotNull
  @Override
  public ID<Key, String> getName() {
    return NAME;
  }

  @NotNull
  @Override
  public DataIndexer<Key, String, FileContent> getIndexer() {
    return this;
  }

  @NotNull
  @Override
  public KeyDescriptor<Key> getKeyDescriptor() {
    return this;
  }

  @NotNull
  @Override
  public DataExternalizer<String> getValueExternalizer() {
    return EnumeratorStringDescriptor.INSTANCE;
  }

  @NotNull
  @Override
  public FileBasedIndex.InputFilter getInputFilter() {
    return new DefaultFileTypeSpecificInputFilter(XmlFileType.INSTANCE) {
      @Override
      public boolean acceptInput(@NotNull VirtualFile file) {
        return file.getName().endsWith(".xml");
      }
    };
  }

  @Override
  public boolean dependsOnFileContent() {
    return true;
  }

  @Override
  public int getVersion() {
    return 2;
  }

  @NotNull
  @Override
  public Map<Key, String> map(@NotNull FileContent inputData) {
    CharSequence text = inputData.getContentAsText();
    if (CharArrayUtil.indexOf(text, HTTP_JAVA_SUN_COM_DTD_PROPERTIES_DTD, 0) == -1) {
      return Collections.emptyMap();
    }

    MyIXMLBuilderAdapter builder = parse(text, false);
    Map<Key, String> map = builder.myMap;
    if (builder.accepted) {
      map.put(MARKER_KEY, "");
    }
    return map;
  }

  static boolean isPropertiesFile(XmlFile file) {
    Project project = file.getProject();
    if (!file.isValid()) return false;
    VirtualFile virtualFile = file.getVirtualFile();
    if (virtualFile == null || DumbService.isDumb(project)) {
      CharSequence contents = file.getViewProvider().getContents();
      return CharArrayUtil.indexOf(contents, HTTP_JAVA_SUN_COM_DTD_PROPERTIES_DTD, 0) != -1 &&
          isAccepted(contents);
    }
    return !FileBasedIndex.getInstance().getFileData(NAME, virtualFile, project).isEmpty();
  }

  private static boolean isAccepted(CharSequence bytes) {
    return parse(bytes, true).accepted;
  }

  @NotNull
  private static MyIXMLBuilderAdapter parse(CharSequence text, boolean stopIfAccepted) {
    StdXMLReader reader = new StdXMLReader(CharArrayUtil.readerFromCharSequence(text)) {
      @Override
      public Reader openStream(String publicID, String systemID) throws IOException {
        if (!HTTP_JAVA_SUN_COM_DTD_PROPERTIES_DTD.equals(systemID)) throw new IOException();
        return new StringReader(" ");
      }
    };
    MyIXMLBuilderAdapter builder = new MyIXMLBuilderAdapter(stopIfAccepted);
    NanoXmlUtil.parse(reader, builder);
    return builder;
  }

  @Override
  public void save(@NotNull DataOutput out, Key value) throws IOException {
    out.writeBoolean(value.isMarker);
    if (value.key != null) {
      IOUtil.writeUTF(out, value.key);
    }
  }

  @Override
  public Key read(@NotNull DataInput in) throws IOException {
    boolean isMarker = in.readBoolean();
    return isMarker ? MARKER_KEY : new Key(IOUtil.readUTF(in));
  }

  @Override
  public int getHashCode(Key value) {
    return value.hashCode();
  }

  @Override
  public boolean isEqual(Key val1, Key val2) {
    return val1.isMarker == val2.isMarker && Comparing.equal(val1.key, val2.key);
  }

  public static class Key {
    final boolean isMarker;
    final String key;

    public Key(String key) {
      this.key = key;
      isMarker = false;
    }

    public Key() {
      isMarker = true;
      key = null;
    }

    @Override
    public int hashCode() {
      return isMarker ? 0 : key.hashCode();
    }

    @Override
    public boolean equals(Object o) {
      if (this == o) return true;
      if (o == null || getClass() != o.getClass()) return false;

      Key key1 = (Key)o;

      if (isMarker != key1.isMarker) return false;
      if (!Objects.equals(key, key1.key)) return false;

      return true;
    }
  }

  private static class MyIXMLBuilderAdapter implements NanoXmlBuilder {

    boolean accepted;
    boolean insideEntry;
    String key;
    private final HashMap<Key, String> myMap = new HashMap<>();
    private final boolean myStopIfAccepted;

    MyIXMLBuilderAdapter(boolean stopIfAccepted) {
      myStopIfAccepted = stopIfAccepted;
    }

    @Override
    public void startElement(String name, String nsPrefix, String nsURI, String systemID, int lineNr)
      throws Exception {
      if (!accepted) {
        if ("properties".equals(name)) {
          accepted = true;
        }
        else throw NanoXmlUtil.ParserStoppedXmlException.INSTANCE;
      }
      else {
        insideEntry = "entry".equals(name);
      }
      if (myStopIfAccepted) throw NanoXmlUtil.ParserStoppedXmlException.INSTANCE;
    }

    @Override
    public void addAttribute(String key, String nsPrefix, String nsURI, String value, String type) {
      if (insideEntry && "key".equals(key)) {
        this.key = value;
      }
    }

    @Override
    public void addPCData(Reader reader, String systemID, int lineNr) throws Exception {
      if (insideEntry && key != null) {
        String value = StreamUtil.readTextFrom(reader);
        myMap.put(new Key(key), value);
      }
    }
  }
}
<commit_msg>properties: do not query index while event processing to determine does xml is properties file EA-225320<commit_after>// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.lang.properties.xml;

import com.intellij.ide.highlighter.XmlFileType;
import com.intellij.openapi.project.DumbService;
import com.intellij.openapi.project.NoAccessDuringPsiEvents;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.io.StreamUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.xml.XmlFile;
import com.intellij.util.indexing.*;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.EnumeratorStringDescriptor;
import com.intellij.util.io.IOUtil;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.text.CharArrayUtil;
import com.intellij.util.xml.NanoXmlBuilder;
import com.intellij.util.xml.NanoXmlUtil;
import net.n3.nanoxml.StdXMLReader;
import org.jetbrains.annotations.NotNull;

import java.io.*;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

/**
 * @author <NAME>
 */
public class XmlPropertiesIndex extends FileBasedIndexExtension<XmlPropertiesIndex.Key, String>
  implements DataIndexer<XmlPropertiesIndex.Key, String, FileContent>,
             KeyDescriptor<XmlPropertiesIndex.Key> {

  public final static Key MARKER_KEY = new Key();
  public static final ID<Key,String> NAME = ID.create("xmlProperties");

  private static final String HTTP_JAVA_SUN_COM_DTD_PROPERTIES_DTD = "http://java.sun.com/dtd/properties.dtd";

  @NotNull
  @Override
  public ID<Key, String> getName() {
    return NAME;
  }

  @NotNull
  @Override
  public DataIndexer<Key, String, FileContent> getIndexer() {
    return this;
  }

  @NotNull
  @Override
  public KeyDescriptor<Key> getKeyDescriptor() {
    return this;
  }

  @NotNull
  @Override
  public DataExternalizer<String> getValueExternalizer() {
    return EnumeratorStringDescriptor.INSTANCE;
  }

  @NotNull
  @Override
  public FileBasedIndex.InputFilter getInputFilter() {
    return new DefaultFileTypeSpecificInputFilter(XmlFileType.INSTANCE) {
      @Override
      public boolean acceptInput(@NotNull VirtualFile file) {
        return file.getName().endsWith(".xml");
      }
    };
  }

  @Override
  public boolean dependsOnFileContent() {
    return true;
  }

  @Override
  public int getVersion() {
    return 2;
  }

  @NotNull
  @Override
  public Map<Key, String> map(@NotNull FileContent inputData) {
    CharSequence text = inputData.getContentAsText();
    if (CharArrayUtil.indexOf(text, HTTP_JAVA_SUN_COM_DTD_PROPERTIES_DTD, 0) == -1) {
      return Collections.emptyMap();
    }

    MyIXMLBuilderAdapter builder = parse(text, false);
    Map<Key, String> map = builder.myMap;
    if (builder.accepted) {
      map.put(MARKER_KEY, "");
    }
    return map;
  }

  static boolean isPropertiesFile(XmlFile file) {
    Project project = file.getProject();
    if (!file.isValid()) return false;
    VirtualFile virtualFile = file.getVirtualFile();
    if (virtualFile == null || DumbService.isDumb(project) || NoAccessDuringPsiEvents.isInsideEventProcessing()) {
      CharSequence contents = file.getViewProvider().getContents();
      return CharArrayUtil.indexOf(contents, HTTP_JAVA_SUN_COM_DTD_PROPERTIES_DTD, 0) != -1 &&
          isAccepted(contents);
    }
    return !FileBasedIndex.getInstance().getFileData(NAME, virtualFile, project).isEmpty();
  }

  private static boolean isAccepted(CharSequence bytes) {
    return parse(bytes, true).accepted;
  }

  @NotNull
  private static MyIXMLBuilderAdapter parse(CharSequence text, boolean stopIfAccepted) {
    StdXMLReader reader = new StdXMLReader(CharArrayUtil.readerFromCharSequence(text)) {
      @Override
      public Reader openStream(String publicID, String systemID) throws IOException {
        if (!HTTP_JAVA_SUN_COM_DTD_PROPERTIES_DTD.equals(systemID)) throw new IOException();
        return new StringReader(" ");
      }
    };
    MyIXMLBuilderAdapter builder = new MyIXMLBuilderAdapter(stopIfAccepted);
    NanoXmlUtil.parse(reader, builder);
    return builder;
  }

  @Override
  public void save(@NotNull DataOutput out, Key value) throws IOException {
    out.writeBoolean(value.isMarker);
    if (value.key != null) {
      IOUtil.writeUTF(out, value.key);
    }
  }

  @Override
  public Key read(@NotNull DataInput in) throws IOException {
    boolean isMarker = in.readBoolean();
    return isMarker ? MARKER_KEY : new Key(IOUtil.readUTF(in));
  }

  @Override
  public int getHashCode(Key value) {
    return value.hashCode();
  }

  @Override
  public boolean isEqual(Key val1, Key val2) {
    return val1.isMarker == val2.isMarker && Comparing.equal(val1.key, val2.key);
  }

  public static class Key {
    final boolean isMarker;
    final String key;

    public Key(String key) {
      this.key = key;
      isMarker = false;
    }

    public Key() {
      isMarker = true;
      key = null;
    }

    @Override
    public int hashCode() {
      return isMarker ? 0 : key.hashCode();
    }

    @Override
    public boolean equals(Object o) {
      if (this == o) return true;
      if (o == null || getClass() != o.getClass()) return false;

      Key key1 = (Key)o;

      if (isMarker != key1.isMarker) return false;
      if (!Objects.equals(key, key1.key)) return false;

      return true;
    }
  }

  private static class MyIXMLBuilderAdapter implements NanoXmlBuilder {

    boolean accepted;
    boolean insideEntry;
    String key;
    private final HashMap<Key, String> myMap = new HashMap<>();
    private final boolean myStopIfAccepted;

    MyIXMLBuilderAdapter(boolean stopIfAccepted) {
      myStopIfAccepted = stopIfAccepted;
    }

    @Override
    public void startElement(String name, String nsPrefix, String nsURI, String systemID, int lineNr)
      throws Exception {
      if (!accepted) {
        if ("properties".equals(name)) {
          accepted = true;
        }
        else throw NanoXmlUtil.ParserStoppedXmlException.INSTANCE;
      }
      else {
        insideEntry = "entry".equals(name);
      }
      if (myStopIfAccepted) throw NanoXmlUtil.ParserStoppedXmlException.INSTANCE;
    }

    @Override
    public void addAttribute(String key, String nsPrefix, String nsURI, String value, String type) {
      if (insideEntry && "key".equals(key)) {
        this.key = value;
      }
    }

    @Override
    public void addPCData(Reader reader, String systemID, int lineNr) throws Exception {
      if (insideEntry && key != null) {
        String value = StreamUtil.readTextFrom(reader);
        myMap.put(new Key(key), value);
      }
    }
  }
}

{'fraction_non_alphanumeric': 0.10257935033078124, 'fraction_numerical': 0.004311306028395153, 'mean_word_length': 3.5087131367292224, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 15}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '145747', 'n_tokens_mistral': 4193, 'n_tokens_neox': 4060, 'n_words': 978}