<commit_before>// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.util.indexing;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.io.ByteArraySequence;
import com.intellij.psi.PsiDocumentManager;
import com.intellij.psi.PsiFile;
import com.intellij.util.CompressionUtil;
import com.intellij.util.ExceptionUtil;
import com.intellij.util.SystemProperties;
import com.intellij.util.indexing.impl.DebugAssertions;
import com.intellij.util.indexing.impl.MapReduceIndex;
import com.intellij.util.indexing.impl.forward.AbstractForwardIndexAccessor;
import com.intellij.util.indexing.impl.forward.PersistentMapBasedForwardIndex;
import com.intellij.util.io.*;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.*;
import java.nio.charset.Charset;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Stream;

class SnapshotInputMappings<Key, Value, Input> implements SnapshotInputMappingIndex<Key, Value, Input> {
  private static final Logger LOG = Logger.getInstance(SnapshotInputMappings.class);
  private static final boolean doReadSavedPersistentData = SystemProperties.getBooleanProperty("idea.read.saved.persistent.index", true);

  private final ID<Key, Value> myIndexId;
  private final InputMapExternalizer<Key, Value> myMapExternalizer;
  private final DataIndexer<Key, Value, Input> myIndexer;
  private final PersistentMapBasedForwardIndex myContents;
  private volatile PersistentHashMap<Integer, String> myIndexingTrace;

  private final HashIdForwardIndexAccessor<Key, Value, Input> myHashIdForwardIndexAccessor;

  private final boolean myIsPsiBackedIndex;

  SnapshotInputMappings(IndexExtension<Key, Value, Input> indexExtension) throws IOException {
    myIndexId = (ID<Key, Value>)indexExtension.getName();
    myIsPsiBackedIndex = indexExtension instanceof PsiDependentIndex;
    myMapExternalizer = new InputMapExternalizer<>(indexExtension);
    myIndexer = indexExtension.getIndexer();
    myContents = createContentsIndex();
    myHashIdForwardIndexAccessor = new HashIdForwardIndexAccessor<>(this);
    myIndexingTrace = DebugAssertions.EXTRA_SANITY_CHECKS ? createIndexingTrace() : null;
  }

  HashIdForwardIndexAccessor<Key, Value, Input> getForwardIndexAccessor() {
    return myHashIdForwardIndexAccessor;
  }

  File getInputIndexStorageFile() {
    return new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId");
  }

  @NotNull
  @Override
  public Map<Key, Value> readData(int hashId) throws IOException {
    ByteArraySequence byteSequence = readContents(hashId);
    if (byteSequence != null) {
      return AbstractForwardIndexAccessor.deserializeFromByteSeq(byteSequence, myMapExternalizer);
    }
    return Collections.emptyMap();
  }

  @Nullable
  @Override
  public Map<Key, Value> readData(@NotNull Input content) throws IOException {
    Map<Key, Value> data = null;

    FileContent fileContent = (FileContent)content;
    if (doReadSavedPersistentData) {
      if (myContents == null || !myContents.isBusyReading() || DebugAssertions.EXTRA_SANITY_CHECKS) { // avoid blocking read, we can calculate index value
        int hashId = getHashOfContent(fileContent);
        ByteArraySequence bytes = readContents(hashId);

        if (bytes != null) {
          data = AbstractForwardIndexAccessor.deserializeFromByteSeq(bytes, myMapExternalizer);
          if (DebugAssertions.EXTRA_SANITY_CHECKS) {
            Map<Key, Value> contentData = myIndexer.map(content);
            boolean sameValueForSavedIndexedResultAndCurrentOne = contentData.equals(data);
            if (!sameValueForSavedIndexedResultAndCurrentOne) {
              DebugAssertions.error(
                "Unexpected difference in indexing of %s by index %s, file type %s, charset %s\ndiff %s\nprevious indexed info %s",
                fileContent.getFile(),
                myIndexId,
                fileContent.getFileType().getName(),
                ((FileContentImpl)fileContent).getCharset(),
                buildDiff(data, contentData),
                myIndexingTrace.get(hashId)
              );
            }
          }
        }
      }
    }
    return data;
  }

  @Override
  public void putData(@Nullable Input content, @NotNull Map<Key, Value> data) throws IOException {
    FileContent fileContent = (FileContent)content;
    int hashId = getHashOfContent(fileContent);
    boolean saved = savePersistentData(data, hashId);
    if (DebugAssertions.EXTRA_SANITY_CHECKS) {
      if (saved) {
        try {
          myIndexingTrace.put(hashId, ((FileContentImpl)fileContent).getCharset() +
                                      "," +
                                      fileContent.getFileType().getName() +
                                      "," +
                                      fileContent.getFile().getPath() +
                                      "," +
                                      ExceptionUtil.getThrowableText(new Throwable()));
        }
        catch (IOException ex) {
          LOG.error(ex);
        }
      }
    }
  }

  @Override<commit_msg>refactor SnapshotInputMappingIndex: reduce type casts to FileContent<commit_after>// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.util.indexing;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.editor.Document;
import com.intellij.openapi.fileEditor.FileDocumentManager;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.io.ByteArraySequence;
import com.intellij.psi.PsiDocumentManager;
import com.intellij.psi.PsiFile;
import com.intellij.util.CompressionUtil;
import com.intellij.util.ExceptionUtil;
import com.intellij.util.SystemProperties;
import com.intellij.util.indexing.impl.DebugAssertions;
import com.intellij.util.indexing.impl.forward.AbstractForwardIndexAccessor;
import com.intellij.util.indexing.impl.forward.PersistentMapBasedForwardIndex;
import com.intellij.util.io.*;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.*;
import java.nio.charset.Charset;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Stream;

class SnapshotInputMappings<Key, Value, Input> implements SnapshotInputMappingIndex<Key, Value, Input> {
  private static final Logger LOG = Logger.getInstance(SnapshotInputMappings.class);
  private static final boolean doReadSavedPersistentData = SystemProperties.getBooleanProperty("idea.read.saved.persistent.index", true);

  private final ID<Key, Value> myIndexId;
  private final InputMapExternalizer<Key, Value> myMapExternalizer;
  private final DataIndexer<Key, Value, Input> myIndexer;
  private final PersistentMapBasedForwardIndex myContents;
  private volatile PersistentHashMap<Integer, String> myIndexingTrace;

  private final HashIdForwardIndexAccessor<Key, Value, Input> myHashIdForwardIndexAccessor;

  private final boolean myIsPsiBackedIndex;

  SnapshotInputMappings(IndexExtension<Key, Value, Input> indexExtension) throws IOException {
    myIndexId = (ID<Key, Value>)indexExtension.getName();
    myIsPsiBackedIndex = indexExtension instanceof PsiDependentIndex;
    myMapExternalizer = new InputMapExternalizer<>(indexExtension);
    myIndexer = indexExtension.getIndexer();
    myContents = createContentsIndex();
    myHashIdForwardIndexAccessor = new HashIdForwardIndexAccessor<>(this);
    myIndexingTrace = DebugAssertions.EXTRA_SANITY_CHECKS ? createIndexingTrace() : null;
  }

  HashIdForwardIndexAccessor<Key, Value, Input> getForwardIndexAccessor() {
    return myHashIdForwardIndexAccessor;
  }

  File getInputIndexStorageFile() {
    return new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId");
  }

  @NotNull
  @Override
  public Map<Key, Value> readData(int hashId) throws IOException {
    ByteArraySequence byteSequence = readContents(hashId);
    if (byteSequence != null) {
      return AbstractForwardIndexAccessor.deserializeFromByteSeq(byteSequence, myMapExternalizer);
    }
    return Collections.emptyMap();
  }

  @Nullable
  @Override
  public Map<Key, Value> readData(@NotNull Input content) throws IOException {
    Map<Key, Value> data = null;

    if (doReadSavedPersistentData) {
      if (myContents == null || !myContents.isBusyReading() || DebugAssertions.EXTRA_SANITY_CHECKS) { // avoid blocking read, we can calculate index value
        int hashId = getHashId(content);
        ByteArraySequence bytes = readContents(hashId);

        if (bytes != null) {
          data = AbstractForwardIndexAccessor.deserializeFromByteSeq(bytes, myMapExternalizer);
          if (DebugAssertions.EXTRA_SANITY_CHECKS) {
            Map<Key, Value> contentData = myIndexer.map(content);
            boolean sameValueForSavedIndexedResultAndCurrentOne = contentData.equals(data);
            if (!sameValueForSavedIndexedResultAndCurrentOne) {
              DebugAssertions.error(
                "Unexpected difference in indexing of %s by index %s\ndiff %s\nprevious indexed info %s",
                getContentDebugData(content),
                myIndexId,
                buildDiff(data, contentData),
                myIndexingTrace.get(hashId)
              );
            }
          }
        }
      }
    }
    return data;
  }

  @Override
  public void putData(@Nullable Input content, @NotNull Map<Key, Value> data) throws IOException {
    int hashId = getHashId(content);
    boolean saved = savePersistentData(data, hashId);
    if (DebugAssertions.EXTRA_SANITY_CHECKS) {
      if (saved) {
        try {
          myIndexingTrace.put(hashId, getContentDebugData(content) +
                                      "," +
                                      ExceptionUtil.getThrowableText(new Throwable()));
        }
        catch (IOException ex) {
          LOG.error(ex);
        }
      }
    }
  }

  @NotNull
  private String getContentDebugData(Input input) {
    FileContentImpl content = (FileContentImpl) input;
    return "[" + content.getFile().getPath() + ";" + content.getFileType().getName() + ";" + content.getCharset() + "]";
  }

  @Override
{'fraction_non_alphanumeric': 0.09090056285178236, 'fraction_numerical': 0.001876172607879925, 'mean_word_length': 3.643292682926829, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7062041', 'n_tokens_mistral': 3019, 'n_tokens_neox': 2946, 'n_words': 623}