<commit_before>package com.intellij.compilerOutputIndex.api.indexer;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.extensions.ExtensionPointName;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Factory;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.util.ThrowableComputable;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.indexing.*;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.IOUtil;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentHashMap;
import org.jetbrains.asm4.tree.ClassNode;

import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.atomic.AtomicBoolean;

import static com.intellij.util.indexing.IndexInfrastructure.*;

/**
 * @author <NAME>
 */
public abstract class CompilerOutputBaseIndex<K, V> {
  public final static ExtensionPointName<CompilerOutputBaseIndex> EXTENSION_POINT_NAME =
    ExtensionPointName.create("com.intellij.java.compilerOutputIndex");

  private final static Logger LOG = Logger.getInstance(CompilerOutputBaseIndex.class);
  private final KeyDescriptor<K> myKeyDescriptor;
  private final DataExternalizer<V> myValueExternalizer;
  protected volatile MapReduceIndex<K, V, ClassNode> myIndex;

  protected final Project myProject;

  protected volatile AtomicBoolean myInitialized = new AtomicBoolean(false);

  public CompilerOutputBaseIndex(final KeyDescriptor<K> keyDescriptor, final DataExternalizer<V> valueExternalizer, final Project project) {
    myProject = project;
    myKeyDescriptor = keyDescriptor;
    myValueExternalizer = valueExternalizer;
  }

  public final boolean initIfNeed() {
    if (myInitialized.compareAndSet(false, true)) {
      final MapReduceIndex<K, V, ClassNode> index;
      final Ref<Boolean> rewriteIndex = new Ref<Boolean>(false);
      try {
        final ID<K, V> indexId = getIndexId();
        if (!IndexInfrastructure.getIndexRootDir(indexId).exists()) {
          rewriteIndex.set(true);
        }
        final File storageFile = getStorageFile(indexId);
        final MapIndexStorage<K, V> indexStorage = new MapIndexStorage<K, V>(storageFile, myKeyDescriptor, myValueExternalizer, 1024);
        index = new MapReduceIndex<K, V, ClassNode>(indexId, getIndexer(), indexStorage);
        index.setInputIdToDataKeysIndex(new Factory<PersistentHashMap<Integer, Collection<K>>>() {
          @Override
          public PersistentHashMap<Integer, Collection<K>> create() {
            try {
              return IOUtil.openCleanOrResetBroken(
                new ThrowableComputable<PersistentHashMap<Integer, Collection<K>>, IOException>() {
                  @Override
                  public PersistentHashMap<Integer, Collection<K>> compute() throws IOException {
                    return FileBasedIndexImpl.createIdToDataKeysIndex(indexId, myKeyDescriptor, new MemoryIndexStorage<K, V>(indexStorage));
                  }
                },
                new Runnable() {
                  @Override
                  public void run() {
                    FileUtil.delete(getInputIndexStorageFile(getIndexId()));
                    rewriteIndex.set(true);
                  }
                }
              );
            }
            catch (IOException e) {
              throw new RuntimeException("couldn't create index", e);
            }
          }
        });
        final File versionFile = getVersionFile(indexId);
        if (versionFile.exists()) {<commit_msg>another fix for EA-49431 - PEBCE: PersistentEnumeratorBase.<init><commit_after>package com.intellij.compilerOutputIndex.api.indexer;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.extensions.ExtensionPointName;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.Factory;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.util.ThrowableComputable;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.indexing.*;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.IOUtil;
import com.intellij.util.io.KeyDescriptor;
import com.intellij.util.io.PersistentHashMap;
import org.jetbrains.asm4.tree.ClassNode;

import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.atomic.AtomicBoolean;

import static com.intellij.util.indexing.IndexInfrastructure.*;

/**
 * @author <NAME>
 */
public abstract class CompilerOutputBaseIndex<K, V> {
  public final static ExtensionPointName<CompilerOutputBaseIndex> EXTENSION_POINT_NAME =
    ExtensionPointName.create("com.intellij.java.compilerOutputIndex");

  private final static Logger LOG = Logger.getInstance(CompilerOutputBaseIndex.class);
  private final KeyDescriptor<K> myKeyDescriptor;
  private final DataExternalizer<V> myValueExternalizer;
  protected volatile MapReduceIndex<K, V, ClassNode> myIndex;

  protected final Project myProject;

  protected volatile AtomicBoolean myInitialized = new AtomicBoolean(false);

  public CompilerOutputBaseIndex(final KeyDescriptor<K> keyDescriptor, final DataExternalizer<V> valueExternalizer, final Project project) {
    myProject = project;
    myKeyDescriptor = keyDescriptor;
    myValueExternalizer = valueExternalizer;
  }

  public final boolean initIfNeed() {
    if (myInitialized.compareAndSet(false, true)) {
      final MapReduceIndex<K, V, ClassNode> index;
      final Ref<Boolean> rewriteIndex = new Ref<Boolean>(false);
      try {
        final ID<K, V> indexId = getIndexId();
        if (!IndexInfrastructure.getIndexRootDir(indexId).exists()) {
          rewriteIndex.set(true);
        }
        final File storageFile = getStorageFile(indexId);
        final MapIndexStorage<K, V> indexStorage = IOUtil.openCleanOrResetBroken(
          new ThrowableComputable<MapIndexStorage<K, V>, IOException>() {
            @Override
            public MapIndexStorage<K, V> compute() throws IOException {
              return new MapIndexStorage<K, V>(storageFile, myKeyDescriptor, myValueExternalizer, 1024);
            }
          },
          new Runnable() {
            @Override
            public void run() {
              IOUtil.deleteAllFilesStartingWith(storageFile);
              rewriteIndex.set(true);
            }
          }
        );
        index = new MapReduceIndex<K, V, ClassNode>(indexId, getIndexer(), indexStorage);
        index.setInputIdToDataKeysIndex(new Factory<PersistentHashMap<Integer, Collection<K>>>() {
          @Override
          public PersistentHashMap<Integer, Collection<K>> create() {
            try {
              return IOUtil.openCleanOrResetBroken(
                new ThrowableComputable<PersistentHashMap<Integer, Collection<K>>, IOException>() {
                  @Override
                  public PersistentHashMap<Integer, Collection<K>> compute() throws IOException {
                    return FileBasedIndexImpl.createIdToDataKeysIndex(indexId, myKeyDescriptor, new MemoryIndexStorage<K, V>(indexStorage));
                  }
                },
                new Runnable() {
                  @Override
                  public void run() {
                    FileUtil.delete(getInputIndexStorageFile(getIndexId()));
                    rewriteIndex.set(true);
                  }
                }
              );
            }
            catch (IOException e) {
              throw new RuntimeException("couldn't create index", e);
            }
          }
        });
        final File versionFile = getVersionFile(indexId);
        if (versionFile.exists()) {
{'fraction_non_alphanumeric': 0.09482870913117103, 'fraction_numerical': 0.0019538882375928096, 'mean_word_length': 3.086216072378925, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5222051', 'n_tokens_mistral': 2113, 'n_tokens_neox': 2070, 'n_words': 418}