<commit_before>// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.psi.stubs;

import com.intellij.openapi.Disposable;
import com.intellij.openapi.Forceable;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Computable;
import com.intellij.openapi.util.ShutDownTracker;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.StubFileElementType;
import com.intellij.util.indexing.FileBasedIndex;
import com.intellij.util.io.DataEnumeratorEx;
import com.intellij.util.io.IOUtil;
import com.intellij.util.io.InMemoryDataEnumerator;
import com.intellij.util.io.PersistentStringEnumerator;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.*;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;

@ApiStatus.Internal
public final class SerializationManagerImpl extends SerializationManagerEx implements Disposable {
  private static final Logger LOG = Logger.getInstance(SerializationManagerImpl.class);

  private final AtomicBoolean myNameStorageCrashed = new AtomicBoolean(false);
  private final Path myFile;
  private final boolean myUnmodifiable;
  private final AtomicBoolean myShutdownPerformed = new AtomicBoolean(false);
  private DataEnumeratorEx<String> myNameStorage;
  private volatile StubSerializationHelper myStubSerializationHelper;

  private volatile boolean mySerializersLoaded;

  @SuppressWarnings("unused") // used from componentSets/Lang.xml:14
  public SerializationManagerImpl() {
    this(FileBasedIndex.USE_IN_MEMORY_INDEX ? null : new File(PathManager.getIndexRoot(), "rep.names").toPath(), false);
  }

  public SerializationManagerImpl(@Nullable Path nameStorageFile, boolean unmodifiable) {
    myFile = nameStorageFile;
    myUnmodifiable = unmodifiable;
    try {
      // we need to cache last id -> String mappings due to StringRefs and stubs indexing that initially creates stubs (doing enumerate on String)
      // and then index them (valueOf), also similar string items are expected to be enumerated during stubs processing
      myNameStorage = openNameStorage();
      myStubSerializationHelper = new StubSerializationHelper(myNameStorage, unmodifiable, this);
    }
    catch (IOException e) {
      nameStorageCrashed();
      LOG.info(e);
      repairNameStorage(); // need this in order for myNameStorage not to be null
      nameStorageCrashed();
    }
    finally {
      ShutDownTracker.getInstance().registerShutdownTask(this::performShutdown);
    }

    StubElementTypeHolderEP.EP_NAME.addChangeListener(this::dropSerializerData, this);
  }

  @NotNull
  private DataEnumeratorEx<String> openNameStorage() throws IOException {
    return myFile == null ? new InMemoryDataEnumerator<>() : new PersistentStringEnumerator(myFile, true);
  }

  @Override
  public boolean isNameStorageCorrupted() {
    return myNameStorageCrashed.get();
  }

  @Override
  public void repairNameStorage() {
    if (myNameStorageCrashed.getAndSet(false)) {<commit_msg>index: unmodifiable SerializationManager should not use write access WEB-45822<commit_after>// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.psi.stubs;

import com.intellij.openapi.Disposable;
import com.intellij.openapi.Forceable;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Computable;
import com.intellij.openapi.util.ShutDownTracker;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.StubFileElementType;
import com.intellij.util.indexing.FileBasedIndex;
import com.intellij.util.io.*;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.DataOutputStream;
import java.io.*;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;

@ApiStatus.Internal
public final class SerializationManagerImpl extends SerializationManagerEx implements Disposable {
  private static final Logger LOG = Logger.getInstance(SerializationManagerImpl.class);

  private final AtomicBoolean myNameStorageCrashed = new AtomicBoolean(false);
  private final Path myFile;
  private final boolean myUnmodifiable;
  private final AtomicBoolean myShutdownPerformed = new AtomicBoolean(false);
  private DataEnumeratorEx<String> myNameStorage;
  private volatile StubSerializationHelper myStubSerializationHelper;

  private volatile boolean mySerializersLoaded;

  @SuppressWarnings("unused") // used from componentSets/Lang.xml:14
  public SerializationManagerImpl() {
    this(FileBasedIndex.USE_IN_MEMORY_INDEX ? null : new File(PathManager.getIndexRoot(), "rep.names").toPath(), false);
  }

  public SerializationManagerImpl(@Nullable Path nameStorageFile, boolean unmodifiable) {
    myFile = nameStorageFile;
    myUnmodifiable = unmodifiable;
    try {
      // we need to cache last id -> String mappings due to StringRefs and stubs indexing that initially creates stubs (doing enumerate on String)
      // and then index them (valueOf), also similar string items are expected to be enumerated during stubs processing
      myNameStorage = openNameStorage();
      myStubSerializationHelper = new StubSerializationHelper(myNameStorage, unmodifiable, this);
    }
    catch (IOException e) {
      nameStorageCrashed();
      LOG.info(e);
      repairNameStorage(); // need this in order for myNameStorage not to be null
      nameStorageCrashed();
    }
    finally {
      ShutDownTracker.getInstance().registerShutdownTask(this::performShutdown);
    }

    StubElementTypeHolderEP.EP_NAME.addChangeListener(this::dropSerializerData, this);
  }

  @NotNull
  private DataEnumeratorEx<String> openNameStorage() throws IOException {
    if (myFile == null) {
      return new InMemoryDataEnumerator<>();
    }
    Boolean lastValue = null;
    if (myUnmodifiable) {
      lastValue = PersistentHashMapValueStorage.CreationTimeOptions.READONLY.get();
      PersistentHashMapValueStorage.CreationTimeOptions.READONLY.set(Boolean.TRUE);
    }
    try {
      return new PersistentStringEnumerator(myFile, true);
    } finally {
      if (myUnmodifiable) {
        PersistentHashMapValueStorage.CreationTimeOptions.READONLY.set(lastValue);
      }
    }
  }

  @Override
  public boolean isNameStorageCorrupted() {
    return myNameStorageCrashed.get();
  }

  @Override
  public void repairNameStorage() {
    if (myNameStorageCrashed.getAndSet(false)) {
{'fraction_non_alphanumeric': 0.08354023659997079, 'fraction_numerical': 0.004235431575872645, 'mean_word_length': 4.918755401901469, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '262057', 'n_tokens_mistral': 1964, 'n_tokens_neox': 1937, 'n_words': 477}