<commit_before>/*
 * @author max
 */
package com.intellij.openapi.vfs.newvfs.persistent;

import com.intellij.openapi.components.ApplicationComponent;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileManager;
import com.intellij.openapi.vfs.VirtualFileSystem;
import com.intellij.openapi.vfs.newvfs.*;
import com.intellij.openapi.vfs.newvfs.events.*;
import com.intellij.openapi.vfs.newvfs.impl.FakeVirtualFile;
import com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl;
import com.intellij.openapi.vfs.newvfs.impl.VirtualFileSystemEntry;
import com.intellij.util.ArrayUtil;
import com.intellij.util.io.DupOutputStream;
import com.intellij.util.io.LimitedInputStream;
import com.intellij.util.io.ReplicatorInputStream;
import com.intellij.util.messages.MessageBus;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.*;
import java.util.*;

public class PersistentFS extends ManagingFS implements ApplicationComponent {
  private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.vfs.newvfs.persistent.PersistentFS");
  private final static FileAttribute FILE_CONTENT = new FileAttribute("PersistentFS.File.Contents", 1);

  private static final int LOAD_THROUGH_BUFFER_THRESHOULD = 1024 * 1024; // megabyte

  private static final int CHILDREN_CACHED_FLAG = 0x01;
  private static final int IS_DIRECTORY_FLAG = 0x02;
  private static final int IS_READ_ONLY = 0x04;
  private static final int MUST_RELOAD_CONTENT = 0x08;

  private final FSRecords myRecords;
  private final MessageBus myEventsBus;

  private Map<String, NewVirtualFile> myRoots = new HashMap<String, NewVirtualFile>();

  public PersistentFS(MessageBus bus) {
    myEventsBus = bus;
    myRecords = new FSRecords();
  }

  public void disposeComponent() {
    myRecords.dispose();
  }

  @NonNls
  @NotNull
  public String getComponentName() {
    return "app.component.PersistentFS";
  }

  public void initComponent() {
    try {
      myRecords.connect();
    }
    catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private static NewVirtualFileSystem getDelegate(VirtualFile file) {
    return (NewVirtualFileSystem)file.getFileSystem();
  }

  public boolean areChildrenLoaded(final VirtualFile dir) {
    return areChildrenLoaded(getFileId(dir));
  }

  public String[] list(final VirtualFile file) {
    int id = getFileId(file);
    if (areChildrenLoaded(id)) {
      return listPersisted(file);
    }
    else {
      return persistAllChildren(file);
    }
  }

  public String[] listPersisted(final VirtualFile file) {
    return listPersisted(getFileId(file));
  }

  private String[] listPersisted(final int id) {
    final int[] childrenIds = myRecords.list(id);
    String[] names = new String[childrenIds.length];
    for (int i = 0; i < childrenIds.length; i++) {
      names[i] = myRecords.getName(childrenIds[i]);
    }
    return names;
  }

  private String[] persistAllChildren(final VirtualFile file) {
    int id = getFileId(file);
    String[] currentNames = listPersisted(file);
    int[] currentIds = myRecords.list(id);

    final NewVirtualFileSystem delegate = getDelegate(file);
    String[] names = delegate.list(file);
    final int[] childrenIds = new int[names.length];

    for (int i = 0; i < names.length; i++) {
      final String name = names[i];
      int idx = ArrayUtil.indexOf(currentNames, name);
      if (idx >= 0) {
        childrenIds[i] = currentIds[idx];
      }
      else {
        int childId = myRecords.createRecord();
        copyRecordFromDelegateFS(childId, id, new FakeVirtualFile(name, file), delegate);
        childrenIds[i] = childId;
      }
    }

    myRecords.updateList(id, childrenIds);
    int flags = myRecords.getFlags(id);
    myRecords.setFlags(id, flags | CHILDREN_CACHED_FLAG);

    return names;
  }

  public int[] listIds(VirtualFile parent) {
    final int parentId = getFileId(parent);

    if (!areChildrenLoaded(parentId)) {
      list(parent);
    }

    return myRecords.list(parentId);
  }


  public boolean areChildrenLoaded(final int parentId) {
    final int mask = CHILDREN_CACHED_FLAG;
    return (myRecords.getFlags(parentId) & mask) != 0;
  }

  @Nullable
  public DataInputStream readAttribute(final VirtualFile file, final FileAttribute att) {
    return myRecords.readAttribute(getFileId(file), att.getId());
  }

  public DataOutputStream writeAttribute(final VirtualFile file, final FileAttribute att) {
    return myRecords.writeAttribute(getFileId(file), att.getId());
  }

  public int getModificationCount(final VirtualFile file) {
    final int id = getFileId(file);
    return myRecords.getModCount(id);
  }

  public int getFilesystemModificationCount() {
    return myRecords.getModCount();
  }

  private void copyRecordFromDelegateFS(final int id, final int parentId, final VirtualFile file, NewVirtualFileSystem delegate) {
    if (id == parentId) {
      LOG.error("Cyclic parent-child relations for file: " + file);
      return;
    }

    String name = file.getName();

    if (name.length() > 0 && namesEqual(delegate, name, myRecords.getName(id))) return; // TODO: Handle root attributes change.

    if (name.length() == 0) {            // TODO: hack
      if (areChildrenLoaded(id)) return;
    }

    myRecords.setParent(id, parentId);
    myRecords.setName(id, name);

    myRecords.setCRC(id, delegate.getCRC(file));
    myRecords.setTimestamp(id, delegate.getTimeStamp(file));
    myRecords.setFlags(id, (delegate.isDirectory(file) ? IS_DIRECTORY_FLAG : 0) | (!delegate.isWritable(file) ? IS_READ_ONLY : 0));

    myRecords.setLength(id, -1);

    // TODO!!!: More attributes?
  }

  public boolean isDirectory(final VirtualFile file) {
    final int id = getFileId(file);
    return isDirectory(id);
  }

  public boolean isDirectory(final int id) {
    return (myRecords.getFlags(id) & IS_DIRECTORY_FLAG) != 0;
  }

  private static boolean namesEqual(VirtualFileSystem fs, String n1, String n2) {
    return ((NewVirtualFileSystem)fs).isCaseSensitive() ? n1.equals(n2) : n1.equalsIgnoreCase(n2);
  }

  public boolean exists(final VirtualFile fileOrDirectory) {
    return ((NewVirtualFile)fileOrDirectory).getId() > 0;
  }

  public long getTimeStamp(final VirtualFile file) {
    final int id = getFileId(file);
    return myRecords.getTimestamp(id);
  }

  public void setTimeStamp(final VirtualFile file, final long modstamp) throws IOException {
    final int id = getFileId(file);

    myRecords.setTimestamp(id, modstamp);
    getDelegate(file).setTimeStamp(file, modstamp);
  }

  private static int getFileId(final VirtualFile file) {
    final int id = ((NewVirtualFile)file).getId();
    assert id > 0;
    return id;
  }

  public boolean isWritable(final VirtualFile file) {
    final int id = getFileId(file);

    return (myRecords.getFlags(id) & IS_READ_ONLY) == 0;
  }

  public void setWritable(final VirtualFile file, final boolean writableFlag) throws IOException {
    getDelegate(file).setWritable(file, writableFlag);
    processEvent(new VFilePropertyChangeEvent(this, file, VirtualFile.PROP_WRITABLE, isWritable(file), writableFlag, false));
  }

  public long getCRC(final VirtualFile file) {
    final int id = getFileId(file);

    return myRecords.getCRC(id);
  }

  public int getId(final VirtualFile parent, final String childName) {
    final NewVirtualFileSystem delegate = getDelegate(parent);
    final int parentId = getFileId(parent);

    final int[] children = myRecords.list(parentId);
    for (final int childId : children) {
      if (namesEqual(delegate, childName, myRecords.getName(childId))) return childId;
    }

    VirtualFile fake = new FakeVirtualFile(childName, parent);
    if (delegate.exists(fake)) {
      int child = myRecords.createRecord();
      copyRecordFromDelegateFS(child, parentId, fake, delegate);
      myRecords.updateList(parentId, ArrayUtil.append(children, child));
      return child;
    }

    return 0;
  }

  public long getLength(final VirtualFile file) {
    final int id = getFileId(file);

    int len = myRecords.getLength(id);
    if (len == -1) {
      len = (int)getDelegate(file).getLength(file);
      myRecords.setLength(id, len);
    }

    return len;
  }

  public VirtualFile copyFile(final Object requestor, final VirtualFile file, final VirtualFile newParent, final String copyName) throws IOException {
    getDelegate(file).copyFile(requestor, file, newParent, copyName);
    processEvent(new VFileCopyEvent(requestor, file, newParent, copyName));

    final VirtualFile child = newParent.findChild(copyName);
    if (child == null) {
      throw new IOException("Cannot create child");
    }
    return child;
  }

  public VirtualFile createChildDirectory(final Object requestor, final VirtualFile parent, final String dir) throws IOException {
    getDelegate(parent).createChildDirectory(requestor, parent, dir);
    processEvent(new VFileCreateEvent(requestor, parent, dir, true, false));

    final VirtualFile child = parent.findChild(dir);
    if (child == null) {
      throw new IOException("Cannot create child directory '" + dir + "' at " + parent.getPath());
    }
    return child;
  }

  public VirtualFile createChildFile(final Object requestor, final VirtualFile parent, final String file) throws IOException {
    getDelegate(parent).createChildFile(requestor, parent, file);
    processEvent(new VFileCreateEvent(requestor, parent, file, false, false));

    final VirtualFile child = parent.findChild(file);
    if (child == null) {
      throw new IOException("Cannot create child file '" + file + "' at " + parent.getPath());
    }
    return child;
  }

  public void deleteFile(final Object requestor, final VirtualFile file) throws IOException {
    final NewVirtualFileSystem delegate = getDelegate(file);
    delegate.deleteFile(requestor, file);

    if (!delegate.exists(file)) {
      processEvent(new VFileDeleteEvent(requestor, file, false));
    }
  }

  public void renameFile(final Object requestor, final VirtualFile file, final String newName) throws IOException {
    final NewVirtualFileSystem delegate = getDelegate(file);
    delegate.renameFile(requestor, file, newName);

    if (newName.equalsIgnoreCase(file.getName()) || !delegate.exists(file)) {
      processEvent(new VFilePropertyChangeEvent(requestor, file, VirtualFile.PROP_NAME, file.getName(), newName, false));
    }
    else {
      processEvent(new VFileCreateEvent(requestor, file.getParent(), newName, file.isDirectory(), false));
    }
  }

  public InputStream getInputStream(final VirtualFile file) throws IOException {
    InputStream contentStream = FILE_CONTENT.readAttribute(file);
    if (contentStream == null || mustReloadContent(file)) {
      if (contentStream != null) contentStream.close();
      setFlag(file, MUST_RELOAD_CONTENT, false);

      final NewVirtualFileSystem delegate = getDelegate(file);
      final int len = (int)delegate.getLength(file);
      final InputStream nativeStream = delegate.getInputStream(file);
      //noinspection IOResourceOpenedButNotSafelyClosed
      ReplicatorInputStream result = new ReplicatorInputStream(nativeStream, new BufferedOutputStream(FILE_CONTENT.writeAttribute(file))) {
          public void close() throws IOException {
            super.close();<commit_msg>svn rename package fixed again: previous version didn't work because renaming a package actually does both PsiPackage and PsiDirectory rename (see RenameProcessor.preparePackageRenaming() and RenameProcessor.prepareDirectoryRenaming()), and the second rename failed on checkSetName() because a directory with the new name already existed. Now we fire an event on rename unconditionally, and refresh directory on command end to cause directory with old name to reappear.<commit_after>/*
 * @author max
 */
package com.intellij.openapi.vfs.newvfs.persistent;

import com.intellij.openapi.components.ApplicationComponent;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.VirtualFileManager;
import com.intellij.openapi.vfs.VirtualFileSystem;
import com.intellij.openapi.vfs.newvfs.*;
import com.intellij.openapi.vfs.newvfs.events.*;
import com.intellij.openapi.vfs.newvfs.impl.FakeVirtualFile;
import com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl;
import com.intellij.openapi.vfs.newvfs.impl.VirtualFileSystemEntry;
import com.intellij.util.ArrayUtil;
import com.intellij.util.io.DupOutputStream;
import com.intellij.util.io.LimitedInputStream;
import com.intellij.util.io.ReplicatorInputStream;
import com.intellij.util.messages.MessageBus;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.*;
import java.util.*;

public class PersistentFS extends ManagingFS implements ApplicationComponent {
  private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.vfs.newvfs.persistent.PersistentFS");
  private final static FileAttribute FILE_CONTENT = new FileAttribute("PersistentFS.File.Contents", 1);

  private static final int LOAD_THROUGH_BUFFER_THRESHOULD = 1024 * 1024; // megabyte

  private static final int CHILDREN_CACHED_FLAG = 0x01;
  private static final int IS_DIRECTORY_FLAG = 0x02;
  private static final int IS_READ_ONLY = 0x04;
  private static final int MUST_RELOAD_CONTENT = 0x08;

  private final FSRecords myRecords;
  private final MessageBus myEventsBus;

  private Map<String, NewVirtualFile> myRoots = new HashMap<String, NewVirtualFile>();

  public PersistentFS(MessageBus bus) {
    myEventsBus = bus;
    myRecords = new FSRecords();
  }

  public void disposeComponent() {
    myRecords.dispose();
  }

  @NonNls
  @NotNull
  public String getComponentName() {
    return "app.component.PersistentFS";
  }

  public void initComponent() {
    try {
      myRecords.connect();
    }
    catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private static NewVirtualFileSystem getDelegate(VirtualFile file) {
    return (NewVirtualFileSystem)file.getFileSystem();
  }

  public boolean areChildrenLoaded(final VirtualFile dir) {
    return areChildrenLoaded(getFileId(dir));
  }

  public String[] list(final VirtualFile file) {
    int id = getFileId(file);
    if (areChildrenLoaded(id)) {
      return listPersisted(file);
    }
    else {
      return persistAllChildren(file);
    }
  }

  public String[] listPersisted(final VirtualFile file) {
    return listPersisted(getFileId(file));
  }

  private String[] listPersisted(final int id) {
    final int[] childrenIds = myRecords.list(id);
    String[] names = new String[childrenIds.length];
    for (int i = 0; i < childrenIds.length; i++) {
      names[i] = myRecords.getName(childrenIds[i]);
    }
    return names;
  }

  private String[] persistAllChildren(final VirtualFile file) {
    int id = getFileId(file);
    String[] currentNames = listPersisted(file);
    int[] currentIds = myRecords.list(id);

    final NewVirtualFileSystem delegate = getDelegate(file);
    String[] names = delegate.list(file);
    final int[] childrenIds = new int[names.length];

    for (int i = 0; i < names.length; i++) {
      final String name = names[i];
      int idx = ArrayUtil.indexOf(currentNames, name);
      if (idx >= 0) {
        childrenIds[i] = currentIds[idx];
      }
      else {
        int childId = myRecords.createRecord();
        copyRecordFromDelegateFS(childId, id, new FakeVirtualFile(name, file), delegate);
        childrenIds[i] = childId;
      }
    }

    myRecords.updateList(id, childrenIds);
    int flags = myRecords.getFlags(id);
    myRecords.setFlags(id, flags | CHILDREN_CACHED_FLAG);

    return names;
  }

  public int[] listIds(VirtualFile parent) {
    final int parentId = getFileId(parent);

    if (!areChildrenLoaded(parentId)) {
      list(parent);
    }

    return myRecords.list(parentId);
  }


  public boolean areChildrenLoaded(final int parentId) {
    final int mask = CHILDREN_CACHED_FLAG;
    return (myRecords.getFlags(parentId) & mask) != 0;
  }

  @Nullable
  public DataInputStream readAttribute(final VirtualFile file, final FileAttribute att) {
    return myRecords.readAttribute(getFileId(file), att.getId());
  }

  public DataOutputStream writeAttribute(final VirtualFile file, final FileAttribute att) {
    return myRecords.writeAttribute(getFileId(file), att.getId());
  }

  public int getModificationCount(final VirtualFile file) {
    final int id = getFileId(file);
    return myRecords.getModCount(id);
  }

  public int getFilesystemModificationCount() {
    return myRecords.getModCount();
  }

  private void copyRecordFromDelegateFS(final int id, final int parentId, final VirtualFile file, NewVirtualFileSystem delegate) {
    if (id == parentId) {
      LOG.error("Cyclic parent-child relations for file: " + file);
      return;
    }

    String name = file.getName();

    if (name.length() > 0 && namesEqual(delegate, name, myRecords.getName(id))) return; // TODO: Handle root attributes change.

    if (name.length() == 0) {            // TODO: hack
      if (areChildrenLoaded(id)) return;
    }

    myRecords.setParent(id, parentId);
    myRecords.setName(id, name);

    myRecords.setCRC(id, delegate.getCRC(file));
    myRecords.setTimestamp(id, delegate.getTimeStamp(file));
    myRecords.setFlags(id, (delegate.isDirectory(file) ? IS_DIRECTORY_FLAG : 0) | (!delegate.isWritable(file) ? IS_READ_ONLY : 0));

    myRecords.setLength(id, -1);

    // TODO!!!: More attributes?
  }

  public boolean isDirectory(final VirtualFile file) {
    final int id = getFileId(file);
    return isDirectory(id);
  }

  public boolean isDirectory(final int id) {
    return (myRecords.getFlags(id) & IS_DIRECTORY_FLAG) != 0;
  }

  private static boolean namesEqual(VirtualFileSystem fs, String n1, String n2) {
    return ((NewVirtualFileSystem)fs).isCaseSensitive() ? n1.equals(n2) : n1.equalsIgnoreCase(n2);
  }

  public boolean exists(final VirtualFile fileOrDirectory) {
    return ((NewVirtualFile)fileOrDirectory).getId() > 0;
  }

  public long getTimeStamp(final VirtualFile file) {
    final int id = getFileId(file);
    return myRecords.getTimestamp(id);
  }

  public void setTimeStamp(final VirtualFile file, final long modstamp) throws IOException {
    final int id = getFileId(file);

    myRecords.setTimestamp(id, modstamp);
    getDelegate(file).setTimeStamp(file, modstamp);
  }

  private static int getFileId(final VirtualFile file) {
    final int id = ((NewVirtualFile)file).getId();
    assert id > 0;
    return id;
  }

  public boolean isWritable(final VirtualFile file) {
    final int id = getFileId(file);

    return (myRecords.getFlags(id) & IS_READ_ONLY) == 0;
  }

  public void setWritable(final VirtualFile file, final boolean writableFlag) throws IOException {
    getDelegate(file).setWritable(file, writableFlag);
    processEvent(new VFilePropertyChangeEvent(this, file, VirtualFile.PROP_WRITABLE, isWritable(file), writableFlag, false));
  }

  public long getCRC(final VirtualFile file) {
    final int id = getFileId(file);

    return myRecords.getCRC(id);
  }

  public int getId(final VirtualFile parent, final String childName) {
    final NewVirtualFileSystem delegate = getDelegate(parent);
    final int parentId = getFileId(parent);

    final int[] children = myRecords.list(parentId);
    for (final int childId : children) {
      if (namesEqual(delegate, childName, myRecords.getName(childId))) return childId;
    }

    VirtualFile fake = new FakeVirtualFile(childName, parent);
    if (delegate.exists(fake)) {
      int child = myRecords.createRecord();
      copyRecordFromDelegateFS(child, parentId, fake, delegate);
      myRecords.updateList(parentId, ArrayUtil.append(children, child));
      return child;
    }

    return 0;
  }

  public long getLength(final VirtualFile file) {
    final int id = getFileId(file);

    int len = myRecords.getLength(id);
    if (len == -1) {
      len = (int)getDelegate(file).getLength(file);
      myRecords.setLength(id, len);
    }

    return len;
  }

  public VirtualFile copyFile(final Object requestor, final VirtualFile file, final VirtualFile newParent, final String copyName) throws IOException {
    getDelegate(file).copyFile(requestor, file, newParent, copyName);
    processEvent(new VFileCopyEvent(requestor, file, newParent, copyName));

    final VirtualFile child = newParent.findChild(copyName);
    if (child == null) {
      throw new IOException("Cannot create child");
    }
    return child;
  }

  public VirtualFile createChildDirectory(final Object requestor, final VirtualFile parent, final String dir) throws IOException {
    getDelegate(parent).createChildDirectory(requestor, parent, dir);
    processEvent(new VFileCreateEvent(requestor, parent, dir, true, false));

    final VirtualFile child = parent.findChild(dir);
    if (child == null) {
      throw new IOException("Cannot create child directory '" + dir + "' at " + parent.getPath());
    }
    return child;
  }

  public VirtualFile createChildFile(final Object requestor, final VirtualFile parent, final String file) throws IOException {
    getDelegate(parent).createChildFile(requestor, parent, file);
    processEvent(new VFileCreateEvent(requestor, parent, file, false, false));

    final VirtualFile child = parent.findChild(file);
    if (child == null) {
      throw new IOException("Cannot create child file '" + file + "' at " + parent.getPath());
    }
    return child;
  }

  public void deleteFile(final Object requestor, final VirtualFile file) throws IOException {
    final NewVirtualFileSystem delegate = getDelegate(file);
    delegate.deleteFile(requestor, file);

    if (!delegate.exists(file)) {
      processEvent(new VFileDeleteEvent(requestor, file, false));
    }
  }

  public void renameFile(final Object requestor, final VirtualFile file, final String newName) throws IOException {
    getDelegate(file).renameFile(requestor, file, newName);
    processEvent(new VFilePropertyChangeEvent(requestor, file, VirtualFile.PROP_NAME, file.getName(), newName, false));
  }

  public InputStream getInputStream(final VirtualFile file) throws IOException {
    InputStream contentStream = FILE_CONTENT.readAttribute(file);
    if (contentStream == null || mustReloadContent(file)) {
      if (contentStream != null) contentStream.close();
      setFlag(file, MUST_RELOAD_CONTENT, false);

      final NewVirtualFileSystem delegate = getDelegate(file);
      final int len = (int)delegate.getLength(file);
      final InputStream nativeStream = delegate.getInputStream(file);
      //noinspection IOResourceOpenedButNotSafelyClosed
      ReplicatorInputStream result = new ReplicatorInputStream(nativeStream, new BufferedOutputStream(FILE_CONTENT.writeAttribute(file))) {
          public void close() throws IOException {
            super.close();
{'fraction_non_alphanumeric': 0.10322218867273433, 'fraction_numerical': 0.0036233446922313765, 'mean_word_length': 4.149711239449133, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5012599', 'n_tokens_mistral': 6980, 'n_tokens_neox': 6752, 'n_words': 1706}