<commit_before>/*
 *
 *  *  Copyright 2014 Orient Technologies LTD (info(at)orientechnologies.com)
 *  *
 *  *  Licensed under the Apache License, Version 2.0 (the "License");
 *  *  you may not use this file except in compliance with the License.
 *  *  You may obtain a copy of the License at
 *  *
 *  *       http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *  Unless required by applicable law or agreed to in writing, software
 *  *  distributed under the License is distributed on an "AS IS" BASIS,
 *  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *  See the License for the specific language governing permissions and
 *  *  limitations under the License.
 *  *
 *  * For more information: http://www.orientechnologies.com
 *
 */

package com.orientechnologies.orient.core.storage.impl.local;

import com.orientechnologies.common.concur.lock.OLockManager;
import com.orientechnologies.common.concur.lock.OModificationLock;
import com.orientechnologies.common.exception.OException;
import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.common.types.OModifiableBoolean;
import com.orientechnologies.common.util.OCommonConst;
import com.orientechnologies.orient.core.OOrientShutdownListener;
import com.orientechnologies.orient.core.OOrientStartupListener;
import com.orientechnologies.orient.core.Orient;
import com.orientechnologies.orient.core.command.OCommandExecutor;
import com.orientechnologies.orient.core.command.OCommandManager;
import com.orientechnologies.orient.core.command.OCommandOutputListener;
import com.orientechnologies.orient.core.command.OCommandRequestText;
import com.orientechnologies.orient.core.config.OGlobalConfiguration;
import com.orientechnologies.orient.core.config.OStorageClusterConfiguration;
import com.orientechnologies.orient.core.config.OStoragePaginatedClusterConfiguration;
import com.orientechnologies.orient.core.conflict.ORecordConflictStrategy;
import com.orientechnologies.orient.core.db.ODatabaseDocumentInternal;
import com.orientechnologies.orient.core.db.ODatabaseListener;
import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal;
import com.orientechnologies.orient.core.db.record.OCurrentStorageComponentsFactory;
import com.orientechnologies.orient.core.db.record.ORecordOperation;
import com.orientechnologies.orient.core.db.record.ridbag.sbtree.OSBTreeCollectionManagerShared;
import com.orientechnologies.orient.core.exception.OCommandExecutionException;
import com.orientechnologies.orient.core.exception.OConcurrentModificationException;
import com.orientechnologies.orient.core.exception.OConfigurationException;
import com.orientechnologies.orient.core.exception.OFastConcurrentModificationException;
import com.orientechnologies.orient.core.exception.OLowDiskSpaceException;
import com.orientechnologies.orient.core.exception.ORecordNotFoundException;
import com.orientechnologies.orient.core.exception.OStorageException;
import com.orientechnologies.orient.core.id.ORID;
import com.orientechnologies.orient.core.id.ORecordId;
import com.orientechnologies.orient.core.metadata.OMetadataDefault;
import com.orientechnologies.orient.core.metadata.OMetadataInternal;
import com.orientechnologies.orient.core.metadata.schema.OClass;
import com.orientechnologies.orient.core.metadata.security.OSecurityUser;
import com.orientechnologies.orient.core.metadata.security.OToken;
import com.orientechnologies.orient.core.record.ORecord;
import com.orientechnologies.orient.core.record.ORecordInternal;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.record.impl.ODocumentInternal;
import com.orientechnologies.orient.core.storage.OCluster;
import com.orientechnologies.orient.core.storage.OIdentifiableStorage;
import com.orientechnologies.orient.core.storage.OPhysicalPosition;
import com.orientechnologies.orient.core.storage.ORawBuffer;
import com.orientechnologies.orient.core.storage.ORecordCallback;
import com.orientechnologies.orient.core.storage.ORecordMetadata;
import com.orientechnologies.orient.core.storage.OStorageAbstract;
import com.orientechnologies.orient.core.storage.OStorageOperationResult;
import com.orientechnologies.orient.core.storage.cache.OCacheEntry;
import com.orientechnologies.orient.core.storage.cache.OCachePointer;
import com.orientechnologies.orient.core.storage.cache.OPageDataVerificationError;
import com.orientechnologies.orient.core.storage.cache.OReadCache;
import com.orientechnologies.orient.core.storage.cache.OWriteCache;
import com.orientechnologies.orient.core.storage.impl.local.paginated.OOfflineCluster;
import com.orientechnologies.orient.core.storage.impl.local.paginated.OOfflineClusterException;
import com.orientechnologies.orient.core.storage.impl.local.paginated.ORecordSerializationContext;
import com.orientechnologies.orient.core.storage.impl.local.paginated.OStorageTransaction;
import com.orientechnologies.orient.core.storage.impl.local.paginated.atomicoperations.OAtomicOperationsManager;
import com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage;
import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.*;
import com.orientechnologies.orient.core.tx.OTransaction;
import com.orientechnologies.orient.core.tx.OTransactionAbstract;
import com.orientechnologies.orient.core.tx.OTxListener;
import com.orientechnologies.orient.core.type.tree.provider.OMVRBTreeRIDProvider;
import com.orientechnologies.orient.core.version.ORecordVersion;
import com.orientechnologies.orient.core.version.OSimpleVersion;
import com.orientechnologies.orient.core.version.OVersionFactory;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * @author <NAME>
 * @since 28.03.13
 */
public abstract class OAbstractPaginatedStorage extends OStorageAbstract implements OLowDiskSpaceListener,
    OFullCheckpointRequestListener, OIdentifiableStorage, OOrientStartupListener, OOrientShutdownListener {
  private static final int RECORD_LOCK_TIMEOUT = OGlobalConfiguration.STORAGE_RECORD_LOCK_TIMEOUT.getValueAsInteger();

  private final OLockManager<ORID>                  lockManager;
  private final String                              PROFILER_CREATE_RECORD;
  private final String                              PROFILER_READ_RECORD;
  private final String                              PROFILER_UPDATE_RECORD;
  private final String                              PROFILER_DELETE_RECORD;
  private final Map<String, OCluster>               clusterMap           = new HashMap<String, OCluster>();
  private volatile ThreadLocal<OStorageTransaction> transaction          = new ThreadLocal<OStorageTransaction>();
  private final OModificationLock                   modificationLock     = new OModificationLock();
  private final AtomicBoolean                       checkpointInProgress = new AtomicBoolean();
  protected volatile OWriteAheadLog                 writeAheadLog;

  protected volatile OReadCache  readCache;
  protected volatile OWriteCache writeCache;

  private volatile ORecordConflictStrategy  recordConflictStrategy                     = Orient.instance()
      .getRecordConflictStrategy().newInstanceOfDefaultClass();
  private List<OCluster>                    clusters                                   = new ArrayList<OCluster>();
  private volatile int                      defaultClusterId                           = -1;
  private volatile OAtomicOperationsManager atomicOperationsManager;
  private volatile boolean                  wereDataRestoredAfterOpen                  = false;
  private volatile boolean                  wereNonTxOperationsPerformedInPreviousOpen = false;
  private boolean                           makeFullCheckPointAfterClusterCreate       = OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CLUSTER_CREATE
      .getValueAsBoolean();
  private volatile OLowDiskSpaceInformation lowDiskSpace                               = null;
  private volatile boolean                  checkpointRequest                          = false;

  private final int id;

  public OAbstractPaginatedStorage(String name, String filePath, String mode, int id) {
    super(name, filePath, mode, OGlobalConfiguration.STORAGE_LOCK_TIMEOUT.getValueAsInteger());

    this.id = id;
    lockManager = new OLockManager<ORID>(true, -1) {
      @Override
      protected ORID getImmutableResourceId(ORID iResourceId) {
        return new ORecordId(iResourceId);
      }
    };

    PROFILER_CREATE_RECORD = "db." + this.name + ".createRecord";
    PROFILER_READ_RECORD = "db." + this.name + ".readRecord";
    PROFILER_UPDATE_RECORD = "db." + this.name + ".updateRecord";
    PROFILER_DELETE_RECORD = "db." + this.name + ".deleteRecord";
  }

  public void open(final String iUserName, final String iUserPassword, final Map<String, Object> iProperties) {
    stateLock.acquireReadLock();
    try {
      if (status == STATUS.OPEN)
        // ALREADY OPENED: THIS IS THE CASE WHEN A STORAGE INSTANCE IS
        // REUSED
        return;
    } finally {
      stateLock.releaseReadLock();
    }

    stateLock.acquireWriteLock();
    try {

      if (status == STATUS.OPEN)
        // ALREADY OPENED: THIS IS THE CASE WHEN A STORAGE INSTANCE IS
        // REUSED
        return;

      if (!exists())
        throw new OStorageException("Cannot open the storage '" + name + "' because it does not exist in path: " + url);

      configuration.load();
      componentsFactory = new OCurrentStorageComponentsFactory(configuration);

      preOpenSteps();

      initWalAndDiskCache();

      atomicOperationsManager = new OAtomicOperationsManager(this);
      try {
        atomicOperationsManager.registerMBean();
      } catch (Exception e) {
        OLogManager.instance().error(this, "MBean for atomic operations manager cannot be registered.", e);
      }

      restoreIfNeeded();

      // OPEN BASIC SEGMENTS
      int pos;
      addDefaultClusters();

      // REGISTER CLUSTER
      for (int i = 0; i < configuration.clusters.size(); ++i) {
        final OStorageClusterConfiguration clusterConfig = configuration.clusters.get(i);

        if (clusterConfig != null) {
          pos = createClusterFromConfig(clusterConfig);

          try {
            if (pos == -1) {
              clusters.get(i).open();
            } else {
              if (clusterConfig.getName().equals(CLUSTER_DEFAULT_NAME))
                defaultClusterId = pos;

              clusters.get(pos).open();
            }
          } catch (FileNotFoundException e) {
            OLogManager.instance().warn(this, "Error on loading cluster '" + clusters.get(i).getName() + "' (" + i
                + "): file not found. It will be excluded from current database '" + getName() + "'.");

            clusterMap.remove(clusters.get(i).getName().toLowerCase());

            setCluster(i, null);
          }
        } else {
          setCluster(i, null);
        }
      }

      if (OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_OPEN.getValueAsBoolean())
        makeFullCheckpoint();

      writeCache.startFuzzyCheckpoints();

      status = STATUS.OPEN;
    } catch (Exception e) {
      status = STATUS.CLOSED;
      throw new OStorageException("Cannot open local storage '" + url + "' with mode=" + mode, e);
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public void open(final OToken iToken, final Map<String, Object> iProperties) {
    open(iToken.getUserName(), "", iProperties);
  }

  public void create(final Map<String, Object> iProperties) {
    stateLock.acquireWriteLock();
    try {

      if (status != STATUS.CLOSED)
        throw new OStorageException("Cannot create new storage '" + getURL() + "' because it is not closed");

      if (exists())
        throw new OStorageException("Cannot create new storage '" + getURL() + "' because it already exists");

      if (!configuration.getContextConfiguration().getContextKeys()
          .contains(OGlobalConfiguration.STORAGE_COMPRESSION_METHOD.getKey()))

        // SAVE COMPRESSION IN STORAGE CFG
        configuration.getContextConfiguration().setValue(OGlobalConfiguration.STORAGE_COMPRESSION_METHOD,
            OGlobalConfiguration.STORAGE_COMPRESSION_METHOD.getValue());

      componentsFactory = new OCurrentStorageComponentsFactory(configuration);
      initWalAndDiskCache();

      atomicOperationsManager = new OAtomicOperationsManager(this);
      try {
        atomicOperationsManager.registerMBean();
      } catch (Exception e) {
        OLogManager.instance().error(this, "MBean for atomic operations manager cannot be registered.", e);
      }

      preCreateSteps();

      status = STATUS.OPEN;

      // ADD THE METADATA CLUSTER TO STORE INTERNAL STUFF
      doAddCluster(OMetadataDefault.CLUSTER_INTERNAL_NAME, false, null);

      configuration.create();

      // ADD THE INDEX CLUSTER TO STORE, BY DEFAULT, ALL THE RECORDS OF
      // INDEXING
      doAddCluster(OMetadataDefault.CLUSTER_INDEX_NAME, false, null);

      // ADD THE INDEX CLUSTER TO STORE, BY DEFAULT, ALL THE RECORDS OF
      // INDEXING
      doAddCluster(OMetadataDefault.CLUSTER_MANUAL_INDEX_NAME, false, null);

      // ADD THE DEFAULT CLUSTER
      defaultClusterId = doAddCluster(CLUSTER_DEFAULT_NAME, false, null);

      clearStorageDirty();
      if (OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CREATE.getValueAsBoolean())
        makeFullCheckpoint();

      writeCache.startFuzzyCheckpoints();
      postCreateSteps();

    } catch (OStorageException e) {
      close();
      throw e;
    } catch (IOException e) {
      close();
      throw new OStorageException("Error on creation of storage '" + name + "'", e);
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  @Override
  public void onShutdown() {
    transaction = null;
  }

  @Override
  public void onStartup() {
    if (transaction == null)
      transaction = new ThreadLocal<OStorageTransaction>();
  }

  public void startAtomicOperation(boolean rollbackOnlyMode) throws IOException {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();
      makeStorageDirty();

      atomicOperationsManager.startAtomicOperation((String) null, rollbackOnlyMode);
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public void commitAtomicOperation() throws IOException {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();
      atomicOperationsManager.endAtomicOperation(false, null);
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public void rollbackAtomicOperation() throws IOException {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();
      atomicOperationsManager.endAtomicOperation(true, null);
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public void markDirty() throws IOException {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();
      makeStorageDirty();
    } finally {
      stateLock.releaseReadLock();
    }
  }

  @Override
  public void close(final boolean force, boolean onDelete) {
    doClose(force, onDelete);
  }

  public void delete() {
    final long timer = Orient.instance().getProfiler().startChrono();

    stateLock.acquireWriteLock();
    try {
      dataLock.acquireExclusiveLock();
      try {
        // CLOSE THE DATABASE BY REMOVING THE CURRENT USER
        doClose(true, true);

        try {
          Orient.instance().unregisterStorage(this);
        } catch (Exception e) {
          OLogManager.instance().error(this, "Cannot unregister storage", e);
        }

        if (writeAheadLog != null)
          writeAheadLog.delete();

        if (writeCache != null) {
          if (readCache != null)
            readCache.deleteStorage(writeCache);
          else
            writeCache.delete();
        }

        postDeleteSteps();

      } catch (IOException e) {
        throw new OStorageException("Cannot delete database '" + name + "'.", e);
      } finally {
        dataLock.releaseExclusiveLock();

        Orient.instance().getProfiler().stopChrono("db." + name + ".drop", "Drop a database", timer, "db.*.drop");
      }
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public boolean check(final boolean verbose, final OCommandOutputListener listener) {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      dataLock.acquireExclusiveLock();
      try {
        checkOpeness();
        final long start = System.currentTimeMillis();

        OPageDataVerificationError[] pageErrors = writeCache.checkStoredPages(verbose ? listener : null);

        listener.onMessage("Check of storage completed in " + (System.currentTimeMillis() - start) + "ms. "
            + (pageErrors.length > 0 ? pageErrors.length + " with errors." : " without errors."));

        return pageErrors.length == 0;
      } finally {
        dataLock.releaseExclusiveLock();
      }
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public void enableFullCheckPointAfterClusterCreate() {
    checkOpeness();
    stateLock.acquireWriteLock();
    try {
      checkOpeness();
      makeFullCheckPointAfterClusterCreate = true;
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public void disableFullCheckPointAfterClusterCreate() {
    checkOpeness();
    stateLock.acquireWriteLock();
    try {

      checkOpeness();
      makeFullCheckPointAfterClusterCreate = false;
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public boolean isMakeFullCheckPointAfterClusterCreate() {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();

      return makeFullCheckPointAfterClusterCreate;
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public int addCluster(String clusterName, boolean forceListBased, final Object... parameters) {
    checkOpeness();
    checkLowDiskSpaceAndFullCheckpointRequests();

    stateLock.acquireWriteLock();
    try {
      checkOpeness();

      makeStorageDirty();
      return doAddCluster(clusterName, true, parameters);

    } catch (Exception e) {
      throw new OStorageException("Error in creation of new cluster '" + clusterName, e);
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public int addCluster(String clusterName, int requestedId, boolean forceListBased, Object... parameters) {
    checkOpeness();
    checkLowDiskSpaceAndFullCheckpointRequests();
    stateLock.acquireWriteLock();
    try {
      checkOpeness();

      if (requestedId < 0) {
        throw new OConfigurationException("Cluster id must be positive!");
      }
      if (requestedId < clusters.size() && clusters.get(requestedId) != null) {
        throw new OConfigurationException("Requested cluster ID [" + requestedId + "] is occupied by cluster with name ["
            + clusters.get(requestedId).getName() + "]");
      }

      makeStorageDirty();
      return addClusterInternal(clusterName, requestedId, true, parameters);

    } catch (Exception e) {
      throw new OStorageException("Error in creation of new cluster '" + clusterName + "'", e);
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public boolean dropCluster(final int clusterId, final boolean iTruncate) {
    checkOpeness();
    checkLowDiskSpaceAndFullCheckpointRequests();

    stateLock.acquireWriteLock();
    try {

      checkOpeness();
      if (clusterId < 0 || clusterId >= clusters.size())
        throw new IllegalArgumentException("Cluster id '" + clusterId + "' is outside the of range of configured clusters (0-"
            + (clusters.size() - 1) + ") in database '" + name + "'");

      final OCluster cluster = clusters.get(clusterId);
      if (cluster == null)
        return false;

      if (iTruncate)
        cluster.truncate();
      cluster.delete();

      makeStorageDirty();
      clusterMap.remove(cluster.getName().toLowerCase());
      clusters.set(clusterId, null);

      // UPDATE CONFIGURATION
      configuration.dropCluster(clusterId);

      makeFullCheckpoint();
      return true;
    } catch (Exception e) {
      throw new OStorageException("Error while removing cluster '" + clusterId + "'", e);

    } finally {
      stateLock.releaseWriteLock();
    }
  }

  @Override
  public int getId() {
    return id;
  }

  public boolean setClusterStatus(final int clusterId, final OStorageClusterConfiguration.STATUS iStatus) {
    checkOpeness();
    stateLock.acquireWriteLock();
    try {
      checkOpeness();
      if (clusterId < 0 || clusterId >= clusters.size())
        throw new IllegalArgumentException("Cluster id '" + clusterId + "' is outside the of range of configured clusters (0-"
            + (clusters.size() - 1) + ") in database '" + name + "'");

      final OCluster cluster = clusters.get(clusterId);
      if (cluster == null)
        return false;

      if (iStatus == OStorageClusterConfiguration.STATUS.OFFLINE && cluster instanceof OOfflineCluster
          || iStatus == OStorageClusterConfiguration.STATUS.ONLINE && !(cluster instanceof OOfflineCluster))
        return false;

      final OCluster newCluster;
      if (iStatus == OStorageClusterConfiguration.STATUS.OFFLINE) {
        cluster.close(true);
        newCluster = new OOfflineCluster(this, clusterId, cluster.getName());
      } else {

        newCluster = OPaginatedClusterFactory.INSTANCE.createCluster(cluster.getName(), configuration.version, this);
        newCluster.configure(this, clusterId, cluster.getName());
        newCluster.open();
      }

      clusterMap.put(cluster.getName().toLowerCase(), newCluster);
      clusters.set(clusterId, newCluster);

      // UPDATE CONFIGURATION
      makeStorageDirty();
      configuration.setClusterStatus(clusterId, iStatus);

      makeFullCheckpoint();
      return true;
    } catch (Exception e) {
      throw new OStorageException("Error while removing cluster '" + clusterId + "'", e);
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  @Override
  public Class<OSBTreeCollectionManagerShared> getCollectionManagerClass() {
    return OSBTreeCollectionManagerShared.class;
  }

  public OReadCache getReadCache() {
    return readCache;
  }

  public OWriteCache getWriteCache() {
    return writeCache;
  }

  public void freeze(boolean throwException, int clusterId) {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();

      final OCluster cluster = getClusterById(clusterId);

      final String name = cluster.getName();
      if (OMetadataDefault.CLUSTER_INDEX_NAME.equals(name) || OMetadataDefault.CLUSTER_MANUAL_INDEX_NAME.equals(name)) {
        throw new IllegalArgumentException("It is impossible to freeze and release index or manual index cluster!");
      }

      cluster.getExternalModificationLock().prohibitModifications(throwException);

      try {
        cluster.synch();
        cluster.setSoftlyClosed(true);
      } catch (IOException e) {
        throw new OStorageException("Error on synch cluster '" + name + "'", e);
      }
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public void release(int clusterId) {
    stateLock.acquireReadLock();
    try {
      final OCluster cluster = getClusterById(clusterId);

      final String name = cluster.getName();
      if (OMetadataDefault.CLUSTER_INDEX_NAME.equals(name) || OMetadataDefault.CLUSTER_MANUAL_INDEX_NAME.equals(name)) {
        throw new IllegalArgumentException("It is impossible to freeze and release index or manualindex cluster!");
      }

      try {
        cluster.setSoftlyClosed(false);
      } catch (IOException e) {
        throw new OStorageException("Error on unfreeze storage '" + name + "'", e);
      }

      cluster.getExternalModificationLock().allowModifications();
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public long count(final int iClusterId) {
    return count(iClusterId, false);
  }

  @Override
  public long count(int clusterId, boolean countTombstones) {
    if (clusterId == -1)
      throw new OStorageException("Cluster Id " + clusterId + " is invalid in database '" + name + "'");

    // COUNT PHYSICAL CLUSTER IF ANY
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      dataLock.acquireSharedLock();
      try {
        checkOpeness();

        final OCluster cluster = clusters.get(clusterId);
        if (cluster == null)
          return 0;

        if (countTombstones)
          return cluster.getEntries();
<commit_msg>Softly closed was removed.<commit_after>/*
 *
 *  *  Copyright 2014 Orient Technologies LTD (info(at)orientechnologies.com)
 *  *
 *  *  Licensed under the Apache License, Version 2.0 (the "License");
 *  *  you may not use this file except in compliance with the License.
 *  *  You may obtain a copy of the License at
 *  *
 *  *       http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *  Unless required by applicable law or agreed to in writing, software
 *  *  distributed under the License is distributed on an "AS IS" BASIS,
 *  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *  See the License for the specific language governing permissions and
 *  *  limitations under the License.
 *  *
 *  * For more information: http://www.orientechnologies.com
 *
 */

package com.orientechnologies.orient.core.storage.impl.local;

import com.orientechnologies.common.concur.lock.OLockManager;
import com.orientechnologies.common.concur.lock.OModificationLock;
import com.orientechnologies.common.exception.OException;
import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.common.types.OModifiableBoolean;
import com.orientechnologies.common.util.OCommonConst;
import com.orientechnologies.orient.core.OOrientShutdownListener;
import com.orientechnologies.orient.core.OOrientStartupListener;
import com.orientechnologies.orient.core.Orient;
import com.orientechnologies.orient.core.command.OCommandExecutor;
import com.orientechnologies.orient.core.command.OCommandManager;
import com.orientechnologies.orient.core.command.OCommandOutputListener;
import com.orientechnologies.orient.core.command.OCommandRequestText;
import com.orientechnologies.orient.core.config.OGlobalConfiguration;
import com.orientechnologies.orient.core.config.OStorageClusterConfiguration;
import com.orientechnologies.orient.core.config.OStoragePaginatedClusterConfiguration;
import com.orientechnologies.orient.core.conflict.ORecordConflictStrategy;
import com.orientechnologies.orient.core.db.ODatabaseDocumentInternal;
import com.orientechnologies.orient.core.db.ODatabaseListener;
import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal;
import com.orientechnologies.orient.core.db.record.OCurrentStorageComponentsFactory;
import com.orientechnologies.orient.core.db.record.ORecordOperation;
import com.orientechnologies.orient.core.db.record.ridbag.sbtree.OSBTreeCollectionManagerShared;
import com.orientechnologies.orient.core.exception.OCommandExecutionException;
import com.orientechnologies.orient.core.exception.OConcurrentModificationException;
import com.orientechnologies.orient.core.exception.OConfigurationException;
import com.orientechnologies.orient.core.exception.OFastConcurrentModificationException;
import com.orientechnologies.orient.core.exception.OLowDiskSpaceException;
import com.orientechnologies.orient.core.exception.ORecordNotFoundException;
import com.orientechnologies.orient.core.exception.OStorageException;
import com.orientechnologies.orient.core.id.ORID;
import com.orientechnologies.orient.core.id.ORecordId;
import com.orientechnologies.orient.core.metadata.OMetadataDefault;
import com.orientechnologies.orient.core.metadata.OMetadataInternal;
import com.orientechnologies.orient.core.metadata.schema.OClass;
import com.orientechnologies.orient.core.metadata.security.OSecurityUser;
import com.orientechnologies.orient.core.metadata.security.OToken;
import com.orientechnologies.orient.core.record.ORecord;
import com.orientechnologies.orient.core.record.ORecordInternal;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.record.impl.ODocumentInternal;
import com.orientechnologies.orient.core.storage.OCluster;
import com.orientechnologies.orient.core.storage.OIdentifiableStorage;
import com.orientechnologies.orient.core.storage.OPhysicalPosition;
import com.orientechnologies.orient.core.storage.ORawBuffer;
import com.orientechnologies.orient.core.storage.ORecordCallback;
import com.orientechnologies.orient.core.storage.ORecordMetadata;
import com.orientechnologies.orient.core.storage.OStorageAbstract;
import com.orientechnologies.orient.core.storage.OStorageOperationResult;
import com.orientechnologies.orient.core.storage.cache.OCacheEntry;
import com.orientechnologies.orient.core.storage.cache.OCachePointer;
import com.orientechnologies.orient.core.storage.cache.OPageDataVerificationError;
import com.orientechnologies.orient.core.storage.cache.OReadCache;
import com.orientechnologies.orient.core.storage.cache.OWriteCache;
import com.orientechnologies.orient.core.storage.impl.local.paginated.OOfflineCluster;
import com.orientechnologies.orient.core.storage.impl.local.paginated.OOfflineClusterException;
import com.orientechnologies.orient.core.storage.impl.local.paginated.ORecordSerializationContext;
import com.orientechnologies.orient.core.storage.impl.local.paginated.OStorageTransaction;
import com.orientechnologies.orient.core.storage.impl.local.paginated.atomicoperations.OAtomicOperationsManager;
import com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage;
import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.*;
import com.orientechnologies.orient.core.tx.OTransaction;
import com.orientechnologies.orient.core.tx.OTransactionAbstract;
import com.orientechnologies.orient.core.tx.OTxListener;
import com.orientechnologies.orient.core.type.tree.provider.OMVRBTreeRIDProvider;
import com.orientechnologies.orient.core.version.ORecordVersion;
import com.orientechnologies.orient.core.version.OSimpleVersion;
import com.orientechnologies.orient.core.version.OVersionFactory;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * @author <NAME>
 * @since 28.03.13
 */
public abstract class OAbstractPaginatedStorage extends OStorageAbstract implements OLowDiskSpaceListener,
    OFullCheckpointRequestListener, OIdentifiableStorage, OOrientStartupListener, OOrientShutdownListener {
  private static final int RECORD_LOCK_TIMEOUT = OGlobalConfiguration.STORAGE_RECORD_LOCK_TIMEOUT.getValueAsInteger();

  private final OLockManager<ORID>                  lockManager;
  private final String                              PROFILER_CREATE_RECORD;
  private final String                              PROFILER_READ_RECORD;
  private final String                              PROFILER_UPDATE_RECORD;
  private final String                              PROFILER_DELETE_RECORD;
  private final Map<String, OCluster>               clusterMap           = new HashMap<String, OCluster>();
  private volatile ThreadLocal<OStorageTransaction> transaction          = new ThreadLocal<OStorageTransaction>();
  private final OModificationLock                   modificationLock     = new OModificationLock();
  private final AtomicBoolean                       checkpointInProgress = new AtomicBoolean();
  protected volatile OWriteAheadLog                 writeAheadLog;

  protected volatile OReadCache  readCache;
  protected volatile OWriteCache writeCache;

  private volatile ORecordConflictStrategy  recordConflictStrategy                     = Orient.instance()
      .getRecordConflictStrategy().newInstanceOfDefaultClass();
  private List<OCluster>                    clusters                                   = new ArrayList<OCluster>();
  private volatile int                      defaultClusterId                           = -1;
  private volatile OAtomicOperationsManager atomicOperationsManager;
  private volatile boolean                  wereDataRestoredAfterOpen                  = false;
  private volatile boolean                  wereNonTxOperationsPerformedInPreviousOpen = false;
  private boolean                           makeFullCheckPointAfterClusterCreate       = OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CLUSTER_CREATE
      .getValueAsBoolean();
  private volatile OLowDiskSpaceInformation lowDiskSpace                               = null;
  private volatile boolean                  checkpointRequest                          = false;

  private final int id;

  public OAbstractPaginatedStorage(String name, String filePath, String mode, int id) {
    super(name, filePath, mode, OGlobalConfiguration.STORAGE_LOCK_TIMEOUT.getValueAsInteger());

    this.id = id;
    lockManager = new OLockManager<ORID>(true, -1) {
      @Override
      protected ORID getImmutableResourceId(ORID iResourceId) {
        return new ORecordId(iResourceId);
      }
    };

    PROFILER_CREATE_RECORD = "db." + this.name + ".createRecord";
    PROFILER_READ_RECORD = "db." + this.name + ".readRecord";
    PROFILER_UPDATE_RECORD = "db." + this.name + ".updateRecord";
    PROFILER_DELETE_RECORD = "db." + this.name + ".deleteRecord";
  }

  public void open(final String iUserName, final String iUserPassword, final Map<String, Object> iProperties) {
    stateLock.acquireReadLock();
    try {
      if (status == STATUS.OPEN)
        // ALREADY OPENED: THIS IS THE CASE WHEN A STORAGE INSTANCE IS
        // REUSED
        return;
    } finally {
      stateLock.releaseReadLock();
    }

    stateLock.acquireWriteLock();
    try {

      if (status == STATUS.OPEN)
        // ALREADY OPENED: THIS IS THE CASE WHEN A STORAGE INSTANCE IS
        // REUSED
        return;

      if (!exists())
        throw new OStorageException("Cannot open the storage '" + name + "' because it does not exist in path: " + url);

      configuration.load();
      componentsFactory = new OCurrentStorageComponentsFactory(configuration);

      preOpenSteps();

      initWalAndDiskCache();

      atomicOperationsManager = new OAtomicOperationsManager(this);
      try {
        atomicOperationsManager.registerMBean();
      } catch (Exception e) {
        OLogManager.instance().error(this, "MBean for atomic operations manager cannot be registered.", e);
      }

      restoreIfNeeded();

      // OPEN BASIC SEGMENTS
      int pos;
      addDefaultClusters();

      // REGISTER CLUSTER
      for (int i = 0; i < configuration.clusters.size(); ++i) {
        final OStorageClusterConfiguration clusterConfig = configuration.clusters.get(i);

        if (clusterConfig != null) {
          pos = createClusterFromConfig(clusterConfig);

          try {
            if (pos == -1) {
              clusters.get(i).open();
            } else {
              if (clusterConfig.getName().equals(CLUSTER_DEFAULT_NAME))
                defaultClusterId = pos;

              clusters.get(pos).open();
            }
          } catch (FileNotFoundException e) {
            OLogManager.instance().warn(this, "Error on loading cluster '" + clusters.get(i).getName() + "' (" + i
                + "): file not found. It will be excluded from current database '" + getName() + "'.");

            clusterMap.remove(clusters.get(i).getName().toLowerCase());

            setCluster(i, null);
          }
        } else {
          setCluster(i, null);
        }
      }

      if (OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_OPEN.getValueAsBoolean())
        makeFullCheckpoint();

      writeCache.startFuzzyCheckpoints();

      status = STATUS.OPEN;
    } catch (Exception e) {
      status = STATUS.CLOSED;
      throw new OStorageException("Cannot open local storage '" + url + "' with mode=" + mode, e);
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public void open(final OToken iToken, final Map<String, Object> iProperties) {
    open(iToken.getUserName(), "", iProperties);
  }

  public void create(final Map<String, Object> iProperties) {
    stateLock.acquireWriteLock();
    try {

      if (status != STATUS.CLOSED)
        throw new OStorageException("Cannot create new storage '" + getURL() + "' because it is not closed");

      if (exists())
        throw new OStorageException("Cannot create new storage '" + getURL() + "' because it already exists");

      if (!configuration.getContextConfiguration().getContextKeys()
          .contains(OGlobalConfiguration.STORAGE_COMPRESSION_METHOD.getKey()))

        // SAVE COMPRESSION IN STORAGE CFG
        configuration.getContextConfiguration().setValue(OGlobalConfiguration.STORAGE_COMPRESSION_METHOD,
            OGlobalConfiguration.STORAGE_COMPRESSION_METHOD.getValue());

      componentsFactory = new OCurrentStorageComponentsFactory(configuration);
      initWalAndDiskCache();

      atomicOperationsManager = new OAtomicOperationsManager(this);
      try {
        atomicOperationsManager.registerMBean();
      } catch (Exception e) {
        OLogManager.instance().error(this, "MBean for atomic operations manager cannot be registered.", e);
      }

      preCreateSteps();

      status = STATUS.OPEN;

      // ADD THE METADATA CLUSTER TO STORE INTERNAL STUFF
      doAddCluster(OMetadataDefault.CLUSTER_INTERNAL_NAME, false, null);

      configuration.create();

      // ADD THE INDEX CLUSTER TO STORE, BY DEFAULT, ALL THE RECORDS OF
      // INDEXING
      doAddCluster(OMetadataDefault.CLUSTER_INDEX_NAME, false, null);

      // ADD THE INDEX CLUSTER TO STORE, BY DEFAULT, ALL THE RECORDS OF
      // INDEXING
      doAddCluster(OMetadataDefault.CLUSTER_MANUAL_INDEX_NAME, false, null);

      // ADD THE DEFAULT CLUSTER
      defaultClusterId = doAddCluster(CLUSTER_DEFAULT_NAME, false, null);

      clearStorageDirty();
      if (OGlobalConfiguration.STORAGE_MAKE_FULL_CHECKPOINT_AFTER_CREATE.getValueAsBoolean())
        makeFullCheckpoint();

      writeCache.startFuzzyCheckpoints();
      postCreateSteps();

    } catch (OStorageException e) {
      close();
      throw e;
    } catch (IOException e) {
      close();
      throw new OStorageException("Error on creation of storage '" + name + "'", e);
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  @Override
  public void onShutdown() {
    transaction = null;
  }

  @Override
  public void onStartup() {
    if (transaction == null)
      transaction = new ThreadLocal<OStorageTransaction>();
  }

  public void startAtomicOperation(boolean rollbackOnlyMode) throws IOException {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();
      makeStorageDirty();

      atomicOperationsManager.startAtomicOperation((String) null, rollbackOnlyMode);
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public void commitAtomicOperation() throws IOException {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();
      atomicOperationsManager.endAtomicOperation(false, null);
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public void rollbackAtomicOperation() throws IOException {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();
      atomicOperationsManager.endAtomicOperation(true, null);
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public void markDirty() throws IOException {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();
      makeStorageDirty();
    } finally {
      stateLock.releaseReadLock();
    }
  }

  @Override
  public void close(final boolean force, boolean onDelete) {
    doClose(force, onDelete);
  }

  public void delete() {
    final long timer = Orient.instance().getProfiler().startChrono();

    stateLock.acquireWriteLock();
    try {
      dataLock.acquireExclusiveLock();
      try {
        // CLOSE THE DATABASE BY REMOVING THE CURRENT USER
        doClose(true, true);

        try {
          Orient.instance().unregisterStorage(this);
        } catch (Exception e) {
          OLogManager.instance().error(this, "Cannot unregister storage", e);
        }

        if (writeAheadLog != null)
          writeAheadLog.delete();

        if (writeCache != null) {
          if (readCache != null)
            readCache.deleteStorage(writeCache);
          else
            writeCache.delete();
        }

        postDeleteSteps();

      } catch (IOException e) {
        throw new OStorageException("Cannot delete database '" + name + "'.", e);
      } finally {
        dataLock.releaseExclusiveLock();

        Orient.instance().getProfiler().stopChrono("db." + name + ".drop", "Drop a database", timer, "db.*.drop");
      }
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public boolean check(final boolean verbose, final OCommandOutputListener listener) {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      dataLock.acquireExclusiveLock();
      try {
        checkOpeness();
        final long start = System.currentTimeMillis();

        OPageDataVerificationError[] pageErrors = writeCache.checkStoredPages(verbose ? listener : null);

        listener.onMessage("Check of storage completed in " + (System.currentTimeMillis() - start) + "ms. "
            + (pageErrors.length > 0 ? pageErrors.length + " with errors." : " without errors."));

        return pageErrors.length == 0;
      } finally {
        dataLock.releaseExclusiveLock();
      }
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public void enableFullCheckPointAfterClusterCreate() {
    checkOpeness();
    stateLock.acquireWriteLock();
    try {
      checkOpeness();
      makeFullCheckPointAfterClusterCreate = true;
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public void disableFullCheckPointAfterClusterCreate() {
    checkOpeness();
    stateLock.acquireWriteLock();
    try {

      checkOpeness();
      makeFullCheckPointAfterClusterCreate = false;
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public boolean isMakeFullCheckPointAfterClusterCreate() {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();

      return makeFullCheckPointAfterClusterCreate;
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public int addCluster(String clusterName, boolean forceListBased, final Object... parameters) {
    checkOpeness();
    checkLowDiskSpaceAndFullCheckpointRequests();

    stateLock.acquireWriteLock();
    try {
      checkOpeness();

      makeStorageDirty();
      return doAddCluster(clusterName, true, parameters);

    } catch (Exception e) {
      throw new OStorageException("Error in creation of new cluster '" + clusterName, e);
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public int addCluster(String clusterName, int requestedId, boolean forceListBased, Object... parameters) {
    checkOpeness();
    checkLowDiskSpaceAndFullCheckpointRequests();
    stateLock.acquireWriteLock();
    try {
      checkOpeness();

      if (requestedId < 0) {
        throw new OConfigurationException("Cluster id must be positive!");
      }
      if (requestedId < clusters.size() && clusters.get(requestedId) != null) {
        throw new OConfigurationException("Requested cluster ID [" + requestedId + "] is occupied by cluster with name ["
            + clusters.get(requestedId).getName() + "]");
      }

      makeStorageDirty();
      return addClusterInternal(clusterName, requestedId, true, parameters);

    } catch (Exception e) {
      throw new OStorageException("Error in creation of new cluster '" + clusterName + "'", e);
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  public boolean dropCluster(final int clusterId, final boolean iTruncate) {
    checkOpeness();
    checkLowDiskSpaceAndFullCheckpointRequests();

    stateLock.acquireWriteLock();
    try {

      checkOpeness();
      if (clusterId < 0 || clusterId >= clusters.size())
        throw new IllegalArgumentException("Cluster id '" + clusterId + "' is outside the of range of configured clusters (0-"
            + (clusters.size() - 1) + ") in database '" + name + "'");

      final OCluster cluster = clusters.get(clusterId);
      if (cluster == null)
        return false;

      if (iTruncate)
        cluster.truncate();
      cluster.delete();

      makeStorageDirty();
      clusterMap.remove(cluster.getName().toLowerCase());
      clusters.set(clusterId, null);

      // UPDATE CONFIGURATION
      configuration.dropCluster(clusterId);

      makeFullCheckpoint();
      return true;
    } catch (Exception e) {
      throw new OStorageException("Error while removing cluster '" + clusterId + "'", e);

    } finally {
      stateLock.releaseWriteLock();
    }
  }

  @Override
  public int getId() {
    return id;
  }

  public boolean setClusterStatus(final int clusterId, final OStorageClusterConfiguration.STATUS iStatus) {
    checkOpeness();
    stateLock.acquireWriteLock();
    try {
      checkOpeness();
      if (clusterId < 0 || clusterId >= clusters.size())
        throw new IllegalArgumentException("Cluster id '" + clusterId + "' is outside the of range of configured clusters (0-"
            + (clusters.size() - 1) + ") in database '" + name + "'");

      final OCluster cluster = clusters.get(clusterId);
      if (cluster == null)
        return false;

      if (iStatus == OStorageClusterConfiguration.STATUS.OFFLINE && cluster instanceof OOfflineCluster
          || iStatus == OStorageClusterConfiguration.STATUS.ONLINE && !(cluster instanceof OOfflineCluster))
        return false;

      final OCluster newCluster;
      if (iStatus == OStorageClusterConfiguration.STATUS.OFFLINE) {
        cluster.close(true);
        newCluster = new OOfflineCluster(this, clusterId, cluster.getName());
      } else {

        newCluster = OPaginatedClusterFactory.INSTANCE.createCluster(cluster.getName(), configuration.version, this);
        newCluster.configure(this, clusterId, cluster.getName());
        newCluster.open();
      }

      clusterMap.put(cluster.getName().toLowerCase(), newCluster);
      clusters.set(clusterId, newCluster);

      // UPDATE CONFIGURATION
      makeStorageDirty();
      configuration.setClusterStatus(clusterId, iStatus);

      makeFullCheckpoint();
      return true;
    } catch (Exception e) {
      throw new OStorageException("Error while removing cluster '" + clusterId + "'", e);
    } finally {
      stateLock.releaseWriteLock();
    }
  }

  @Override
  public Class<OSBTreeCollectionManagerShared> getCollectionManagerClass() {
    return OSBTreeCollectionManagerShared.class;
  }

  public OReadCache getReadCache() {
    return readCache;
  }

  public OWriteCache getWriteCache() {
    return writeCache;
  }

  public void freeze(boolean throwException, int clusterId) {
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      checkOpeness();

      final OCluster cluster = getClusterById(clusterId);

      final String name = cluster.getName();
      if (OMetadataDefault.CLUSTER_INDEX_NAME.equals(name) || OMetadataDefault.CLUSTER_MANUAL_INDEX_NAME.equals(name)) {
        throw new IllegalArgumentException("It is impossible to freeze and release index or manual index cluster!");
      }

      cluster.getExternalModificationLock().prohibitModifications(throwException);

      try {
        cluster.synch();
      } catch (IOException e) {
        throw new OStorageException("Error on synch cluster '" + name + "'", e);
      }
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public void release(int clusterId) {
    stateLock.acquireReadLock();
    try {
      final OCluster cluster = getClusterById(clusterId);

      final String name = cluster.getName();
      if (OMetadataDefault.CLUSTER_INDEX_NAME.equals(name) || OMetadataDefault.CLUSTER_MANUAL_INDEX_NAME.equals(name)) {
        throw new IllegalArgumentException("It is impossible to freeze and release index or manualindex cluster!");
      }


      cluster.getExternalModificationLock().allowModifications();
    } finally {
      stateLock.releaseReadLock();
    }
  }

  public long count(final int iClusterId) {
    return count(iClusterId, false);
  }

  @Override
  public long count(int clusterId, boolean countTombstones) {
    if (clusterId == -1)
      throw new OStorageException("Cluster Id " + clusterId + " is invalid in database '" + name + "'");

    // COUNT PHYSICAL CLUSTER IF ANY
    checkOpeness();
    stateLock.acquireReadLock();
    try {
      dataLock.acquireSharedLock();
      try {
        checkOpeness();

        final OCluster cluster = clusters.get(clusterId);
        if (cluster == null)
          return 0;

        if (countTombstones)
          return cluster.getEntries();

{'fraction_non_alphanumeric': 0.09768119412515965, 'fraction_numerical': 0.0011574074074074073, 'mean_word_length': 3.3414190418435417, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6894717', 'n_tokens_mistral': 13917, 'n_tokens_neox': 13650, 'n_words': 2895}