<commit_before>/*
 *
 *  *  Copyright 2010-2016 OrientDB LTD (http://orientdb.com)
 *  *
 *  *  Licensed under the Apache License, Version 2.0 (the "License");
 *  *  you may not use this file except in compliance with the License.
 *  *  You may obtain a copy of the License at
 *  *
 *  *       http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *  Unless required by applicable law or agreed to in writing, software
 *  *  distributed under the License is distributed on an "AS IS" BASIS,
 *  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *  See the License for the specific language governing permissions and
 *  *  limitations under the License.
 *  *
 *  * For more information: http://orientdb.com
 *
 */

package com.orientechnologies.orient.core.storage.impl.local.paginated;

import com.orientechnologies.common.io.OIOUtils;
import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.orient.core.config.OGlobalConfiguration;
import com.orientechnologies.orient.core.exception.OStorageException;
import net.jpountz.xxhash.XXHash64;
import net.jpountz.xxhash.XXHashFactory;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * @author <NAME> (<EMAIL>)
 * @since 5/6/14
 */
public class StorageStartupMetadata {
  private static final long     XX_HASH_SEED = 0xADF678FE45L;
  private static final XXHash64 XX_HASH_64;

  private static final int XX_HASH_OFFSET        = 0;
  private static final int VERSION_OFFSET        = XX_HASH_OFFSET + 8;
  private static final int DIRTY_FLAG_OFFSET     = VERSION_OFFSET + 4;
  private static final int TRANSACTION_ID_OFFSET = DIRTY_FLAG_OFFSET + 1;
  private static final int METADATA_LEN_OFFSET   = TRANSACTION_ID_OFFSET + 8;

  static {
    final XXHashFactory xxHashFactory = XXHashFactory.fastestInstance();
    XX_HASH_64 = xxHashFactory.hash64();
  }

  private static final int VERSION = 3;

  private final Path        filePath;
  private       FileChannel channel;
  private       FileLock    fileLock;

  private volatile boolean dirtyFlag;
  private volatile long    lastTxId;
  private volatile byte[]  txMetadata;

  private final Lock lock = new ReentrantLock();

  public void addFileToArchive(ZipOutputStream zos, String name) throws IOException {
    final ZipEntry ze = new ZipEntry(name);
    zos.putNextEntry(ze);
    try {
      final ByteBuffer byteBuffer = serialize();
      byteBuffer.put(DIRTY_FLAG_OFFSET, (byte) 0);
      zos.write(byteBuffer.array());
    } finally {
      zos.closeEntry();
    }
  }

  public StorageStartupMetadata(Path filePath) {
    this.filePath = filePath;
  }

  public void create() throws IOException {
    lock.lock();
    try {

      if (Files.exists(filePath)) {
        Files.delete(filePath);
      }

      channel = FileChannel
          .open(filePath, StandardOpenOption.READ, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.SYNC);
      if (OGlobalConfiguration.FILE_LOCK.getValueAsBoolean()) {
        lockFile();
      }

      final ByteBuffer buffer = ByteBuffer.allocate(1 + 8 + 8 + 4 + 4);//version + dirty flag + transaction id + tx metadata len

      buffer.position(8);

      buffer.putInt(VERSION);
      //dirty flag
      buffer.put((byte) 1);
      //transaction id
      buffer.putLong(-1);
      //tx metadata len
      buffer.putInt(-1);

      final long xxHash = XX_HASH_64.hash(buffer, 8, buffer.capacity() - 8, XX_HASH_SEED);
      buffer.putLong(0, xxHash);

      buffer.rewind();

      OIOUtils.writeByteBuffer(buffer, channel, 0);

      dirtyFlag = true;
      lastTxId = -1;
    } finally {
      lock.unlock();
    }
  }

  private void lockFile() throws IOException {
    try {
      fileLock = channel.tryLock();
    } catch (OverlappingFileLockException e) {
      OLogManager.instance().warn(this, "File is already locked by other thread", e);
    }

    if (fileLock == null)
      throw new OStorageException("Database is locked by another process, please shutdown process and try again");
  }

  public boolean exists() {
    lock.lock();
    try {
      return Files.exists(filePath);
    } finally {
      lock.unlock();
    }
  }

  public void open() throws IOException {
    lock.lock();
    try {
      if (!Files.exists(filePath)) {
        OLogManager.instance().infoNoDb(this, "File with startup metadata does not exist, creating new one");
        create();
        return;
      } else {
        channel = FileChannel
            .open(filePath, StandardOpenOption.SYNC, StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE);

        final long size = channel.size();

        if (size == 1) {
          ByteBuffer buffer = ByteBuffer.allocate(1);
          OIOUtils.readByteBuffer(buffer, channel, 0, true);

          buffer.position(0);
          dirtyFlag = buffer.get() > 0;
        } else if (size == 9) {
          ByteBuffer buffer = ByteBuffer.allocate(8 + 1);
          OIOUtils.readByteBuffer(buffer, channel, 0, true);

          buffer.position(0);
          dirtyFlag = buffer.get() > 0;
          lastTxId = buffer.getLong();
        } else {
          final ByteBuffer buffer = ByteBuffer.allocate((int) size);
          OIOUtils.readByteBuffer(buffer, channel);

          buffer.rewind();

          final long xxHash = XX_HASH_64.hash(buffer, 8, buffer.capacity() - 8, XX_HASH_SEED);
          if (xxHash != buffer.getLong(0)) {<commit_msg>Bug of binary compatibility with previous versions of startup metadata was fixed.<commit_after>/*
 *
 *  *  Copyright 2010-2016 OrientDB LTD (http://orientdb.com)
 *  *
 *  *  Licensed under the Apache License, Version 2.0 (the "License");
 *  *  you may not use this file except in compliance with the License.
 *  *  You may obtain a copy of the License at
 *  *
 *  *       http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *  Unless required by applicable law or agreed to in writing, software
 *  *  distributed under the License is distributed on an "AS IS" BASIS,
 *  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *  See the License for the specific language governing permissions and
 *  *  limitations under the License.
 *  *
 *  * For more information: http://orientdb.com
 *
 */

package com.orientechnologies.orient.core.storage.impl.local.paginated;

import com.orientechnologies.common.io.OIOUtils;
import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.orient.core.config.OGlobalConfiguration;
import com.orientechnologies.orient.core.exception.OStorageException;
import net.jpountz.xxhash.XXHash64;
import net.jpountz.xxhash.XXHashFactory;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * @author <NAME> (<EMAIL>)
 * @since 5/6/14
 */
public class StorageStartupMetadata {
  private static final long     XX_HASH_SEED = 0xADF678FE45L;
  private static final XXHash64 XX_HASH_64;

  private static final int XX_HASH_OFFSET        = 0;
  private static final int VERSION_OFFSET        = XX_HASH_OFFSET + 8;
  private static final int DIRTY_FLAG_OFFSET     = VERSION_OFFSET + 4;
  private static final int TRANSACTION_ID_OFFSET = DIRTY_FLAG_OFFSET + 1;
  private static final int METADATA_LEN_OFFSET   = TRANSACTION_ID_OFFSET + 8;

  static {
    final XXHashFactory xxHashFactory = XXHashFactory.fastestInstance();
    XX_HASH_64 = xxHashFactory.hash64();
  }

  private static final int VERSION = 3;

  private final Path        filePath;
  private       FileChannel channel;
  private       FileLock    fileLock;

  private volatile boolean dirtyFlag;
  private volatile long    lastTxId;
  private volatile byte[]  txMetadata;

  private final Lock lock = new ReentrantLock();

  public void addFileToArchive(ZipOutputStream zos, String name) throws IOException {
    final ZipEntry ze = new ZipEntry(name);
    zos.putNextEntry(ze);
    try {
      final ByteBuffer byteBuffer = serialize();
      byteBuffer.put(DIRTY_FLAG_OFFSET, (byte) 0);
      zos.write(byteBuffer.array());
    } finally {
      zos.closeEntry();
    }
  }

  public StorageStartupMetadata(Path filePath) {
    this.filePath = filePath;
  }

  public void create() throws IOException {
    lock.lock();
    try {

      if (Files.exists(filePath)) {
        Files.delete(filePath);
      }

      channel = FileChannel
          .open(filePath, StandardOpenOption.READ, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.SYNC);
      if (OGlobalConfiguration.FILE_LOCK.getValueAsBoolean()) {
        lockFile();
      }

      final ByteBuffer buffer = ByteBuffer.allocate(1 + 8 + 8 + 4 + 4);//version + dirty flag + transaction id + tx metadata len

      buffer.position(8);

      buffer.putInt(VERSION);
      //dirty flag
      buffer.put((byte) 1);
      //transaction id
      buffer.putLong(-1);
      //tx metadata len
      buffer.putInt(-1);

      final long xxHash = XX_HASH_64.hash(buffer, 8, buffer.capacity() - 8, XX_HASH_SEED);
      buffer.putLong(0, xxHash);

      buffer.rewind();

      OIOUtils.writeByteBuffer(buffer, channel, 0);

      dirtyFlag = true;
      lastTxId = -1;
    } finally {
      lock.unlock();
    }
  }

  private void lockFile() throws IOException {
    try {
      fileLock = channel.tryLock();
    } catch (OverlappingFileLockException e) {
      OLogManager.instance().warn(this, "File is already locked by other thread", e);
    }

    if (fileLock == null)
      throw new OStorageException("Database is locked by another process, please shutdown process and try again");
  }

  public boolean exists() {
    lock.lock();
    try {
      return Files.exists(filePath);
    } finally {
      lock.unlock();
    }
  }

  public void open() throws IOException {
    lock.lock();
    try {
      if (!Files.exists(filePath)) {
        OLogManager.instance().infoNoDb(this, "File with startup metadata does not exist, creating new one");
        create();
        return;
      } else {
        channel = FileChannel
            .open(filePath, StandardOpenOption.SYNC, StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE);

        final long size = channel.size();

        if (size < 9) {
          ByteBuffer buffer = ByteBuffer.allocate(1);
          OIOUtils.readByteBuffer(buffer, channel, 0, true);

          buffer.position(0);
          dirtyFlag = buffer.get() > 0;
        } else if (size == 9) {
          ByteBuffer buffer = ByteBuffer.allocate(8 + 1);
          OIOUtils.readByteBuffer(buffer, channel, 0, true);

          buffer.position(0);
          dirtyFlag = buffer.get() > 0;
          lastTxId = buffer.getLong();
        } else {
          final ByteBuffer buffer = ByteBuffer.allocate((int) size);
          OIOUtils.readByteBuffer(buffer, channel);

          buffer.rewind();

          final long xxHash = XX_HASH_64.hash(buffer, 8, buffer.capacity() - 8, XX_HASH_SEED);
          if (xxHash != buffer.getLong(0)) {
{'fraction_non_alphanumeric': 0.10243277848911651, 'fraction_numerical': 0.012121212121212121, 'mean_word_length': 3.0082107423879574, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5533573', 'n_tokens_mistral': 3621, 'n_tokens_neox': 3453, 'n_words': 911}