<commit_before>/*
 * Copyright 1999-2012 <NAME> (l.garulli--at--orientechnologies.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.orientechnologies.common.directmemory;

import java.lang.reflect.Field;
import java.security.AccessController;
import java.security.PrivilegedAction;

import sun.misc.Unsafe;

import com.orientechnologies.common.serialization.types.OBinarySerializer;

/**
 * @author <NAME>
 * @since 2/4/13
 */
@SuppressWarnings("restriction")
public class OUnsafeMemory implements ODirectMemory {
  public static final OUnsafeMemory INSTANCE              = new OUnsafeMemory();

  private static final Unsafe       unsafe;
  private static final boolean      unaligned;

  private static final long         UNSAFE_COPY_THRESHOLD = 1024L * 1024L;

  static {
    unsafe = (Unsafe) AccessController.doPrivileged(new PrivilegedAction<Object>() {
      public Object run() {
        try {
          Field f = Unsafe.class.getDeclaredField("theUnsafe");
          f.setAccessible(true);
          return f.get(null);
        } catch (NoSuchFieldException e) {
          throw new Error();
        } catch (IllegalAccessException e) {
          throw new Error();
        }
      }
    });

    String arch = System.getProperty("os.arch");
    unaligned = arch.equals("i386") || arch.equals("x86") || arch.equals("amd64") || arch.equals("x86_64");
  }

  @Override
  public long allocate(byte[] bytes) {
    final long pointer = unsafe.allocateMemory(bytes.length);
    set(pointer, bytes, bytes.length);

    return pointer;
  }

  @Override
  public long allocate(long size) {
    return unsafe.allocateMemory(size);
  }

  @Override
  public void free(long pointer) {
    unsafe.freeMemory(pointer);
  }

  @Override
  public byte[] get(long pointer, int length) {
    final byte[] result = new byte[length];
    for (int i = 0; i < length; i++)
      result[i] = unsafe.getByte(pointer++);
    return result;
  }

  @Override
  public void get(long pointer, byte[] array, int arrayOffset, int length) {
    pointer += arrayOffset;
    for (int i = arrayOffset; i < length + arrayOffset; i++)
      array[i] = unsafe.getByte(pointer++);
  }

  @Override
  public void set(long pointer, byte[] content, int length) {
    for (int i = 0; i < length; i++)
      unsafe.putByte(pointer++, content[i]);
  }

  @Override
  public <T> T get(long pointer, OBinarySerializer<T> serializer) {
    return serializer.deserializeFromDirectMemory(this, pointer);
  }

  @Override
  public <T> void set(long pointer, T data, OBinarySerializer<T> serializer) {
    serializer.serializeInDirectMemory(data, this, pointer);
  }
<commit_msg>replace byte by byte copy with copyMemory function from unsafe to increase performance<commit_after>/*
 * Copyright 1999-2012 <NAME> (l.garulli--at--orientechnologies.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.orientechnologies.common.directmemory;

import java.lang.reflect.Field;
import java.security.AccessController;
import java.security.PrivilegedAction;

import sun.misc.Unsafe;

import com.orientechnologies.common.serialization.types.OBinarySerializer;

/**
 * @author <NAME>
 * @since 2/4/13
 */
@SuppressWarnings("restriction")
public class OUnsafeMemory implements ODirectMemory {
  public static final OUnsafeMemory INSTANCE              = new OUnsafeMemory();

  private static final Unsafe       unsafe;
  private static final boolean      unaligned;

  private static final long         UNSAFE_COPY_THRESHOLD = 1024L * 1024L;

  private static final int          NATIVE_BYTE_ARRAY_OFFSET;

  private static boolean            copyMemoryAllowed;

  static {
    unsafe = (Unsafe) AccessController.doPrivileged(new PrivilegedAction<Object>() {
      public Object run() {
        try {
          Field f = Unsafe.class.getDeclaredField("theUnsafe");
          f.setAccessible(true);
          return f.get(null);
        } catch (NoSuchFieldException e) {
          throw new Error();
        } catch (IllegalAccessException e) {
          throw new Error();
        }
      }
    });

    NATIVE_BYTE_ARRAY_OFFSET = unsafe.arrayBaseOffset(byte[].class);

    try {
      unsafe.getClass().getDeclaredMethod("copyMemory", Object.class, long.class, Object.class, long.class, long.class);
      copyMemoryAllowed = true;
    } catch (NoSuchMethodException e) {
      copyMemoryAllowed = false;
    }

    String arch = System.getProperty("os.arch");
    unaligned = arch.equals("i386") || arch.equals("x86") || arch.equals("amd64") || arch.equals("x86_64");
  }

  @Override
  public long allocate(byte[] bytes) {
    final long pointer = unsafe.allocateMemory(bytes.length);
    set(pointer, bytes, bytes.length);

    return pointer;
  }

  @Override
  public long allocate(long size) {
    return unsafe.allocateMemory(size);
  }

  @Override
  public void free(long pointer) {
    unsafe.freeMemory(pointer);
  }

  @Override
  public byte[] get(long pointer, final int length) {
    final byte[] result = new byte[length];
    if (copyMemoryAllowed) {
      unsafe.copyMemory(null, pointer, result, NATIVE_BYTE_ARRAY_OFFSET, length);
    } else {
      for (int i = 0; i < length; i++)
        result[i] = unsafe.getByte(pointer++);
    }
    return result;
  }

  @Override
  public void get(long pointer, byte[] array, int arrayOffset, int length) {
    pointer += arrayOffset;
    if (copyMemoryAllowed) {
      unsafe.copyMemory(null, pointer, array, arrayOffset + NATIVE_BYTE_ARRAY_OFFSET, length);
    } else {
      for (int i = arrayOffset; i < length + arrayOffset; i++)
        array[i] = unsafe.getByte(pointer++);
    }
  }

  @Override
  public void set(long pointer, byte[] content, int length) {
    if (copyMemoryAllowed) {
      unsafe.copyMemory(content, NATIVE_BYTE_ARRAY_OFFSET, null, pointer, length);
    } else {
      for (int i = 0; i < length; i++)
        unsafe.putByte(pointer++, content[i]);
    }
  }

  @Override
  public <T> T get(long pointer, OBinarySerializer<T> serializer) {
    return serializer.deserializeFromDirectMemory(this, pointer);
  }

  @Override
  public <T> void set(long pointer, T data, OBinarySerializer<T> serializer) {
    serializer.serializeInDirectMemory(data, this, pointer);
  }

{'fraction_non_alphanumeric': 0.10830921868540719, 'fraction_numerical': 0.010197051123053604, 'mean_word_length': 3.3565426170468187, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1796827', 'n_tokens_mistral': 2172, 'n_tokens_neox': 2092, 'n_words': 621}