<commit_before>// Copyright 2016 Yahoo Inc.
// Licensed under the terms of the New-BSD license. Please see LICENSE file in the project root for terms.
package com.yahoo.wildwest;

import java.lang.reflect.Field;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;

import sun.misc.Unsafe;
import sun.misc.VM;

public class MUnsafe {
    public static final Unsafe unsafe;
    public static final long charArrayBaseOffset;
    public static final long charArrayIndexScale;
    public static final long byteArrayBaseOffset;
    public static final long byteArrayIndexScale;
    public static final long ISA_HOLDER_OFFSET;
    // public static final long ISA_ADDR_OFFSET;

    static {
        try {
            Field f = Unsafe.class.getDeclaredField("theUnsafe");
            f.setAccessible(true);
            unsafe = (Unsafe) f.get(null);
            charArrayBaseOffset = unsafe.arrayBaseOffset(char[].class);
            charArrayIndexScale = unsafe.arrayIndexScale(char[].class);
            byteArrayBaseOffset = unsafe.arrayBaseOffset(byte[].class);
            byteArrayIndexScale = unsafe.arrayIndexScale(byte[].class);

            ISA_HOLDER_OFFSET = unsafe.objectFieldOffset(InetSocketAddress.class.getDeclaredField("holder"));



        } catch (ReflectiveOperationException | SecurityException | IllegalArgumentException e) {
            throw new Error(e);
        }
    }

    public static Unsafe getUnsafe() {
        return unsafe;
    }

    public static long charArrayOffset(int index) {
        return calculateOffset(index, charArrayBaseOffset, charArrayIndexScale);
    }

    public static long charArraySize(char[] from) {
        return from.length * charArrayIndexScale;
    }

    public static long byteArrayOffset(int index) {
        return calculateOffset(index, byteArrayBaseOffset, byteArrayIndexScale);
    }

    public static long byteArraySize(byte[] from) {
        return from.length * byteArrayIndexScale;
    }

    /**
     * The object referred to by o is an array, and the offset is an integer of the form B+N*S, where N is a valid index
     * into the array, and B and S are the values obtained by #arrayBaseOffset and #arrayIndexScale (respectively) from
     * the array's class. The value referred to is the Nth element of the array.
     * 
     * @param index into the array to find the offset for.
     * @param base base from Unsafe to use.
     * @param scale scale from Unsafe to use
     * @return value usable by copyMemory
     **/
    public static long calculateOffset(int index, long base, long scale) {
        return (base + (index * scale));
    }

    // Cribbed from DiretByteBuffer
    public static long allocate(long cap) {
        boolean isDirectMemoryPageAligned = VM.isDirectMemoryPageAligned();
        int pageSize = unsafe.pageSize();
        long size = Math.max(1L, (long) cap + (isDirectMemoryPageAligned ? pageSize : 0));
        long base = unsafe.allocateMemory(size);
        unsafe.setMemory(base, size, (byte) 0);

        long address = 0;
        if (isDirectMemoryPageAligned && (base % pageSize != 0)) {
            // Round up to page boundary
            address = base + pageSize - (base & (pageSize - 1));
        } else {
            address = base;
        }

        return address;
    }

    public static void copyMemory(long destAddress, long totalSize, byte[] from) {
        unsafe.copyMemory(from, byteArrayBaseOffset, null, index(destAddress, 0), totalSize);
    }

    public static void copyMemory(byte[] dest, long srcAddress, long totalSize) {
        unsafe.copyMemory(null, index(srcAddress, 0), dest, byteArrayBaseOffset, totalSize);
    }

    private static long index(long address, int i) {
        return address + ((long) i << 0);
    }

    public static void freeMemory(long address) {
        if (0 != address) {
            unsafe.freeMemory(address);
            address = 0;
        }
    }

    /**
     * Given a String, encode it into MissingFingers address,length tuple
     * 
     * @param s input
     * @return MissingFingers, allocated address + length
     */
    public static MissingFingers encodeString(String s) {
        if (null == s || 0 == s.length() || s.isEmpty()) {
            // null begets null
            return new MissingFingers(0, 0);
        }

        byte[] fromBytes = s.getBytes(StandardCharsets.UTF_8);

        // gotta null terminate
        long totalSize = byteArraySize(fromBytes) + 1;
        long destAddress = unsafe.allocateMemory(totalSize);
        copyMemory(destAddress, totalSize, fromBytes);

        return new MissingFingers(destAddress, totalSize);
    }

    /**
     * Given a UTF8 string pointed to by address of len bytes long, decode it, and then free the memory pointed at by
     * it.
     * 
     * @param srcAddress
     * @param len
     * @return Java String, null on null, 0 on empty string
     */
    public static String decodeStringAndFree(long srcAddress, long len) {
        try {
            return decodeString(srcAddress, len);
        } finally {
            if (0 != srcAddress) {
                unsafe.freeMemory(srcAddress);
            }
        }
    }

    /**
     * Given a UTF8 string pointed to by address of len bytes long, decode it
     * 
     * @param srcAddress
     * @param len
     * @return Java String, null on null, 0 on empty string
     */
    public static String decodeString(long srcAddress, long len) {
        if (0 == srcAddress) {
            return null;
        }

        if (0 == len) {
            return "";
        }<commit_msg>Debugging, we screwed up the pointers somewhere<commit_after>// Copyright 2016 Yahoo Inc.
// Licensed under the terms of the New-BSD license. Please see LICENSE file in the project root for terms.
package com.yahoo.wildwest;

import java.lang.reflect.Field;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;

import sun.misc.Unsafe;
import sun.misc.VM;

public class MUnsafe {
    public static final Unsafe unsafe;
    public static final long charArrayBaseOffset;
    public static final long charArrayIndexScale;
    public static final long byteArrayBaseOffset;
    public static final long byteArrayIndexScale;
    public static final long ISA_HOLDER_OFFSET;
    // public static final long ISA_ADDR_OFFSET;

    static {
        try {
            Field f = Unsafe.class.getDeclaredField("theUnsafe");
            f.setAccessible(true);
            unsafe = (Unsafe) f.get(null);
            charArrayBaseOffset = unsafe.arrayBaseOffset(char[].class);
            charArrayIndexScale = unsafe.arrayIndexScale(char[].class);
            byteArrayBaseOffset = unsafe.arrayBaseOffset(byte[].class);
            byteArrayIndexScale = unsafe.arrayIndexScale(byte[].class);

            ISA_HOLDER_OFFSET = unsafe.objectFieldOffset(InetSocketAddress.class.getDeclaredField("holder"));



        } catch (ReflectiveOperationException | SecurityException | IllegalArgumentException e) {
            throw new Error(e);
        }
    }

    public static Unsafe getUnsafe() {
        return unsafe;
    }

    public static long charArrayOffset(int index) {
        return calculateOffset(index, charArrayBaseOffset, charArrayIndexScale);
    }

    public static long charArraySize(char[] from) {
        return from.length * charArrayIndexScale;
    }

    public static long byteArrayOffset(int index) {
        return calculateOffset(index, byteArrayBaseOffset, byteArrayIndexScale);
    }

    public static long byteArraySize(byte[] from) {
        return from.length * byteArrayIndexScale;
    }

    /**
     * The object referred to by o is an array, and the offset is an integer of the form B+N*S, where N is a valid index
     * into the array, and B and S are the values obtained by #arrayBaseOffset and #arrayIndexScale (respectively) from
     * the array's class. The value referred to is the Nth element of the array.
     * 
     * @param index into the array to find the offset for.
     * @param base base from Unsafe to use.
     * @param scale scale from Unsafe to use
     * @return value usable by copyMemory
     **/
    public static long calculateOffset(int index, long base, long scale) {
        return (base + (index * scale));
    }

    // Cribbed from DiretByteBuffer
    public static long allocate(long cap) {
        boolean isDirectMemoryPageAligned = VM.isDirectMemoryPageAligned();
        int pageSize = unsafe.pageSize();
        long size = Math.max(1L, (long) cap + (isDirectMemoryPageAligned ? pageSize : 0));
        long base = unsafe.allocateMemory(size);
        unsafe.setMemory(base, size, (byte) 0);

        long address = 0;
        if (isDirectMemoryPageAligned && (base % pageSize != 0)) {
            // Round up to page boundary
            address = base + pageSize - (base & (pageSize - 1));
        } else {
            address = base;
        }

        return address;
    }

    public static void copyMemory(long destAddress, long totalSize, byte[] from) {
        unsafe.copyMemory(from, byteArrayBaseOffset, null, index(destAddress, 0), totalSize);
    }

    public static void copyMemory(byte[] dest, long srcAddress, long totalSize) {
        unsafe.copyMemory(null, index(srcAddress, 0), dest, byteArrayBaseOffset, totalSize);
    }

    private static long index(long address, int i) {
        return address + ((long) i << 0);
    }

    public static void freeMemory(long address) {
        if (0 != address) {
            unsafe.freeMemory(address);
            address = 0;
        }
    }

    /**
     * Given a String, encode it into MissingFingers address,length tuple
     * 
     * @param s input
     * @return MissingFingers, allocated address + length
     */
    public static MissingFingers encodeString(String s) {
        if (null == s || 0 == s.length() || s.isEmpty()) {
            // null begets null
            return new MissingFingers(0, 0);
        }

        byte[] fromBytes = s.getBytes(StandardCharsets.UTF_8);

        // gotta null terminate
        long totalSize = byteArraySize(fromBytes) + 1;
        long destAddress = unsafe.allocateMemory(totalSize);
        copyMemory(destAddress, totalSize, fromBytes);

        return new MissingFingers(destAddress, totalSize);
    }

    /**
     * Given a UTF8 string pointed to by address of len bytes long, decode it, and then free the memory pointed at by
     * it.
     * 
     * @param srcAddress
     * @param len
     * @return Java String, null on null, 0 on empty string
     */
    public static String decodeStringAndFree(long srcAddress, long len) {
        try {
            System.err.println("asked to decode " + srcAddress + " len " + len);
            srcAddress = 0;
            // return decodeString(srcAddress, len);
            return "";
        } finally {
            if (0 != srcAddress) {
                unsafe.freeMemory(srcAddress);
            }
        }
    }

    /**
     * Given a UTF8 string pointed to by address of len bytes long, decode it
     * 
     * @param srcAddress
     * @param len
     * @return Java String, null on null, 0 on empty string
     */
    public static String decodeString(long srcAddress, long len) {
        if (0 == srcAddress) {
            return null;
        }

        if (0 == len) {
            return "";
        }
{'fraction_non_alphanumeric': 0.08830486202365309, 'fraction_numerical': 0.004818221638195357, 'mean_word_length': 2.4458194989435555, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '361739', 'n_tokens_mistral': 3187, 'n_tokens_neox': 3044, 'n_words': 1013}