<commit_before>package io.sqooba.traildb;

import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;

import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This class is used to perform native call to the TrailDB C library. Base on the available Python bindings.
 * 
 * @author <NAME>
 *
 */
public enum TrailDBNative implements TrailDBInterface {

    INSTANCE;

    /**
     * Convert a raw 16-byte UUID into its hexadecimal string representation.
     * 
     * @param rawUUID 16-byte UUID.
     * @return A 32-byte hexadecimal string representation.
     */
    public String UUIDHex(byte[] rawUUID) {
        return Hex.encodeHexString(rawUUID);
    }

    /**
     * Convert a 32-byte hexadecimal string representation of an UUID into a raw 16-byte UUID.
     * 
     * @param hexUUID The UUID to be converted.
     * @return The raw 16-byte UUID.
     */
    public byte[] UUIDRaw(String hexUUID) {
        byte[] b = null;
        try {
            b = Hex.decodeHex(hexUUID.toCharArray());
        } catch(DecoderException e) {
            LOGGER.error("Failed to convert hexstring to string.", e);
        }
        return b;
    }

    private static final Logger LOGGER = LoggerFactory.getLogger(TrailDBNative.class.getName());

    static {
        loadLib("traildbjava");
    }

    /**
     * Extract the library from the jar/project, copy it outside so we can load it because this is not possible from
     * inside the jar.
     *
     * FixMe: Check how it's done in Snappy
     *
     * @param name The name of the library, without prefix/suffix.
     */
    private static void loadLib(String name) {
        name = System.mapLibraryName(name);
        try {
            InputStream in = TrailDBNative.class.getResourceAsStream("/" + name);

            File dirOut = new File("TrailDBWrapper/");
            dirOut.mkdir();

            File fileOut = File.createTempFile("traildbjava", name.substring(name.indexOf(".")), dirOut);

            OutputStream out = FileUtils.openOutputStream(fileOut);
            IOUtils.copy(in, out);
            LOGGER.info("Lib copied to: " + fileOut.getAbsolutePath());
            in.close();
            out.close();

            fileOut.deleteOnExit();

            System.load(fileOut.getAbsolutePath());
        } catch(Exception e) {
            LOGGER.error("Failed to load library.", e);
            System.exit(-1);
        }
    }

    // ========================================================================
    // Construct a new TrailDB
    // ========================================================================

    /** tdb_cons *tdb_cons_init(void) */
    private native ByteBuffer tdbConsInit();

    /** tdb_error tdb_cons_open(tdb_const *cons, const char *root, const char **ofield_names, uint64_t num_ofields) */
    private native int tdbConsOpen(ByteBuffer cons, String root, String[] ofieldNames, long numOfields);

    /** void tdb_cons_close(tdb_cons *cons) */
    private native void tdbConsClose(ByteBuffer cons);

    /**
     * tdb_error tdb_cons_add(tdb_cons *cons,const uint8_t uuid[16],const uint64_t timestamp,const char **values,const
     * uint64_t *value_lengths)
     */
    private native int tdbConsAdd(ByteBuffer cons, byte[] uuid, long timestamp, String[] values,
            long[] valueLengths);

    /** tdb_error tdb_cons_append(tdb_cons *cons, const tdb *db) */
    private native int tdbConsAppend(ByteBuffer cons, ByteBuffer db);
<commit_msg>🚑 Fix issue #18.<commit_after>package io.sqooba.traildb;

import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;

import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This class is used to perform native call to the TrailDB C library. Base on the available Python bindings.
 * 
 * @author <NAME>
 *
 */
public enum TrailDBNative implements TrailDBInterface {

    INSTANCE;

    /**
     * Convert a raw 16-byte UUID into its hexadecimal string representation.
     * 
     * @param rawUUID 16-byte UUID.
     * @return A 32-byte hexadecimal string representation.
     */
    public String UUIDHex(byte[] rawUUID) {
        return Hex.encodeHexString(rawUUID);
    }

    /**
     * Convert a 32-byte hexadecimal string representation of an UUID into a raw 16-byte UUID.
     * 
     * @param hexUUID The UUID to be converted.
     * @return The raw 16-byte UUID.
     */
    public byte[] UUIDRaw(String hexUUID) {
        byte[] b = null;
        try {
            b = Hex.decodeHex(hexUUID.toCharArray());
        } catch(DecoderException e) {
            LOGGER.error("Failed to convert hexstring to string.", e);
        }
        return b;
    }

    private static final Logger LOGGER = LoggerFactory.getLogger(TrailDBNative.class.getName());

    static {
        loadLib("traildbjava");
    }

    /**
     * Extract the library from the jar/project, copy it outside so we can load it because this is not possible from
     * inside the jar.
     *
     * FixMe: Check how it's done in Snappy
     *
     * @param name The name of the library, without prefix/suffix.
     */
    private static void loadLib(String name) {
        name = System.mapLibraryName(name);
        File fileOut = null;
        try {
            InputStream in = TrailDBNative.class.getResourceAsStream("/" + name);

            File dirOut = new File("TrailDBWrapper/");
            dirOut.mkdir();

            fileOut = File.createTempFile("traildbjava", name.substring(name.indexOf(".")), dirOut);

            OutputStream out = FileUtils.openOutputStream(fileOut);
            IOUtils.copy(in, out);
            LOGGER.info("Lib copied to: " + fileOut.getAbsolutePath());
            in.close();
            out.close();

            fileOut.deleteOnExit();

            System.load(fileOut.getAbsolutePath());
        } catch(Exception e) {
            LOGGER.error("Failed to load library.", e);
            System.exit(-1);
        } finally {
            if (fileOut != null) {
                fileOut.delete();
            }
        }
    }

    // ========================================================================
    // Construct a new TrailDB
    // ========================================================================

    /** tdb_cons *tdb_cons_init(void) */
    private native ByteBuffer tdbConsInit();

    /** tdb_error tdb_cons_open(tdb_const *cons, const char *root, const char **ofield_names, uint64_t num_ofields) */
    private native int tdbConsOpen(ByteBuffer cons, String root, String[] ofieldNames, long numOfields);

    /** void tdb_cons_close(tdb_cons *cons) */
    private native void tdbConsClose(ByteBuffer cons);

    /**
     * tdb_error tdb_cons_add(tdb_cons *cons,const uint8_t uuid[16],const uint64_t timestamp,const char **values,const
     * uint64_t *value_lengths)
     */
    private native int tdbConsAdd(ByteBuffer cons, byte[] uuid, long timestamp, String[] values,
            long[] valueLengths);

    /** tdb_error tdb_cons_append(tdb_cons *cons, const tdb *db) */
    private native int tdbConsAppend(ByteBuffer cons, ByteBuffer db);

{'fraction_non_alphanumeric': 0.14112473347547974, 'fraction_numerical': 0.006663113006396588, 'mean_word_length': 2.702516033547114, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3626080', 'n_tokens_mistral': 2242, 'n_tokens_neox': 2158, 'n_words': 588}