<commit_before>package org.normandra.orientdb;

import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.db.record.ORecordLazyList;
import com.orientechnologies.orient.core.db.record.ORecordLazySet;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.record.impl.ODocument;
import org.normandra.meta.ColumnMeta;
import org.normandra.meta.DiscriminatorMeta;
import org.normandra.meta.EmbeddedCollectionMeta;
import org.normandra.meta.EntityContext;
import org.normandra.meta.EntityMeta;
import org.normandra.meta.JoinCollectionMeta;
import org.normandra.meta.JoinColumnMeta;
import org.normandra.meta.MappedColumnMeta;
import org.normandra.meta.SingleEntityContext;
import org.normandra.meta.TableMeta;
import org.normandra.util.ArraySet;
import org.normandra.util.DataUtils;

import java.io.Serializable;
import java.math.BigDecimal;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;

/**
 * collection of common orient utilities
 * <p>
 * User: bowen Date: 5/15/14
 */
public class OrientUtils
{
    private static final String genericEdgeLabel = "Edge";

    public static Object unpackValue(final ODocument document, final ColumnMeta column)
    {
        if (null == document || null == column)
        {
            return null;
        }

        final String name = column.getName();
        final Object raw = document.field(name);
        if (null == raw)
        {
            return null;
        }

        return OrientUtils.unpackRaw(column, raw);
    }

    public static Object unpackKey(final EntityContext context, final ODocument document)
    {
        if (null == context || null == document)
        {
            return null;
        }

        final Collection<ColumnMeta> keys = context.getPrimaryKeys();
        final Set<ColumnMeta> columns = new ArraySet<>(keys.size() + 1);
        columns.addAll(keys);
        for (final EntityMeta meta : context.getEntities())
        {
            final DiscriminatorMeta descrim = meta.getDiscriminator();
            if (descrim != null)
            {
                columns.add(descrim.getColumn());
            }
        }

        final Map<String, Object> data = new TreeMap<>();
        for (final ColumnMeta column : columns)
        {
            final Object value = unpackValue(document, column);
            if (value != null)
            {
                data.put(column.getName(), value);
            }
        }
        return context.getId().toKey(data);
    }

    public static Object unpackKey(final EntityMeta meta, final ODocument document)
    {
        if (null == meta || null == document)
        {
            return null;
        }

        final Collection<ColumnMeta> keys = new SingleEntityContext(meta).getPrimaryKeys();
        final Set<ColumnMeta> columns = new ArraySet<>(keys.size() + 1);
        columns.addAll(keys);
        final DiscriminatorMeta descrim = meta.getDiscriminator();
        if (descrim != null)
        {
            columns.add(descrim.getColumn());
        }

        final Map<String, Object> data = new TreeMap<>();
        for (final ColumnMeta column : columns)
        {
            final Object value = unpackValue(document, column);
            if (value != null)
            {
                data.put(column.getName(), value);
            }
        }
        return meta.getId().toKey(data);
    }

    public static Map<ColumnMeta, Object> unpackValues(final EntityContext context, final ODocument document)
    {
        if (null == context || null == document)
        {
            return Collections.emptyMap();
        }

        final Map<ColumnMeta, Object> data = new TreeMap<>();
        for (final TableMeta table : context.getPrimaryTables())
        {
            for (final ColumnMeta column : table.getColumns())
            {
                final String name = column.getName();
                final Object raw = document.field(name);
                final Object value = raw != null ? OrientUtils.unpackRaw(column, raw) : null;
                if (value != null)
                {
                    data.put(column, value);
                }
            }
        }
        return Collections.unmodifiableMap(data);
    }

    private static Object unpackRaw(final ColumnMeta column, final Object value)
    {
        if (null == column || null == value)
        {
            return null;
        }

        if (value instanceof ORecordLazySet)
        {
            return value;
        }
        if (value instanceof ORecordLazyList)
        {
            return value;
        }

        final Class<?> clazz = column.getType();

        if (column.isCollection() && value instanceof Collection)
        {
            final Collection list = (Collection) value;
            final Collection packed;
            if (value instanceof List)
            {
                packed = new ArrayList<>(list.size());
            }
            else
            {
                packed = new ArraySet<>(list.size());
            }
            for (final Object item : list)
            {
                final Object pack = unpackPrimitive(clazz, item);<commit_msg>fixed issue with saving embedded collections in orientdb<commit_after>package org.normandra.orientdb;

import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.db.record.ORecordLazyList;
import com.orientechnologies.orient.core.db.record.ORecordLazySet;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.record.impl.ODocument;
import org.normandra.meta.ColumnMeta;
import org.normandra.meta.DiscriminatorMeta;
import org.normandra.meta.EmbeddedCollectionMeta;
import org.normandra.meta.EntityContext;
import org.normandra.meta.EntityMeta;
import org.normandra.meta.JoinCollectionMeta;
import org.normandra.meta.JoinColumnMeta;
import org.normandra.meta.MappedColumnMeta;
import org.normandra.meta.SingleEntityContext;
import org.normandra.meta.TableMeta;
import org.normandra.util.ArraySet;
import org.normandra.util.DataUtils;

import java.io.Serializable;
import java.math.BigDecimal;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;

/**
 * collection of common orient utilities
 * <p>
 * User: bowen Date: 5/15/14
 */
public class OrientUtils
{
    private static final String genericEdgeLabel = "Edge";

    public static Object unpackValue(final ODocument document, final ColumnMeta column)
    {
        if (null == document || null == column)
        {
            return null;
        }

        final String name = column.getName();
        final Object raw = document.field(name);
        if (null == raw)
        {
            return null;
        }

        return OrientUtils.unpackRaw(column, raw);
    }

    public static Object unpackKey(final EntityContext context, final ODocument document)
    {
        if (null == context || null == document)
        {
            return null;
        }

        final Collection<ColumnMeta> keys = context.getPrimaryKeys();
        final Set<ColumnMeta> columns = new ArraySet<>(keys.size() + 1);
        columns.addAll(keys);
        for (final EntityMeta meta : context.getEntities())
        {
            final DiscriminatorMeta descrim = meta.getDiscriminator();
            if (descrim != null)
            {
                columns.add(descrim.getColumn());
            }
        }

        final Map<String, Object> data = new TreeMap<>();
        for (final ColumnMeta column : columns)
        {
            final Object value = unpackValue(document, column);
            if (value != null)
            {
                data.put(column.getName(), value);
            }
        }
        return context.getId().toKey(data);
    }

    public static Object unpackKey(final EntityMeta meta, final ODocument document)
    {
        if (null == meta || null == document)
        {
            return null;
        }

        final Collection<ColumnMeta> keys = new SingleEntityContext(meta).getPrimaryKeys();
        final Set<ColumnMeta> columns = new ArraySet<>(keys.size() + 1);
        columns.addAll(keys);
        final DiscriminatorMeta descrim = meta.getDiscriminator();
        if (descrim != null)
        {
            columns.add(descrim.getColumn());
        }

        final Map<String, Object> data = new TreeMap<>();
        for (final ColumnMeta column : columns)
        {
            final Object value = unpackValue(document, column);
            if (value != null)
            {
                data.put(column.getName(), value);
            }
        }
        return meta.getId().toKey(data);
    }

    public static Map<ColumnMeta, Object> unpackValues(final EntityContext context, final ODocument document)
    {
        if (null == context || null == document)
        {
            return Collections.emptyMap();
        }

        final Map<ColumnMeta, Object> data = new TreeMap<>();
        for (final TableMeta table : context.getPrimaryTables())
        {
            for (final ColumnMeta column : table.getColumns())
            {
                final String name = column.getName();
                final Object raw = document.field(name);
                final Object value = raw != null ? OrientUtils.unpackRaw(column, raw) : null;
                if (value != null)
                {
                    data.put(column, value);
                }
            }
        }
        return Collections.unmodifiableMap(data);
    }

    private static Object unpackRaw(final ColumnMeta column, final Object value)
    {
        if (null == column || null == value)
        {
            return null;
        }

        if (value instanceof ORecordLazySet)
        {
            return value;
        }
        if (value instanceof ORecordLazyList)
        {
            return value;
        }

        final Class<?> clazz = column.getType();

        if (column.isCollection() && value instanceof Collection)
        {
            final Collection list = (Collection) value;
            final Class<?> generic;
            if (column instanceof EmbeddedCollectionMeta)
            {
                generic = ((EmbeddedCollectionMeta) column).getGeneric();
            }
            else if (column instanceof JoinCollectionMeta)
            {
                generic = ((JoinCollectionMeta) column).getEntity().getPrimaryKey().getType();
            }
            else
            {
                generic = column.getType();
            }
            final Collection packed;
            if (value instanceof List)
            {
                packed = new ArrayList<>(list.size());
            }
            else
            {
                packed = new ArraySet<>(list.size());
            }
            for (final Object item : list)
            {
                final Object pack = unpackPrimitive(generic, item);
{'fraction_non_alphanumeric': 0.10008802816901409, 'fraction_numerical': 0.0012323943661971832, 'mean_word_length': 2.0385129713827226, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3867070', 'n_tokens_mistral': 3002, 'n_tokens_neox': 2934, 'n_words': 737}