<commit_before>﻿using System.Collections.Generic;
using System;
using FlatBuffers;
using System.Linq;

namespace Service.Serializer {
    public class FlatbufferSerializer {

        public static bool serializing = false;
        /// <summary>
        /// For deserializing mappings
        /// </summary>
        public static Dictionary<int, object> fb2objMapping = new Dictionary<int, object>();
        /// <summary>
        /// For serializing mappings
        /// </summary>
        public static Dictionary<object, int> obj2FSMapping = new Dictionary<object, int>();

        public static Dictionary<int, object> serializeBufferposCheck = new Dictionary<int, object>();

        private static FlatBufferBuilder fbBuilder;

        public static Dictionary<Type, Func<object, FlatBufferBuilder, int>> serializeObjConverters = new Dictionary<Type, Func<object, FlatBufferBuilder, int>>();
        public static Dictionary<Type, Func<object, FlatBufferBuilder, object>> deserializeObjConverters = new Dictionary<Type, Func<object, FlatBufferBuilder, object>>();

        public static bool convertersActivated = false;

        public static int? Convert(FlatBufferBuilder builder, object obj)  {
            if (!convertersActivated) {
                ActivateConverters();
                convertersActivated = true;
            }

            if (serializeObjConverters.TryGetValue(obj.GetType(),out Func<object, FlatBufferBuilder, int> converter)){
                var bufPos = converter(obj, builder);
                return bufPos;
            } else {
                return null;
            }
        }

        public static void ActivateConverters() {
            serializeObjConverters[typeof(UnityEngine.Vector2)] = (data,builder) => {
                var vec2 = (UnityEngine.Vector2)data;
                Serial.FBVector2.StartFBVector2(builder);
                Serial.FBVector2.AddX(builder, vec2.x);
                Serial.FBVector2.AddY(builder, vec2.y);
                return Serial.FBVector2.EndFBVector2(builder).Value;
            };
            serializeObjConverters[typeof(UnityEngine.Vector3)] = (data, builder) => {
                var vec3 = (UnityEngine.Vector3)data;
                Serial.FBVector3.StartFBVector3(builder);
                Serial.FBVector3.AddX(builder, vec3.x);
                Serial.FBVector3.AddY(builder, vec3.y);
                Serial.FBVector3.AddZ(builder, vec3.z);
                return Serial.FBVector3.EndFBVector3(builder).Value;
            };
            serializeObjConverters[typeof(UnityEngine.Vector4)] = (data, builder) => {
                var vec4 = (UnityEngine.Vector4)data;
                Serial.FBVector4.StartFBVector4(builder);
                Serial.FBVector4.AddX(builder, vec4.x);
                Serial.FBVector4.AddY(builder, vec4.y);
                Serial.FBVector4.AddZ(builder, vec4.z);
                Serial.FBVector4.AddW(builder, vec4.w);
                return Serial.FBVector4.EndFBVector4(builder).Value;
            };
            serializeObjConverters[typeof(UnityEngine.Quaternion)] = (data, builder) => {
                var q = (UnityEngine.Quaternion)data;
                Serial.FBQuaternion.StartFBQuaternion(builder);
                Serial.FBQuaternion.AddX(builder, q.x);
                Serial.FBQuaternion.AddY(builder, q.y);
                Serial.FBQuaternion.AddZ(builder, q.z);
                Serial.FBQuaternion.AddW(builder, q.w);
                return Serial.FBQuaternion.EndFBQuaternion(builder).Value;
            };
            serializeObjConverters[typeof(UnityEngine.Material)] = (data, builder) => {
                var material = (UnityEngine.Material)data;
                var fbMatName = builder.CreateString(material.name);

                Serial.FBMaterial.StartFBMaterial(builder);
                Serial.FBMaterial.AddMaterialName(builder, fbMatName);
                return Serial.FBMaterial.EndFBMaterial(builder).Value;
            };
        }

        /*
         var sIntDestcDict = FlatbufferSerializer.CreateDictionary<int,DestructionCosts,int, Offset<Serial.FBDestructionCosts>, Serial.DT_int_FBDestructionCosts>(builder, intDestcDict,  Serial.DT_int_FBDestructionCosts.CreateDT_int_FBDestructionCosts, Serial.FBTestComponent.CreateIntDestcDictVector);
         */
        public static FlatBuffers.VectorOffset? CreateDictionary<TKey, TValue,FBKey,FBValue,S>(FlatBuffers.FlatBufferBuilder builder
                                , Dictionary<TKey, TValue> dict
                                , Func<FlatBufferBuilder, FBKey,FBValue, Offset<S>> fbCreateElement
                                , Func<FlatBufferBuilder, Offset<S>[], VectorOffset> fbCreateList
                                )
                                where S : struct, FlatBuffers.IFlatbufferObject where FBValue : struct {
            if (dict == null) {
                return null;
            }


            int? bufferPos = FindInSerializeCache(dict);
            if (bufferPos.HasValue) {
                return new VectorOffset(bufferPos.Value);
            }

            var tempArray = new FlatBuffers.Offset<S>[dict.Count];
            int amount = dict.Count;
            if (typeof(TKey).IsPrimitive && typeof(TValue).IsPrimitive) {
                // a pure primitive dictionary
                for (int i = 0; i < amount; i++) {
                    var dictElem = dict.ElementAt(i);
                    tempArray[i] = fbCreateElement(builder, (FBKey)((object)dictElem.Key), (FBValue)((object)dictElem.Value));
                }
                var result = fbCreateList(builder, tempArray);
                PutInSerializeCache(dict, result.Value);
                return result;
            }
            else if (typeof(TKey).IsPrimitive && !typeof(TValue).IsPrimitive) {
                for (int i = 0; i < amount; i++) {
                    var dictElem = dict.ElementAt(i);
                    var valueElemOffset = FlatbufferSerializer.GetOrCreateSerialize(builder, (IFBSerializable)dictElem.Value);
                    var offset = (FBValue)Activator.CreateInstance(typeof(FBValue), valueElemOffset);
                    tempArray[i] = fbCreateElement(builder, (FBKey)((object)dictElem.Key), offset);
                }
                var result = fbCreateList(builder, tempArray);
                PutInSerializeCache(dict, result.Value);
                return result;
            } else if (!typeof(TKey).IsPrimitive && typeof(TValue).IsPrimitive) {
                // a pure non primitive dictionary
                for (int i = 0; i < amount; i++) {
                    var dictElem = dict.ElementAt(i);

                    var keyElemOffset = FlatbufferSerializer.GetOrCreateSerialize(builder, (IFBSerializable)dictElem.Key);
                    var offsetKey = (FBKey)Activator.CreateInstance(typeof(FBKey), keyElemOffset);

                    tempArray[i] = fbCreateElement(builder, offsetKey, (FBValue)((object)dictElem.Value));
                }
                var result = fbCreateList(builder, tempArray);
                PutInSerializeCache(dict, result.Value);
                return result;
            } else if (!typeof(TKey).IsPrimitive && !typeof(TValue).IsPrimitive) {
                // a pure non primitive dictionary
                for (int i = 0; i < amount; i++) {
                    var dictElem = dict.ElementAt(i);

                    var keyElemOffset = FlatbufferSerializer.GetOrCreateSerialize(builder, (IFBSerializable)dictElem.Key);
                    var offsetKey = (FBKey)Activator.CreateInstance(typeof(FBKey), keyElemOffset);

                    var valueElemOffset = FlatbufferSerializer.GetOrCreateSerialize(builder, (IFBSerializable)dictElem.Value);
                    var valueOffset = (FBValue)Activator.CreateInstance(typeof(FBValue), valueElemOffset);
                    tempArray[i] = fbCreateElement(builder, offsetKey, valueOffset);
                }
                var result = fbCreateList(builder, tempArray);
                PutInSerializeCache(dict, result.Value);
                return result;
            }
            return null;<commit_msg>serialization: strings in dictionaries<commit_after>﻿using System.Collections.Generic;
using System;
using FlatBuffers;
using System.Linq;

namespace Service.Serializer {
    public class FlatbufferSerializer {

        public static bool serializing = false;
        /// <summary>
        /// For deserializing mappings
        /// </summary>
        public static Dictionary<int, object> fb2objMapping = new Dictionary<int, object>();
        /// <summary>
        /// For serializing mappings
        /// </summary>
        public static Dictionary<object, int> obj2FSMapping = new Dictionary<object, int>();

        public static Dictionary<int, object> serializeBufferposCheck = new Dictionary<int, object>();

        private static FlatBufferBuilder fbBuilder;

        public static Dictionary<Type, Func<object, FlatBufferBuilder, int>> serializeObjConverters = new Dictionary<Type, Func<object, FlatBufferBuilder, int>>();
        public static Dictionary<Type, Func<object, FlatBufferBuilder, object>> deserializeObjConverters = new Dictionary<Type, Func<object, FlatBufferBuilder, object>>();

        public static bool convertersActivated = false;

        public static int? Convert(FlatBufferBuilder builder, object obj)  {
            if (!convertersActivated) {
                ActivateConverters();
                convertersActivated = true;
            }

            if (serializeObjConverters.TryGetValue(obj.GetType(),out Func<object, FlatBufferBuilder, int> converter)){
                var bufPos = converter(obj, builder);
                return bufPos;
            } else {
                return null;
            }
        }

        public static void ActivateConverters() {
            serializeObjConverters[typeof(UnityEngine.Vector2)] = (data,builder) => {
                var vec2 = (UnityEngine.Vector2)data;
                Serial.FBVector2.StartFBVector2(builder);
                Serial.FBVector2.AddX(builder, vec2.x);
                Serial.FBVector2.AddY(builder, vec2.y);
                return Serial.FBVector2.EndFBVector2(builder).Value;
            };
            serializeObjConverters[typeof(UnityEngine.Vector3)] = (data, builder) => {
                var vec3 = (UnityEngine.Vector3)data;
                Serial.FBVector3.StartFBVector3(builder);
                Serial.FBVector3.AddX(builder, vec3.x);
                Serial.FBVector3.AddY(builder, vec3.y);
                Serial.FBVector3.AddZ(builder, vec3.z);
                return Serial.FBVector3.EndFBVector3(builder).Value;
            };
            serializeObjConverters[typeof(UnityEngine.Vector4)] = (data, builder) => {
                var vec4 = (UnityEngine.Vector4)data;
                Serial.FBVector4.StartFBVector4(builder);
                Serial.FBVector4.AddX(builder, vec4.x);
                Serial.FBVector4.AddY(builder, vec4.y);
                Serial.FBVector4.AddZ(builder, vec4.z);
                Serial.FBVector4.AddW(builder, vec4.w);
                return Serial.FBVector4.EndFBVector4(builder).Value;
            };
            serializeObjConverters[typeof(UnityEngine.Quaternion)] = (data, builder) => {
                var q = (UnityEngine.Quaternion)data;
                Serial.FBQuaternion.StartFBQuaternion(builder);
                Serial.FBQuaternion.AddX(builder, q.x);
                Serial.FBQuaternion.AddY(builder, q.y);
                Serial.FBQuaternion.AddZ(builder, q.z);
                Serial.FBQuaternion.AddW(builder, q.w);
                return Serial.FBQuaternion.EndFBQuaternion(builder).Value;
            };
            serializeObjConverters[typeof(UnityEngine.Material)] = (data, builder) => {
                var material = (UnityEngine.Material)data;
                var fbMatName = builder.CreateString(material.name);

                Serial.FBMaterial.StartFBMaterial(builder);
                Serial.FBMaterial.AddMaterialName(builder, fbMatName);
                return Serial.FBMaterial.EndFBMaterial(builder).Value;
            };
        }

        /*
         var sIntDestcDict = FlatbufferSerializer.CreateDictionary<int,DestructionCosts,int, Offset<Serial.FBDestructionCosts>, Serial.DT_int_FBDestructionCosts>(builder, intDestcDict,  Serial.DT_int_FBDestructionCosts.CreateDT_int_FBDestructionCosts, Serial.FBTestComponent.CreateIntDestcDictVector);
         */
        public static FlatBuffers.VectorOffset? CreateDictionary<TKey, TValue,FBKey,FBValue,S>(FlatBuffers.FlatBufferBuilder builder
                                , Dictionary<TKey, TValue> dict
                                , Func<FlatBufferBuilder, FBKey,FBValue, Offset<S>> fbCreateElement
                                , Func<FlatBufferBuilder, Offset<S>[], VectorOffset> fbCreateList
                                )
                                where S : struct, FlatBuffers.IFlatbufferObject where FBValue : struct {
            if (dict == null) {
                return null;
            }


            int? bufferPos = FindInSerializeCache(dict);
            if (bufferPos.HasValue) {
                return new VectorOffset(bufferPos.Value);
            }

            var tempArray = new FlatBuffers.Offset<S>[dict.Count];
            int amount = dict.Count;
            if (typeof(TKey).IsPrimitive && typeof(TValue).IsPrimitive) {
                // a pure primitive dictionary
                for (int i = 0; i < amount; i++) {
                    var dictElem = dict.ElementAt(i);
                    tempArray[i] = fbCreateElement(builder, (FBKey)((object)dictElem.Key), (FBValue)((object)dictElem.Value));
                }
                var result = fbCreateList(builder, tempArray);
                PutInSerializeCache(dict, result.Value);
                return result;
            }
            else if (typeof(TKey).IsPrimitive && !typeof(TValue).IsPrimitive) {
                for (int i = 0; i < amount; i++) {
                    var dictElem = dict.ElementAt(i);

                    FBValue valueElemOffset;
                    if (typeof(TValue) == typeof(string)) {
                        valueElemOffset = (FBValue)(object)builder.CreateString((string)(object)dictElem.Value);
                    } else {
                        var offset = FlatbufferSerializer.GetOrCreateSerialize(builder, (IFBSerializable)dictElem.Value);
                        valueElemOffset = (FBValue)Activator.CreateInstance(typeof(FBValue), offset);
                    }
                    tempArray[i] = fbCreateElement(builder, (FBKey)((object)dictElem.Key), valueElemOffset);
                }
                var result = fbCreateList(builder, tempArray);
                PutInSerializeCache(dict, result.Value);
                return result;
            } else if (!typeof(TKey).IsPrimitive && typeof(TValue).IsPrimitive) {
                // a pure non primitive dictionary
                for (int i = 0; i < amount; i++) {
                    var dictElem = dict.ElementAt(i);

                    FBKey offsetKey;
                    if (typeof(TKey) == typeof(string)) {
                        offsetKey = (FBKey)(object)builder.CreateString((string)(object)dictElem.Key);
                    }else {
                        var keyElemOffset = FlatbufferSerializer.GetOrCreateSerialize(builder, (IFBSerializable)dictElem.Key);
                        offsetKey = (FBKey)Activator.CreateInstance(typeof(FBKey), keyElemOffset);
                    }

                    tempArray[i] = fbCreateElement(builder, offsetKey, (FBValue)((object)dictElem.Value));
                }
                var result = fbCreateList(builder, tempArray);
                PutInSerializeCache(dict, result.Value);
                return result;
            } else if (!typeof(TKey).IsPrimitive && !typeof(TValue).IsPrimitive) {
                // a pure non primitive dictionary
                for (int i = 0; i < amount; i++) {
                    var dictElem = dict.ElementAt(i);



                    FBKey offsetKey;
                    if (typeof(TKey) == typeof(string)) {
                        offsetKey = (FBKey)(object)builder.CreateString((string)(object)dictElem.Key);
                    } else {
                        var keyElemOffset = FlatbufferSerializer.GetOrCreateSerialize(builder, (IFBSerializable)dictElem.Key);
                        offsetKey = (FBKey)Activator.CreateInstance(typeof(FBKey), keyElemOffset);
                    }

                    FBValue valueElemOffset;
                    if (typeof(TValue) == typeof(string)) {
                        valueElemOffset = (FBValue)(object)builder.CreateString((string)(object)dictElem.Key);
                    } else {
                        var offset = FlatbufferSerializer.GetOrCreateSerialize(builder, (IFBSerializable)dictElem.Value);
                        valueElemOffset = (FBValue)Activator.CreateInstance(typeof(FBValue), offset);
                    }
                    tempArray[i] = fbCreateElement(builder, offsetKey, valueElemOffset); 
                }
                var result = fbCreateList(builder, tempArray);
                PutInSerializeCache(dict, result.Value);
                return result;
            }
            return null;
{'fraction_non_alphanumeric': 0.09880308435953504, 'fraction_numerical': 0.005178961905857982, 'mean_word_length': 1.975346687211094, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2564248', 'n_tokens_mistral': 4654, 'n_tokens_neox': 4445, 'n_words': 818}