<commit_before>using System;
using System.Collections.Generic;
using System.Dynamic;
using System.IO;
using System.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Bson;

namespace InfiniMap
{
    public struct StreamInfo
    {
        public readonly byte[] BsonBuffer;

        public StreamInfo(byte[] bytes)
        {
            BsonBuffer = bytes;
        }

        public UInt32 GetOffset(long startPosition)
        {
            return (uint)startPosition + (uint)BsonBuffer.Length;
        }
    }

    public class BlockMetadata : DynamicObject
    {
        public Dictionary<string, dynamic> Dictionary;

        public BlockMetadata()
        {
            Dictionary = new Dictionary<string, dynamic>();
        }

        public override bool TryGetMember(GetMemberBinder binder, out object result)
        {
            return Dictionary.TryGetValue(binder.Name, out result);
        }

        public override bool TrySetMember(SetMemberBinder binder, object value)
        {
            if (value.GetType().IsPrimitive || value is string || value is DateTime)
            {
                Dictionary[binder.Name] = value;
                return true;
            }

            throw new NotSupportedException("Can only store primitive or string types as a value");
        }

        public StreamInfo Write()
        {
            if (Dictionary.Count >= 1)
            {
                var stream = new MemoryStream();
                var serializer = new JsonSerializer();
                var writer = new BsonWriter(stream);

                serializer.Serialize(writer, Dictionary);
                stream.Seek(0, SeekOrigin.Begin);
                return new StreamInfo(stream.GetBuffer());
            }
            else
            {
                return new StreamInfo(Enumerable.Empty<byte>().ToArray());
            }
        }
    }

    public class Block : ISerialize, IDeserialize
    {
        /// <summary>
        /// Combined BlockId and BlockMeta.
        /// </summary>
        public UInt32 BlockData { get; private set; }

        /// <summary>
        /// Holds the block ID
        /// </summary>
        public UInt16 BlockId
        {
            get { return (UInt16) (BlockData & 0xFFFF); }
            set { BlockData = (((UInt32)value)&0xFFFF) | ((UInt32) BlockMeta << 16); }
        }

        /// <summary>
        /// MetaId attached to the block.
        /// </summary>
        public UInt16 BlockMeta
        {
            get { return (UInt16) ((BlockData >> 16) & 0xFFFF); }
            set { BlockData = (((UInt32)BlockId) & 0xFFFF) | ((UInt32)value << 16); }
        }

        /// <summary>
        /// Quick access for a small set of block Flags
        /// </summary>
        public UInt32 Flags;

        /// <summary>
        /// Location in chunk metadata file for optional data
        /// </summary>
        public UInt64 TagDataLocation;

        /// <summary>
        /// Contains optional extended properties for this specific block instance.
        /// </summary>
        public BlockMetadata ExtendedMetadata = new BlockMetadata();

        public dynamic Metadata
        {
            get
            {
                if (ExtendedMetadata == null)
                {
                    throw new NullReferenceException("ExtendedMetaData was null");
                }
                return ExtendedMetadata;
            }
        }

        public Block() : this(0,0) { }

        public Block(UInt32 blockData, UInt32 flags)
        {
            BlockData = blockData;
            Flags = flags;
            TagDataLocation = 0;
        }

        public void SetTagOffset(uint position)
        {
            TagDataLocation = position;
        }

        public void Write(BinaryWriter stream)
        {
            stream.Write(BlockData);
            stream.Write(Flags);
            stream.Write(TagDataLocation);
        }

        public void Read(Stream stream)
        {
            using (var r = new BinaryReader(stream))
            {
                BlockData = r.ReadUInt32();
                Flags = r.ReadUInt32();
                TagDataLocation = r.ReadUInt64();
            }
        }
    }
}<commit_msg>Remove Tag offset data from the block.<commit_after>using System;
using System.Collections.Generic;
using System.Dynamic;
using System.IO;
using System.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Bson;

namespace InfiniMap
{
    public struct StreamInfo
    {
        public readonly byte[] BsonBuffer;

        public StreamInfo(byte[] bytes)
        {
            BsonBuffer = bytes;
        }
    }

    public class BlockMetadata : DynamicObject
    {
        public Dictionary<string, dynamic> Dictionary;

        public BlockMetadata()
        {
            Dictionary = new Dictionary<string, dynamic>();
        }

        public override bool TryGetMember(GetMemberBinder binder, out object result)
        {
            return Dictionary.TryGetValue(binder.Name, out result);
        }

        public override bool TrySetMember(SetMemberBinder binder, object value)
        {
            if (value.GetType().IsPrimitive || value is string || value is DateTime)
            {
                Dictionary[binder.Name] = value;
                return true;
            }

            throw new NotSupportedException("Can only store primitive or string types as a value");
        }

        public StreamInfo Write()
        {
            if (Dictionary.Count >= 1)
            {
                var stream = new MemoryStream();
                var serializer = new JsonSerializer();
                var writer = new BsonWriter(stream);

                serializer.Serialize(writer, Dictionary);
                stream.Seek(0, SeekOrigin.Begin);
                return new StreamInfo(stream.GetBuffer());
            }
            else
            {
                return new StreamInfo(Enumerable.Empty<byte>().ToArray());
            }
        }
    }

    public class Block : ISerialize, IDeserialize
    {
        /// <summary>
        /// Combined BlockId and BlockMeta.
        /// </summary>
        public UInt32 BlockData { get; private set; }

        /// <summary>
        /// Holds the block ID
        /// </summary>
        public UInt16 BlockId
        {
            get { return (UInt16) (BlockData & 0xFFFF); }
            set { BlockData = (((UInt32)value)&0xFFFF) | ((UInt32) BlockMeta << 16); }
        }

        /// <summary>
        /// MetaId attached to the block.
        /// </summary>
        public UInt16 BlockMeta
        {
            get { return (UInt16) ((BlockData >> 16) & 0xFFFF); }
            set { BlockData = (((UInt32)BlockId) & 0xFFFF) | ((UInt32)value << 16); }
        }

        /// <summary>
        /// Quick access for a small set of block Flags
        /// </summary>
        public UInt32 Flags { get; set; }

        /// <summary>
        /// Contains optional extended properties for this specific block instance.
        /// </summary>
        public BlockMetadata ExtendedMetadata = new BlockMetadata();

        public dynamic Metadata
        {
            get
            {
                if (ExtendedMetadata == null)
                {
                    throw new NullReferenceException("ExtendedMetaData was null");
                }
                return ExtendedMetadata;
            }
        }

        public Block() : this(0,0) { }

        public Block(UInt32 blockData, UInt32 flags)
        {
            BlockData = blockData;
            Flags = flags;
        }

        public StreamInfo GetMetadata()
        {
            return ExtendedMetadata.Write();
        }

        public void Write(BinaryWriter stream)
        {
            stream.Write(BlockData);
            stream.Write(Flags);
        }

        public void Read(Stream stream)
        {
            using (var r = new BinaryReader(stream))
            {
                BlockData = r.ReadUInt32();
                Flags = r.ReadUInt32();
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.09511376352107423, 'fraction_numerical': 0.011314186248912098, 'mean_word_length': 1.4788906009244993, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1257892', 'n_tokens_mistral': 2101, 'n_tokens_neox': 1957, 'n_words': 540}