<commit_before>﻿using kh.common;
using System.Collections.Generic;
using System.IO;
using System.Linq;

namespace kh.kh2
{
    public class Msg
    {
        private static uint MagicCode = 1;
        private static byte Terminator = 0;

        public class Entry
        {
            public int Id { get; set; }
            public byte[] Data { get; set; }
        }

        public static List<Entry> Open(Stream stream)
        {
            if (!stream.CanRead || !stream.CanSeek)
                throw new InvalidDataException($"Read or seek must be supported.");

            var reader = new BinaryReader(stream);
            if (stream.Length < 8L || reader.ReadUInt32() != MagicCode)
                throw new InvalidDataException("Invalid header");

            int entriesCount = reader.ReadInt32();
            return Enumerable.Range(0, entriesCount)
                .Select(x => new
                {
                    Id = reader.ReadInt32(),
                    Offset = reader.ReadInt32(),
                })
                .ToList()
                .Select(x =>
                {
                    var data = new List<byte>();
                    reader.BaseStream.Position = x.Offset;

                    byte r;
                    while ((r = reader.ReadByte()) != Terminator)
                        data.Add(r);
                    data.Add(0);

                    return new Entry
                    {
                        Id = x.Id,
                        Data = data.ToArray()
                    };
                })
                .ToList();
        }<commit_msg>Better Msg extractor<commit_after>﻿using kh.common;
using System.Collections.Generic;
using System.IO;
using System.Linq;

namespace kh.kh2
{
    public class Msg
    {
        private static uint MagicCode = 1;
        private static byte Terminator = 0;

        public class Entry
        {
            public int Id { get; set; }
            public byte[] Data { get; set; }
        }

        public static List<Entry> Open(Stream stream)
        {
            if (!stream.CanRead || !stream.CanSeek)
                throw new InvalidDataException($"Read or seek must be supported.");

            var reader = new BinaryReader(stream);
            if (stream.Length < 8L || reader.ReadUInt32() != MagicCode)
                throw new InvalidDataException("Invalid header");

            int entriesCount = reader.ReadInt32();
            return Enumerable.Range(0, entriesCount)
                .Select(x => new
                {
                    Id = reader.ReadInt32(),
                    Offset = reader.ReadInt32(),
                })
                .ToList()
                .Select(x =>
                {
                    reader.BaseStream.Position = x.Offset;

                    return new Entry
                    {
                        Id = x.Id,
                        Data = GetMsgData(reader)
                    };
                })
                .ToList();
        }

        private static byte[] GetMsgData(BinaryReader stream)
        {
            byte r;
            var data = new List<byte>();
            do
            {
                r = stream.ReadByte();
                data.Add(r);

                switch (r)
                {
                    case 0x07:
                        data.Add(stream.ReadByte());
                        data.Add(stream.ReadByte());
                        data.Add(stream.ReadByte());
                        data.Add(stream.ReadByte());
                        break;
                    case 0x09:
                    case 0x0B:
                        data.Add(stream.ReadByte());
                        break;
                }
            } while (r != Terminator);

            data.Add(0);

            return data.ToArray();
        }
{'fraction_non_alphanumeric': 0.09408391972895491, 'fraction_numerical': 0.009382329945269743, 'mean_word_length': 1.0524064171122995, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4173939', 'n_tokens_mistral': 984, 'n_tokens_neox': 883, 'n_words': 228}