<commit_before>﻿using System;
using System.Collections.Generic;
using System.Text;

namespace Secs4Net {
    public static class SecsExtension {
        public static string ToSML(this SecsFormat format) {
            return SecsFormatStrMap[format];
        }

        internal static void CheckNull(this object arg, string argName) {
            if (arg == null) throw new ArgumentNullException(argName);
        }

        static readonly IDictionary<SecsFormat, string> SecsFormatStrMap = new Dictionary<SecsFormat, string>(15) {
            { SecsFormat.List   ,   "L"         },
			{ SecsFormat.ASCII  ,	"A"         },
			{ SecsFormat.JIS8   ,	"J"			},
			{ SecsFormat.Boolean,	"Boolean"	},
			{ SecsFormat.Binary ,	"B"			},
			{ SecsFormat.I1     ,	"I1"		},
			{ SecsFormat.I2     ,	"I2"		},
			{ SecsFormat.I4     ,	"I4"		},
			{ SecsFormat.I8     ,	"I8"		},
			{ SecsFormat.U1     ,	"U1"		},
			{ SecsFormat.U2     ,	"U2"		},
			{ SecsFormat.U4     ,	"U4"		},
			{ SecsFormat.U8     ,	"U8"		},
			{ SecsFormat.F4     ,	"F4"		},
			{ SecsFormat.F8     ,	"F8"		}
		};

        #region Bytes To Item
        internal static readonly IDictionary<SecsFormat, Func<byte[], int, int, Item>> BytesDecoders = new Dictionary<SecsFormat, Func<byte[], int, int, Item>>(14){
            { SecsFormat.ASCII  ,	ToDecoder(Item.A, Item.A, Encoding.ASCII)		            },
            { SecsFormat.JIS8   ,   ToDecoder(Item.J, Item.J, Item.JIS8Encoding)                },
        	{ SecsFormat.Binary ,	ToDecoder<byte>(Item.B, Item.B, sizeof(byte))               },
            { SecsFormat.U1     ,	ToDecoder<byte>(Item.U1, Item.U1, sizeof(byte))             },
        	{ SecsFormat.U2     ,	ToDecoder<ushort>(Item.U2, Item.U2, sizeof(ushort))         },
        	{ SecsFormat.U4     ,	ToDecoder<uint>(Item.U4, Item.U4, sizeof(uint))	            },
        	{ SecsFormat.U8     ,	ToDecoder<ulong>(Item.U8, Item.U8, sizeof(ulong))	        },
        	{ SecsFormat.I1     ,	ToDecoder<sbyte>(Item.I1, Item.I1, sizeof(sbyte))	        },
        	{ SecsFormat.I2     ,	ToDecoder<short>(Item.I2, Item.I2, sizeof(short))	        },
        	{ SecsFormat.I4     ,	ToDecoder<int>(Item.I4, Item.I4, sizeof(int))	            },
        	{ SecsFormat.I8     ,	ToDecoder<long>(Item.I8, Item.I8, sizeof(long))	            },
        	{ SecsFormat.F4     ,	ToDecoder<float>(Item.F4, Item.F4, sizeof(float))	        },
        	{ SecsFormat.F8     ,	ToDecoder<double>(Item.F8, Item.F8, sizeof(double))	        },
            { SecsFormat.Boolean,	ToDecoder<bool>(Item.Boolean, Item.Boolean, sizeof(bool))   }
        };

        static Func<byte[], int, int, Item> ToDecoder(Func<string, Item> creator, Func<Item> emptyCreator, Encoding decode) {
            return (bytes, index, length) => length == 0 ? emptyCreator() : creator(decode.GetString(bytes, index, length));
        }

        static Func<byte[], int, int, Item> ToDecoder<T>(Func<T[], Item> ctor, Func<Item> emptyCreator, int elmSize) where T : struct {<commit_msg>[remove]SML releated function was moved out.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Text;

namespace Secs4Net {
    public static class SecsExtension {
        internal static void CheckNull(this object arg, string argName) {
            if (arg == null) throw new ArgumentNullException(argName);
        }

        #region Bytes To Item
        internal static readonly IDictionary<SecsFormat, Func<byte[], int, int, Item>> BytesDecoders = new Dictionary<SecsFormat, Func<byte[], int, int, Item>>(14){
            { SecsFormat.ASCII  ,	ToDecoder(Item.A, Item.A, Encoding.ASCII)		            },
            { SecsFormat.JIS8   ,   ToDecoder(Item.J, Item.J, Item.JIS8Encoding)                },
        	{ SecsFormat.Binary ,	ToDecoder<byte>(Item.B, Item.B, sizeof(byte))               },
            { SecsFormat.U1     ,	ToDecoder<byte>(Item.U1, Item.U1, sizeof(byte))             },
        	{ SecsFormat.U2     ,	ToDecoder<ushort>(Item.U2, Item.U2, sizeof(ushort))         },
        	{ SecsFormat.U4     ,	ToDecoder<uint>(Item.U4, Item.U4, sizeof(uint))	            },
        	{ SecsFormat.U8     ,	ToDecoder<ulong>(Item.U8, Item.U8, sizeof(ulong))	        },
        	{ SecsFormat.I1     ,	ToDecoder<sbyte>(Item.I1, Item.I1, sizeof(sbyte))	        },
        	{ SecsFormat.I2     ,	ToDecoder<short>(Item.I2, Item.I2, sizeof(short))	        },
        	{ SecsFormat.I4     ,	ToDecoder<int>(Item.I4, Item.I4, sizeof(int))	            },
        	{ SecsFormat.I8     ,	ToDecoder<long>(Item.I8, Item.I8, sizeof(long))	            },
        	{ SecsFormat.F4     ,	ToDecoder<float>(Item.F4, Item.F4, sizeof(float))	        },
        	{ SecsFormat.F8     ,	ToDecoder<double>(Item.F8, Item.F8, sizeof(double))	        },
            { SecsFormat.Boolean,	ToDecoder<bool>(Item.Boolean, Item.Boolean, sizeof(bool))   }
        };

        static Func<byte[], int, int, Item> ToDecoder(Func<string, Item> creator, Func<Item> emptyCreator, Encoding decode) {
            return (bytes, index, length) => length == 0 ? emptyCreator() : creator(decode.GetString(bytes, index, length));
        }

        static Func<byte[], int, int, Item> ToDecoder<T>(Func<T[], Item> ctor, Func<Item> emptyCreator, int elmSize) where T : struct {
{'fraction_non_alphanumeric': 0.1431283219438117, 'fraction_numerical': 0.0180334092634776, 'mean_word_length': 2.2767412935323383, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '349524', 'n_tokens_mistral': 1874, 'n_tokens_neox': 1836, 'n_words': 327}