<commit_before>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using JetBrains.Annotations;
using Newtonsoft.Json;

namespace HeaderArrayConverter
{
    /// <summary>
    /// Represents an immutable dictionary using sequence keys and in which the insertion order is preserved.
    /// </summary>
    /// <typeparam name="TKey">
    /// The key type.
    /// </typeparam>
    /// <typeparam name="TValue">
    /// The item type.
    /// </typeparam>
    [PublicAPI]
    [JsonDictionary]
    public class ImmutableSequenceDictionary<TKey, TValue> : IImmutableDictionary<KeySequence<TKey>, TValue>, ISequenceIndexer<TKey, TValue>
    {
        /// <summary>
        /// Compares dictionary entries based on the keys.
        /// </summary>
        [NotNull]
        private static readonly KeyComparer DistinctKeyComparer = new KeyComparer();

        /// <summary>
        /// The collection stored as an <see cref="ImmutableDictionary{TKey, TValue}"/>.
        /// </summary>
        [NotNull]
        [JsonProperty]
        private readonly IImmutableDictionary<KeySequence<TKey>, TValue> _dictionary;

        /// <summary>
        /// Gets the number of entries stored in the dictionary.
        /// </summary>
        public int Count => _dictionary.Count;

        /// <summary>
        /// Gets the total number of entries represented by the dictionary.
        /// </summary>
        public int Total => Sets.Count;

        /// <summary>
        /// Gets the entry that has the specified key or the entries that begin with the specified key.
        /// </summary>
        [NotNull]
        public ImmutableSequenceDictionary<TKey, TValue> this[params TKey[] keys]
        {
            get
            {
                KeySequence<TKey> key = new KeySequence<TKey>(keys);

                if (!Sets.Any(x => x.Take(key.Count).SequenceEqual(key)))
                {
                    throw new KeyNotFoundException();
                }

                if (_dictionary.ContainsKey(key))
                {
                    return Create(Sets, new KeyValuePair<KeySequence<TKey>, TValue>(key, _dictionary[key]));
                }

                IEnumerable<KeyValuePair<KeySequence<TKey>, TValue>> entries =
                    _dictionary.Where(x => x.Key.Take(key.Count).SequenceEqual(key));

                IEnumerable<KeyValuePair<KeySequence<TKey>, TValue>> virtualEntries =
                    Sets.Where(x => x.Take(key.Count).SequenceEqual(key))
                        .Select(x => new KeyValuePair<KeySequence<TKey>, TValue>(x, default(TValue)))
                        .ToArray();

                return Create(Sets, entries.Concat(virtualEntries).Distinct(DistinctKeyComparer));
            }
        }

        /// <summary>
        /// Gets an <see cref="IEnumerable{T}"/> for the given keys.
        /// </summary>
        /// <param name="keys">
        /// The collection of keys.
        /// </param>
        /// <returns>
        /// An <see cref="IEnumerable{T}"/> for the given keys.
        /// </returns>
        IEnumerable<KeyValuePair<KeySequence<TKey>, TValue>> ISequenceIndexer<TKey, TValue>.this[params TKey[] keys] => this[keys];

        /// <summary>
        /// Gets an <see cref="IEnumerable"/> for the given keys.
        /// </summary>
        /// <param name="keys">
        /// The collection of keys.
        /// </param>
        /// <returns>
        /// An <see cref="IEnumerable"/> for the given keys.
        /// </returns>
        IEnumerable ISequenceIndexer<TKey>.this[params TKey[] keys] => this[keys];

        /// <summary>
        /// Gets the element that has the specified key in the read-only dictionary.
        /// </summary>
        /// <returns>
        /// The element that has the specified key in the read-only dictionary.
        /// </returns>
        /// <param name="key">
        /// The key to locate.
        /// </param>
        TValue IReadOnlyDictionary<KeySequence<TKey>, TValue>.this[KeySequence<TKey> key] => _dictionary[key];

        /// <summary>
        /// Gets an enumerable collection that contains the keys in the read-only dictionary.
        /// </summary>
        [NotNull]
        public IEnumerable<KeySequence<TKey>> Keys => _dictionary.Keys;
        
        /// <summary>
        /// Gets an enumerable collection that contains the values in the read-only dictionary.
        /// </summary>
        [NotNull]
        public IEnumerable<TValue> Values => _dictionary.Values;

        /// <summary>
        /// Gets the sets that define this dictionary.
        /// </summary>
        [NotNull]
        public IImmutableList<KeySequence<TKey>> Sets { get; }

        /// <summary>
        /// Constructs an <see cref="ImmutableSequenceDictionary{TKey, TValue}"/> in which the insertion order is preserved.
        /// </summary>
        /// <param name="sets">
        /// The collection of sets that define this dictionary.
        /// </param>
        /// <param name="source">
        /// The collection from which to create the 
        /// </param>
        private ImmutableSequenceDictionary([NotNull] IEnumerable<KeySequence<TKey>> sets, [NotNull] IEnumerable<KeyValuePair<KeySequence<TKey>, TValue>> source)
        {
            if (source is null)
            {
                throw new ArgumentNullException(nameof(source));
            }

            Sets = sets.ToImmutableArray();
            _dictionary = source.Where(x => !x.Value.Equals(default(TValue))).ToImmutableSortedDictionary(KeySequence<TKey>.ReverseComparer);<commit_msg>Changing from a sorted dictionary to a normal.<commit_after>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using JetBrains.Annotations;
using Newtonsoft.Json;

namespace HeaderArrayConverter
{
    /// <summary>
    /// Represents an immutable dictionary using sequence keys and in which the insertion order is preserved.
    /// </summary>
    /// <typeparam name="TKey">
    /// The key type.
    /// </typeparam>
    /// <typeparam name="TValue">
    /// The item type.
    /// </typeparam>
    [PublicAPI]
    [JsonDictionary]
    public class ImmutableSequenceDictionary<TKey, TValue> : IImmutableDictionary<KeySequence<TKey>, TValue>, ISequenceIndexer<TKey, TValue>
    {
        /// <summary>
        /// Compares dictionary entries based on the keys.
        /// </summary>
        [NotNull]
        private static readonly KeyComparer DistinctKeyComparer = new KeyComparer();

        /// <summary>
        /// The collection stored as an <see cref="ImmutableDictionary{TKey, TValue}"/>.
        /// </summary>
        [NotNull]
        [JsonProperty]
        private readonly IImmutableDictionary<KeySequence<TKey>, TValue> _dictionary;

        /// <summary>
        /// Gets the number of entries stored in the dictionary.
        /// </summary>
        public int Count => _dictionary.Count;

        /// <summary>
        /// Gets the total number of entries represented by the dictionary.
        /// </summary>
        public int Total => Sets.Count;

        /// <summary>
        /// Gets the entry that has the specified key or the entries that begin with the specified key.
        /// </summary>
        [NotNull]
        public ImmutableSequenceDictionary<TKey, TValue> this[params TKey[] keys]
        {
            get
            {
                KeySequence<TKey> key = new KeySequence<TKey>(keys);

                if (!Sets.Any(x => x.Take(key.Count).SequenceEqual(key)))
                {
                    throw new KeyNotFoundException();
                }

                if (_dictionary.ContainsKey(key))
                {
                    return Create(Sets, new KeyValuePair<KeySequence<TKey>, TValue>(key, _dictionary[key]));
                }

                IEnumerable<KeyValuePair<KeySequence<TKey>, TValue>> entries =
                    _dictionary.Where(x => x.Key.Take(key.Count).SequenceEqual(key));

                IEnumerable<KeyValuePair<KeySequence<TKey>, TValue>> virtualEntries =
                    Sets.Where(x => x.Take(key.Count).SequenceEqual(key))
                        .Select(x => new KeyValuePair<KeySequence<TKey>, TValue>(x, default(TValue)))
                        .ToArray();

                return Create(Sets, entries.Concat(virtualEntries).Distinct(DistinctKeyComparer));
            }
        }

        /// <summary>
        /// Gets an <see cref="IEnumerable{T}"/> for the given keys.
        /// </summary>
        /// <param name="keys">
        /// The collection of keys.
        /// </param>
        /// <returns>
        /// An <see cref="IEnumerable{T}"/> for the given keys.
        /// </returns>
        IEnumerable<KeyValuePair<KeySequence<TKey>, TValue>> ISequenceIndexer<TKey, TValue>.this[params TKey[] keys] => this[keys];

        /// <summary>
        /// Gets an <see cref="IEnumerable"/> for the given keys.
        /// </summary>
        /// <param name="keys">
        /// The collection of keys.
        /// </param>
        /// <returns>
        /// An <see cref="IEnumerable"/> for the given keys.
        /// </returns>
        IEnumerable ISequenceIndexer<TKey>.this[params TKey[] keys] => this[keys];

        /// <summary>
        /// Gets the element that has the specified key in the read-only dictionary.
        /// </summary>
        /// <returns>
        /// The element that has the specified key in the read-only dictionary.
        /// </returns>
        /// <param name="key">
        /// The key to locate.
        /// </param>
        TValue IReadOnlyDictionary<KeySequence<TKey>, TValue>.this[KeySequence<TKey> key] => _dictionary[key];

        /// <summary>
        /// Gets an enumerable collection that contains the keys in the read-only dictionary.
        /// </summary>
        [NotNull]
        public IEnumerable<KeySequence<TKey>> Keys => _dictionary.Keys;
        
        /// <summary>
        /// Gets an enumerable collection that contains the values in the read-only dictionary.
        /// </summary>
        [NotNull]
        public IEnumerable<TValue> Values => _dictionary.Values;

        /// <summary>
        /// Gets the sets that define this dictionary.
        /// </summary>
        [NotNull]
        public IImmutableList<KeySequence<TKey>> Sets { get; }

        /// <summary>
        /// Constructs an <see cref="ImmutableSequenceDictionary{TKey, TValue}"/> in which the insertion order is preserved.
        /// </summary>
        /// <param name="sets">
        /// The collection of sets that define this dictionary.
        /// </param>
        /// <param name="source">
        /// The collection from which to create the 
        /// </param>
        private ImmutableSequenceDictionary([NotNull] IEnumerable<KeySequence<TKey>> sets, [NotNull] IEnumerable<KeyValuePair<KeySequence<TKey>, TValue>> source)
        {
            if (source is null)
            {
                throw new ArgumentNullException(nameof(source));
            }

            Sets = sets.ToImmutableArray();
            _dictionary = source.Where(x => !x.Value.Equals(default(TValue))).ToImmutableDictionary();
{'fraction_non_alphanumeric': 0.1307050257503108, 'fraction_numerical': 0.0, 'mean_word_length': 1.9088326446280992, 'pattern_counts': {'":': 0, '<': 182, '<?xml version=': 0, '>': 206, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5833699', 'n_tokens_mistral': 2967, 'n_tokens_neox': 2873, 'n_words': 794}