<commit_before>﻿using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics.Contracts;
using System.Linq;
using Text.Scanning;

namespace Http.Grammar.Rfc7230
{
    public class Token : Element
    {
        private readonly IList<TChar> tCharList;

        public Token(IList<TChar> tCharList, ITextContext context)
            : base(string.Concat(tCharList.Select(tchar => tchar.Data)), context)
        {
            Contract.Requires(tCharList != null);
            Contract.Requires(tCharList.Count > 0);
            Contract.Requires(Contract.ForAll(tCharList, tchar => tchar != null));
            this.tCharList = new ReadOnlyCollection<TChar>(tCharList.ToList());
        }

        public IList<TChar> TCharList
        {
            get
            {
                return this.tCharList;
            }
        }

        [ContractInvariantMethod]
        private void ObjectInvariant()
        {
            Contract.Invariant(this.tCharList != null);
        }

    }
}<commit_msg>Removing fragments from 'Token'<commit_after>﻿using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq;
using Text.Scanning;

namespace Http.Grammar.Rfc7230
{
    public class Token : Element
    {
        public Token(IList<TChar> tCharList, ITextContext context)
            : base(string.Concat(tCharList.Select(tchar => tchar.Data)), context)
        {
            Contract.Requires(tCharList != null);
            Contract.Requires(tCharList.Count > 0);
            Contract.Requires(Contract.ForAll(tCharList, tchar => tchar != null));
        }
    }
}
{'fraction_non_alphanumeric': 0.10412815773259396, 'fraction_numerical': 0.0061614294516327784, 'mean_word_length': 2.4553191489361703, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2994513', 'n_tokens_mistral': 481, 'n_tokens_neox': 472, 'n_words': 84}