<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace SourceBrowser.Generator.Model
{
    public class Token
    {
        public string FullName { get; }

        public string Type { get; }

        public int LineNumber { get; }

        public ICollection<Trivia> LeadingTrivia { get; }

        public string Value { get; }

        public ICollection<Trivia> TrailingTrivia { get; }

        public ILink Link { get; }

        public bool IsDeclaration { get; }

        public DocumentModel Document { get; }

        public Token(DocumentModel document, string fullName, string value, string type, int lineNumber, bool isDeclaration = false)
        {
            Document = document;
            FullName = fullName;
            Value = value;
            Type = type;
            LineNumber = lineNumber;
        }

        private Token(Token oldToken, ILink link) : this(oldToken.Document, oldToken.FullName, oldToken.Value, oldToken.Type, oldToken.LineNumber, oldToken.IsDeclaration)
        {
            Link = link;
            LeadingTrivia = oldToken.LeadingTrivia;
            TrailingTrivia = oldToken.TrailingTrivia;
        }

        private Token(Token oldToken, ICollection<Trivia> leading, ICollection<Trivia> trailing) :
            this(oldToken.Document, oldToken.FullName, oldToken.Value, oldToken.Type, oldToken.LineNumber, oldToken.IsDeclaration)
        {
            Link = oldToken.Link;
            LeadingTrivia = leading;
            TrailingTrivia = trailing;
        }



        /// <summary>
        /// Create a new token with the provided link.
        /// </summary>
        /// <returns>A new token containing the provided link.</returns>
        public Token WithLink(ILink link)
        {
            var newToken = new Token(this, link);
            return newToken;
        }

        /// <summary>
        /// Creates a new token with the provided trivia.
        /// </summary>
        /// <returns>A new token containing the provided trivia.</returns>
        public Token WithTrivia(ICollection<Trivia> leading, ICollection<Trivia> trailing)
        {
            var newToken = new Token(this, leading, trailing);
            return newToken;
        }
    }
}
<commit_msg>Ensure we persist the "IsDeclaration" info.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace SourceBrowser.Generator.Model
{
    public class Token
    {
        public string FullName { get; }

        public string Type { get; }

        public int LineNumber { get; }

        public ICollection<Trivia> LeadingTrivia { get; }

        public string Value { get; }

        public ICollection<Trivia> TrailingTrivia { get; }

        public ILink Link { get; }

        public bool IsDeclaration { get; }

        public DocumentModel Document { get; }

        public Token(DocumentModel document, string fullName, string value, string type, int lineNumber, bool isDeclaration = false)
        {
            Document = document;
            FullName = fullName;
            Value = value;
            Type = type;
            LineNumber = lineNumber;
            IsDeclaration = isDeclaration;
        }

        private Token(Token oldToken, ILink link) : this(oldToken.Document, oldToken.FullName, oldToken.Value, oldToken.Type, oldToken.LineNumber, oldToken.IsDeclaration)
        {
            Link = link;
            LeadingTrivia = oldToken.LeadingTrivia;
            TrailingTrivia = oldToken.TrailingTrivia;
        }

        private Token(Token oldToken, ICollection<Trivia> leading, ICollection<Trivia> trailing) :
            this(oldToken.Document, oldToken.FullName, oldToken.Value, oldToken.Type, oldToken.LineNumber, oldToken.IsDeclaration)
        {
            Link = oldToken.Link;
            LeadingTrivia = leading;
            TrailingTrivia = trailing;
        }



        /// <summary>
        /// Create a new token with the provided link.
        /// </summary>
        /// <returns>A new token containing the provided link.</returns>
        public Token WithLink(ILink link)
        {
            var newToken = new Token(this, link);
            return newToken;
        }

        /// <summary>
        /// Creates a new token with the provided trivia.
        /// </summary>
        /// <returns>A new token containing the provided trivia.</returns>
        public Token WithTrivia(ICollection<Trivia> leading, ICollection<Trivia> trailing)
        {
            var newToken = new Token(this, leading, trailing);
            return newToken;
        }
    }
}

{'fraction_non_alphanumeric': 0.08770439583775748, 'fraction_numerical': 0.0, 'mean_word_length': 2.06640625, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6529407', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1238, 'n_words': 347}