<commit_before>// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.DocAsCode.Metadata.ManagedReference
{
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Linq;
    using System.Text;
    using System.Text.RegularExpressions;
    using System.Web;
    using System.Xml;
    using System.Xml.Linq;
    using System.Xml.XPath;

    using Microsoft.DocAsCode.Common;
    using Microsoft.DocAsCode.Plugins;
    using Microsoft.DocAsCode.DataContracts.ManagedReference;
    using System.Globalization;

    public class TripleSlashCommentModel
    {
        private const string idSelector = @"((?![0-9])[\w_])+[\w\(\)\.\{\}\[\]\|\*\^~#@!`,_<>:]*";
        private static Regex CommentIdRegex = new Regex(@"^(?<type>N|T|M|P|F|E|Overload):(?<id>" + idSelector + ")$", RegexOptions.Compiled);
        private static Regex LineBreakRegex = new Regex(@"\r?\n", RegexOptions.Compiled);
        private static Regex CodeElementRegex = new Regex(@"<code[^>]*>([\s\S]*?)</code>", RegexOptions.Compiled);
        private static Regex RegionRegex = new Regex(@"^\s*#region\s*(.*)$");
        private static Regex EndRegionRegex = new Regex(@"^\s*#endregion\s*.*$");

        private readonly ITripleSlashCommentParserContext _context;

        public string Summary { get; private set; }

        public string Remarks { get; private set; }

        public string Returns { get; private set; }

        public List<ExceptionInfo> Exceptions { get; private set; }

        public List<LinkInfo> Sees { get; private set; }

        public List<LinkInfo> SeeAlsos { get; private set; }

        public List<string> Examples { get; private set; }

        public Dictionary<string, string> Parameters { get; private set; }

        public Dictionary<string, string> TypeParameters { get; private set; }

        public bool IsInheritDoc { get; private set; }

        private TripleSlashCommentModel(string xml, SyntaxLanguage language, ITripleSlashCommentParserContext context)
        {
            // Transform triple slash comment
            XDocument doc = TripleSlashCommentTransformer.Transform(xml, language);

            _context = context;
            if (!context.PreserveRawInlineComments)
            {
                ResolveSeeCref(doc, context.AddReferenceDelegate, context.ResolveCRef);
                ResolveSeeAlsoCref(doc, context.AddReferenceDelegate, context.ResolveCRef);
                ResolveExceptionCref(doc, context.AddReferenceDelegate, context.ResolveCRef);
            }
            ResolveCodeSource(doc, context);
            var nav = doc.CreateNavigator();
            Summary = GetSummary(nav, context);
            Remarks = GetRemarks(nav, context);
            Returns = GetReturns(nav, context);

            Exceptions = GetExceptions(nav, context);
            Sees = GetSees(nav, context);
            SeeAlsos = GetSeeAlsos(nav, context);
            Examples = GetExamples(nav, context);
            Parameters = GetParameters(nav, context);
            TypeParameters = GetTypeParameters(nav, context);
            IsInheritDoc = GetIsInheritDoc(nav, context);
        }

        public static TripleSlashCommentModel CreateModel(string xml, SyntaxLanguage language, ITripleSlashCommentParserContext context)
        {
            if (context == null)
            {
                throw new ArgumentNullException(nameof(context));
            }
            if (string.IsNullOrEmpty(xml)) return null;

            // Quick turnaround for badly formed XML comment
            if (xml.StartsWith("<!-- Badly formed XML comment ignored for member "))
            {
                Logger.LogWarning($"Invalid triple slash comment is ignored: {xml}");
                return null;
            }
            try
            {
                var model = new TripleSlashCommentModel(xml, language, context);
                return model;
            }
            catch (XmlException)
            {
                return null;
            }
        }

        public void CopyInheritedData(TripleSlashCommentModel src)
        {
            if (src == null)
            {
                throw new ArgumentNullException(nameof(src));
            }

            Summary = Summary ?? src.Summary;
            Remarks = Remarks ?? src.Remarks;
            Returns = Returns ?? src.Returns;
            if (Exceptions == null && src.Exceptions != null)
            {
                Exceptions = src.Exceptions.Select(e => e.Clone()).ToList();
            }
            if (Sees == null && src.Sees != null)
            {
                Sees = src.Sees.Select(s => s.Clone()).ToList();
            }
            if (SeeAlsos == null && src.SeeAlsos != null)
            {
                SeeAlsos = src.SeeAlsos.Select(s => s.Clone()).ToList();
            }
            if (Examples == null && src.Examples != null)
            {
                Examples = new List<string>(src.Examples);
            }
            if (Parameters == null && src.Parameters != null)
            {
                Parameters = new Dictionary<string, string>(src.Parameters);
            }
            if (TypeParameters == null && src.TypeParameters != null)
            {
                TypeParameters = new Dictionary<string, string>(src.TypeParameters);
            }
        }

        public string GetParameter(string name)
        {
            if (string.IsNullOrEmpty(name))
            {
                return null;
            }
            return GetValue(name, Parameters);
        }

        public string GetTypeParameter(string name)
        {
            if (string.IsNullOrEmpty(name))
            {
                return null;
            }
            return GetValue(name, TypeParameters);
        }

        private static string GetValue(string name, Dictionary<string, string> dictionary)
        {
            if (dictionary == null)
            {
                return null;
            }
            if (dictionary.TryGetValue(name, out string description))
            {
                return description;
            }
            return null;
        }

        /// <summary>
        /// Get summary node out from triple slash comments
        /// </summary>
        /// <param name="xml"></param>
        /// <param name="normalize"></param>
        /// <returns></returns>
        /// <example>
        /// <code> <see cref="Hello"/></code>
        /// </example>
        private string GetSummary(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            // Resolve <see cref> to @ syntax
            // Also support <seealso cref>
            string selector = "/member/summary";
            return GetSingleNodeValue(nav, selector);
        }

        /// <summary>
        /// Get remarks node out from triple slash comments
        /// </summary>
        /// <remarks>
        /// <para>This is a sample of exception node</para>
        /// </remarks>
        /// <param name="xml"></param>
        /// <param name="normalize"></param>
        /// <returns></returns>
        private string GetRemarks(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            string selector = "/member/remarks";
            return GetSingleNodeValue(nav, selector);
        }

        private string GetReturns(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            // Resolve <see cref> to @ syntax
            // Also support <seealso cref>
            string selector = "/member/returns";
            return GetSingleNodeValue(nav, selector);
        }

        /// <summary>
        /// Get exceptions nodes out from triple slash comments
        /// </summary>
        /// <param name="xml"></param>
        /// <param name="normalize"></param>
        /// <returns></returns>
        /// <exception cref="XmlException">This is a sample of exception node</exception>
        private List<ExceptionInfo> GetExceptions(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            string selector = "/member/exception";
            var result = GetMulitpleCrefInfo(nav, selector).ToList();
            if (result.Count == 0)
            {
                return null;
            }
            return result;
        }

        /// <summary>
        /// To get `see` tags out
        /// </summary>
        /// <param name="xml"></param>
        /// <param name="context"></param>
        /// <returns></returns>
        /// <see cref="SpecIdHelper"/>
        /// <see cref="SourceSwitch"/>
        private List<LinkInfo> GetSees(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            var result = GetMultipleLinkInfo(nav, "/member/see").ToList();
            if (result.Count == 0)
            {
                return null;
            }
            return result;
        }

        /// <summary>
        /// To get `seealso` tags out
        /// </summary>
        /// <param name="xml"></param>
        /// <param name="context"></param>
        /// <returns></returns>
        /// <seealso cref="WaitForChangedResult"/>
        /// <seealso cref="http://google.com">ABCS</seealso>
        private List<LinkInfo> GetSeeAlsos(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            var result = GetMultipleLinkInfo(nav, "/member/seealso").ToList();
            if (result.Count == 0)
            {
                return null;
            }
            return result;
        }

        /// <summary>
        /// To get `example` tags out
        /// </summary>
        /// <param name="xml"></param>
        /// <param name="context"></param>
        /// <returns></returns>
        /// <example>
        /// This sample shows how to call the <see cref="GetExceptions(string, ITripleSlashCommentParserContext)"/> method.
        /// <code>
        /// class TestClass
        /// {
        ///     static int Main()
        ///     {
        ///         return GetExceptions(null, null).Count();
        ///     }
        /// }
        /// </code>
        /// </example>
        private List<string> GetExamples(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            // Resolve <see cref> to @ syntax
            // Also support <seealso cref>
            return GetMultipleExampleNodes(nav, "/member/example").ToList();
        }

        private bool GetIsInheritDoc(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            var node = nav.SelectSingleNode("/member/inheritdoc");
            if (node == null)
            {
                return false;
            }
            if (node.HasAttributes)
            {
                //The Sandcastle implementation of <inheritdoc /> supports two attributes: 'cref' and 'select'.
                //These attributes allow changing the source of the inherited doc and controlling what is inherited.
                //Until these attributes are supported, ignoring inheritdoc elements with attributes, so as not to misinterpret them.
                Logger.LogWarning("Attributes on <inheritdoc /> elements are not supported; inheritdoc element will be ignored.");
                return false;
            }
            return true;
        }

        private void ResolveCodeSource(XDocument doc, ITripleSlashCommentParserContext context)
        {
            foreach (XElement node in doc.XPathSelectElements("//code"))
            {
                var source = node.Attribute("source");
                if (source == null || string.IsNullOrEmpty(source.Value))
                {
                    continue;
                }

                if (context.Source == null || string.IsNullOrEmpty(context.Source.Path))
                {
                    Logger.LogWarning($"Unable to get source file path for {node.ToString()}");
                    return;
                }

                var region = node.Attribute("region");

                var path = source.Value;
                if (!Path.IsPathRooted(path))
                {
                    string currentFilePath = context.Source.Remote != null ? Path.Combine(EnvironmentContext.BaseDirectory, context.Source.Remote.RelativePath) : context.Source.Path;
                    var directory = Path.GetDirectoryName(currentFilePath);
                    path = Path.Combine(directory, path);
                }

                ResolveCodeSource(node, path, region.Value);
            }
        }

        private void ResolveCodeSource(XElement element, string source, string region)
        {
            if (!File.Exists(source))
            {
                Logger.LogWarning($"Source file '{source}' not found.");
                return;
            }

            var builder = new StringBuilder();
            var regionCount = 0;
            foreach (var line in File.ReadLines(source))
            {
                var match = RegionRegex.Match(line);
                if (match.Success)
                {
                    var name = match.Groups[1].Value.Trim();
                    if (name == region)
                    {
                        ++regionCount;
                        continue;
                    }
                    else if (regionCount > 0)
                    {
                        ++regionCount;
                    }
                }
                else if (regionCount > 0 && EndRegionRegex.IsMatch(line))
                {
                    --regionCount;
                    if (regionCount == 0)
                    {
                        break;
                    }
                }

                if (string.IsNullOrEmpty(region) || regionCount > 0)
                {
                    builder.AppendLine(line);
                }
            }

            element.SetValue(builder.ToString());
        }

        private Dictionary<string, string> GetListContent(XPathNavigator navigator, string xpath, string contentType, ITripleSlashCommentParserContext context)
        {
            var iterator = navigator.Select(xpath);
            var result = new Dictionary<string, string>();
            if (iterator == null)
            {
                return result;
            }
            foreach (XPathNavigator nav in iterator)
            {
                string name = nav.GetAttribute("name", string.Empty);<commit_msg>Fix null reference bug (#2662)<commit_after>// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.DocAsCode.Metadata.ManagedReference
{
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Linq;
    using System.Text;
    using System.Text.RegularExpressions;
    using System.Web;
    using System.Xml;
    using System.Xml.Linq;
    using System.Xml.XPath;

    using Microsoft.DocAsCode.Common;
    using Microsoft.DocAsCode.Plugins;
    using Microsoft.DocAsCode.DataContracts.ManagedReference;
    using System.Globalization;

    public class TripleSlashCommentModel
    {
        private const string idSelector = @"((?![0-9])[\w_])+[\w\(\)\.\{\}\[\]\|\*\^~#@!`,_<>:]*";
        private static Regex CommentIdRegex = new Regex(@"^(?<type>N|T|M|P|F|E|Overload):(?<id>" + idSelector + ")$", RegexOptions.Compiled);
        private static Regex LineBreakRegex = new Regex(@"\r?\n", RegexOptions.Compiled);
        private static Regex CodeElementRegex = new Regex(@"<code[^>]*>([\s\S]*?)</code>", RegexOptions.Compiled);
        private static Regex RegionRegex = new Regex(@"^\s*#region\s*(.*)$");
        private static Regex EndRegionRegex = new Regex(@"^\s*#endregion\s*.*$");

        private readonly ITripleSlashCommentParserContext _context;

        public string Summary { get; private set; }

        public string Remarks { get; private set; }

        public string Returns { get; private set; }

        public List<ExceptionInfo> Exceptions { get; private set; }

        public List<LinkInfo> Sees { get; private set; }

        public List<LinkInfo> SeeAlsos { get; private set; }

        public List<string> Examples { get; private set; }

        public Dictionary<string, string> Parameters { get; private set; }

        public Dictionary<string, string> TypeParameters { get; private set; }

        public bool IsInheritDoc { get; private set; }

        private TripleSlashCommentModel(string xml, SyntaxLanguage language, ITripleSlashCommentParserContext context)
        {
            // Transform triple slash comment
            XDocument doc = TripleSlashCommentTransformer.Transform(xml, language);

            _context = context;
            if (!context.PreserveRawInlineComments)
            {
                ResolveSeeCref(doc, context.AddReferenceDelegate, context.ResolveCRef);
                ResolveSeeAlsoCref(doc, context.AddReferenceDelegate, context.ResolveCRef);
                ResolveExceptionCref(doc, context.AddReferenceDelegate, context.ResolveCRef);
            }
            ResolveCodeSource(doc, context);
            var nav = doc.CreateNavigator();
            Summary = GetSummary(nav, context);
            Remarks = GetRemarks(nav, context);
            Returns = GetReturns(nav, context);

            Exceptions = GetExceptions(nav, context);
            Sees = GetSees(nav, context);
            SeeAlsos = GetSeeAlsos(nav, context);
            Examples = GetExamples(nav, context);
            Parameters = GetParameters(nav, context);
            TypeParameters = GetTypeParameters(nav, context);
            IsInheritDoc = GetIsInheritDoc(nav, context);
        }

        public static TripleSlashCommentModel CreateModel(string xml, SyntaxLanguage language, ITripleSlashCommentParserContext context)
        {
            if (context == null)
            {
                throw new ArgumentNullException(nameof(context));
            }
            if (string.IsNullOrEmpty(xml)) return null;

            // Quick turnaround for badly formed XML comment
            if (xml.StartsWith("<!-- Badly formed XML comment ignored for member "))
            {
                Logger.LogWarning($"Invalid triple slash comment is ignored: {xml}");
                return null;
            }
            try
            {
                var model = new TripleSlashCommentModel(xml, language, context);
                return model;
            }
            catch (XmlException)
            {
                return null;
            }
        }

        public void CopyInheritedData(TripleSlashCommentModel src)
        {
            if (src == null)
            {
                throw new ArgumentNullException(nameof(src));
            }

            Summary = Summary ?? src.Summary;
            Remarks = Remarks ?? src.Remarks;
            Returns = Returns ?? src.Returns;
            if (Exceptions == null && src.Exceptions != null)
            {
                Exceptions = src.Exceptions.Select(e => e.Clone()).ToList();
            }
            if (Sees == null && src.Sees != null)
            {
                Sees = src.Sees.Select(s => s.Clone()).ToList();
            }
            if (SeeAlsos == null && src.SeeAlsos != null)
            {
                SeeAlsos = src.SeeAlsos.Select(s => s.Clone()).ToList();
            }
            if (Examples == null && src.Examples != null)
            {
                Examples = new List<string>(src.Examples);
            }
            if (Parameters == null && src.Parameters != null)
            {
                Parameters = new Dictionary<string, string>(src.Parameters);
            }
            if (TypeParameters == null && src.TypeParameters != null)
            {
                TypeParameters = new Dictionary<string, string>(src.TypeParameters);
            }
        }

        public string GetParameter(string name)
        {
            if (string.IsNullOrEmpty(name))
            {
                return null;
            }
            return GetValue(name, Parameters);
        }

        public string GetTypeParameter(string name)
        {
            if (string.IsNullOrEmpty(name))
            {
                return null;
            }
            return GetValue(name, TypeParameters);
        }

        private static string GetValue(string name, Dictionary<string, string> dictionary)
        {
            if (dictionary == null)
            {
                return null;
            }
            if (dictionary.TryGetValue(name, out string description))
            {
                return description;
            }
            return null;
        }

        /// <summary>
        /// Get summary node out from triple slash comments
        /// </summary>
        /// <param name="xml"></param>
        /// <param name="normalize"></param>
        /// <returns></returns>
        /// <example>
        /// <code> <see cref="Hello"/></code>
        /// </example>
        private string GetSummary(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            // Resolve <see cref> to @ syntax
            // Also support <seealso cref>
            string selector = "/member/summary";
            return GetSingleNodeValue(nav, selector);
        }

        /// <summary>
        /// Get remarks node out from triple slash comments
        /// </summary>
        /// <remarks>
        /// <para>This is a sample of exception node</para>
        /// </remarks>
        /// <param name="xml"></param>
        /// <param name="normalize"></param>
        /// <returns></returns>
        private string GetRemarks(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            string selector = "/member/remarks";
            return GetSingleNodeValue(nav, selector);
        }

        private string GetReturns(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            // Resolve <see cref> to @ syntax
            // Also support <seealso cref>
            string selector = "/member/returns";
            return GetSingleNodeValue(nav, selector);
        }

        /// <summary>
        /// Get exceptions nodes out from triple slash comments
        /// </summary>
        /// <param name="xml"></param>
        /// <param name="normalize"></param>
        /// <returns></returns>
        /// <exception cref="XmlException">This is a sample of exception node</exception>
        private List<ExceptionInfo> GetExceptions(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            string selector = "/member/exception";
            var result = GetMulitpleCrefInfo(nav, selector).ToList();
            if (result.Count == 0)
            {
                return null;
            }
            return result;
        }

        /// <summary>
        /// To get `see` tags out
        /// </summary>
        /// <param name="xml"></param>
        /// <param name="context"></param>
        /// <returns></returns>
        /// <see cref="SpecIdHelper"/>
        /// <see cref="SourceSwitch"/>
        private List<LinkInfo> GetSees(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            var result = GetMultipleLinkInfo(nav, "/member/see").ToList();
            if (result.Count == 0)
            {
                return null;
            }
            return result;
        }

        /// <summary>
        /// To get `seealso` tags out
        /// </summary>
        /// <param name="xml"></param>
        /// <param name="context"></param>
        /// <returns></returns>
        /// <seealso cref="WaitForChangedResult"/>
        /// <seealso cref="http://google.com">ABCS</seealso>
        private List<LinkInfo> GetSeeAlsos(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            var result = GetMultipleLinkInfo(nav, "/member/seealso").ToList();
            if (result.Count == 0)
            {
                return null;
            }
            return result;
        }

        /// <summary>
        /// To get `example` tags out
        /// </summary>
        /// <param name="xml"></param>
        /// <param name="context"></param>
        /// <returns></returns>
        /// <example>
        /// This sample shows how to call the <see cref="GetExceptions(string, ITripleSlashCommentParserContext)"/> method.
        /// <code>
        /// class TestClass
        /// {
        ///     static int Main()
        ///     {
        ///         return GetExceptions(null, null).Count();
        ///     }
        /// }
        /// </code>
        /// </example>
        private List<string> GetExamples(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            // Resolve <see cref> to @ syntax
            // Also support <seealso cref>
            return GetMultipleExampleNodes(nav, "/member/example").ToList();
        }

        private bool GetIsInheritDoc(XPathNavigator nav, ITripleSlashCommentParserContext context)
        {
            var node = nav.SelectSingleNode("/member/inheritdoc");
            if (node == null)
            {
                return false;
            }
            if (node.HasAttributes)
            {
                //The Sandcastle implementation of <inheritdoc /> supports two attributes: 'cref' and 'select'.
                //These attributes allow changing the source of the inherited doc and controlling what is inherited.
                //Until these attributes are supported, ignoring inheritdoc elements with attributes, so as not to misinterpret them.
                Logger.LogWarning("Attributes on <inheritdoc /> elements are not supported; inheritdoc element will be ignored.");
                return false;
            }
            return true;
        }

        private void ResolveCodeSource(XDocument doc, ITripleSlashCommentParserContext context)
        {
            foreach (XElement node in doc.XPathSelectElements("//code"))
            {
                var source = node.Attribute("source");
                if (source == null || string.IsNullOrEmpty(source.Value))
                {
                    continue;
                }

                if (context.Source == null || string.IsNullOrEmpty(context.Source.Path))
                {
                    Logger.LogWarning($"Unable to get source file path for {node.ToString()}");
                    return;
                }

                var region = node.Attribute("region");

                var path = source.Value;
                if (!Path.IsPathRooted(path))
                {
                    string currentFilePath = context.Source.Remote != null ? Path.Combine(EnvironmentContext.BaseDirectory, context.Source.Remote.RelativePath) : context.Source.Path;
                    var directory = Path.GetDirectoryName(currentFilePath);
                    path = Path.Combine(directory, path);
                }

                ResolveCodeSource(node, path, region?.Value);
            }
        }

        private void ResolveCodeSource(XElement element, string source, string region)
        {
            if (!File.Exists(source))
            {
                Logger.LogWarning($"Source file '{source}' not found.");
                return;
            }

            var builder = new StringBuilder();
            var regionCount = 0;
            foreach (var line in File.ReadLines(source))
            {
                if (!string.IsNullOrEmpty(region))
                {
                    var match = RegionRegex.Match(line);
                    if (match.Success)
                    {
                        var name = match.Groups[1].Value.Trim();
                        if (name == region)
                        {
                            ++regionCount;
                            continue;
                        }
                        else if (regionCount > 0)
                        {
                            ++regionCount;
                        }
                    }
                    else if (regionCount > 0 && EndRegionRegex.IsMatch(line))
                    {
                        --regionCount;
                        if (regionCount == 0)
                        {
                            break;
                        }
                    }

                    if (regionCount > 0)
                    {
                        builder.AppendLine(line);
                    }
                }
                else
                {
                    builder.AppendLine(line);
                }
            }

            element.SetValue(builder.ToString());
        }

        private Dictionary<string, string> GetListContent(XPathNavigator navigator, string xpath, string contentType, ITripleSlashCommentParserContext context)
        {
            var iterator = navigator.Select(xpath);
            var result = new Dictionary<string, string>();
            if (iterator == null)
            {
                return result;
            }
            foreach (XPathNavigator nav in iterator)
            {
                string name = nav.GetAttribute("name", string.Empty);
{'fraction_non_alphanumeric': 0.1088882830072933, 'fraction_numerical': 0.0008861018335491786, 'mean_word_length': 1.562707423580786, 'pattern_counts': {'":': 0, '<': 201, '<?xml version=': 0, '>': 213, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 26}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2502423', 'n_tokens_mistral': 7337, 'n_tokens_neox': 7057, 'n_words': 1764}