<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Versionr.Objects;
using System.IO;
using System.Text.RegularExpressions;

namespace Versionr
{
    public enum StatusCode
    {
        Unversioned,
        Unchanged,
        Added,
        Modified,
        Missing,
        Deleted,
        Renamed,
        Copied,
        Conflict,
		Ignored,
        Masked,

        Count
    }
    public class Status
    {
        public class StatusEntry
        {
            public StatusCode Code { get; set; }
            public bool Staged { get; set; }
            public Entry FilesystemEntry { get; set; }
            public Record VersionControlRecord { get; set; }
            public string CanonicalName
            {
                get
                {
                    if (FilesystemEntry != null)
                        return FilesystemEntry.CanonicalName;
                    return VersionControlRecord.CanonicalName;
                }
            }
            public string Name
            {
                get
                {
                    if (FilesystemEntry != null)
                        return FilesystemEntry.Name;
                    return VersionControlRecord.Name;
                }
            }
            public bool IsDirectory
            {
                get
                {
                    return FilesystemEntry != null ? FilesystemEntry.IsDirectory : VersionControlRecord.IsDirectory;
                }
            }

			public bool IsSymlink
			{
				get
				{
					return FilesystemEntry != null ? FilesystemEntry.Attributes.HasFlag(Objects.Attributes.Symlink) : VersionControlRecord.Attributes.HasFlag(Objects.Attributes.Symlink);
				}
			}

            public bool DataEquals(Record x)
            {
                if (FilesystemEntry != null)
                {
                    if (FilesystemEntry.IsDirectory)
                        return x.Fingerprint == FilesystemEntry.CanonicalName;
					if (FilesystemEntry.IsSymlink)
						return x.Fingerprint == FilesystemEntry.SymlinkTarget;
                    if (Code == StatusCode.Unchanged)
                        return x.DataEquals(VersionControlRecord);
                    return FilesystemEntry.DataEquals(x.Fingerprint, x.Size);
                }
                return false;
            }

            public string Hash
            {
                get
                {
                    if (Code == StatusCode.Unchanged)
                        return VersionControlRecord.Fingerprint;
                    return FilesystemEntry.Hash;
                }
            }

            public long Length
            {
                get
                {
                    return FilesystemEntry != null ? FilesystemEntry.Length : 0;
                }
            }

            public bool Removed
            {
                get
                {
                    return FilesystemEntry == null;
                }
            }
        }
        public List<Objects.Record> VersionControlRecords { get; set; }
        public Objects.Version CurrentVersion { get; set; }
        public Branch Branch { get; set; }
        public List<StatusEntry> Elements { get; set; }
        public List<Objects.Version> MergeInputs { get; set; }
		public Dictionary<string, StatusEntry> Map { get; set; }
		public List<LocalState.StageOperation> Stage { get; set; }
        public Area Workspace { get; set; }
        public string RestrictedPath { get; set; }
        public int Files { get; set; }
        public int Directories { get; set; }
        public int IgnoredObjects { get; set; }
        public bool HasData
        {
            get
            {
                foreach (var x in Elements)
                {
                    if (x.Staged)
                        return true;
                }
                return false;
            }
        }
        public bool HasModifications(bool requireStaging)
        {
            HashSet<string> addedFiles = new HashSet<string>(Stage.Where(x => x.Type == LocalState.StageOperationType.Add).Select(x => x.Operand1));
            foreach (var x in Elements)
            {
                if (x.Staged == true)
                    return true;
                else if (x.Code == StatusCode.Modified && !requireStaging)
                {
                    return true;
                }
            }
            return false;
        }
        [Flags]
        internal enum StageFlags
        {
            Recorded = 1,
            Removed = 2,
            Renamed = 4,
            Conflicted = 8,
            MergeInfo = 16
        }
        int UpdatedFileTimeCount = 0;
        class StatusPercentage
        {
            public FileStatus Snapshot { get; set; }
            public System.Threading.ManualResetEvent EndEvent { get; set; }
        }

        internal Status(Area workspace, WorkspaceDB db, LocalDB ldb, FileStatus currentSnapshot, string restrictedPath = null, bool updateFileTimes = true, bool findCopies = true)
        {
            if (!string.IsNullOrEmpty(restrictedPath) && !restrictedPath.EndsWith("/"))
                restrictedPath += "/";
            System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
            sw.Start();
            StatusPercentage pct = new StatusPercentage()
            {
                Snapshot = currentSnapshot,
                EndEvent = new System.Threading.ManualResetEvent(false)
            };
            var progressLog = Task.Run(() =>
            {
                Printer.InteractivePrinter ip = null;
                while (true)
                {
                    if (pct.EndEvent.WaitOne(500))
                    {
                        if (ip != null)
                            ip.End(0);
                        return;
                    }
                    if (sw.ElapsedMilliseconds > 2000)
                    {
                        if (ip == null)
                            ip = Printer.CreateSpinnerPrinter(string.Format("Computing status for {0} objects", pct.Snapshot.Entries.Count), (obj) => { return string.Empty; });
                        ip.Update(0);
                    }
                }
            });
            RestrictedPath = restrictedPath;
            Workspace = workspace;
            CurrentVersion = db.Version;
            Branch = db.Branch;
            Elements = new List<StatusEntry>();
            var tasks = new List<Task<StatusEntry>>();
            Dictionary<string, Entry> snapshotData = new Dictionary<string, Entry>();
            foreach (var x in currentSnapshot.Entries)
                snapshotData[x.CanonicalName] = x;
            HashSet<Entry> foundEntries = new HashSet<Entry>();
            var records = db.GetCachedRecords(CurrentVersion);
            var stage = ldb.StageOperations;
            Stage = stage;
            VersionControlRecords = records;
            Dictionary<string, string> caseInsensitiveNames = new Dictionary<string, string>();
            foreach (var x in records)
                caseInsensitiveNames[x.CanonicalName.ToLowerInvariant()] = x.CanonicalName;
            MergeInputs = new List<Objects.Version>();
            Dictionary<string, StageFlags> stageInformation = new Dictionary<string, StageFlags>();
            Dictionary<Record, StatusEntry> statusMap = new Dictionary<Record, StatusEntry>();
            Dictionary<string, bool> parentIgnoredList = new Dictionary<string, bool>();
            foreach (var x in stage)
            {
                if (x.Type == LocalState.StageOperationType.Merge)
                    MergeInputs.Add(Workspace.GetVersion(new Guid(x.Operand1)));
                if (!x.IsFileOperation)
                    continue;
                StageFlags ops;
                if (!stageInformation.TryGetValue(x.Operand1, out ops))
                    stageInformation[x.Operand1] = ops;
                if (x.Type == LocalState.StageOperationType.Add)
                    ops |= StageFlags.Recorded;
                if (x.Type == LocalState.StageOperationType.Conflict)
                    ops |= StageFlags.Conflicted;
                if (x.Type == LocalState.StageOperationType.Remove)
                    ops |= StageFlags.Removed;
                if (x.Type == LocalState.StageOperationType.Rename)
                    ops |= StageFlags.Renamed;
                if (x.Type == LocalState.StageOperationType.MergeRecord)
                    ops |= StageFlags.MergeInfo;
                stageInformation[x.Operand1] = ops;
            }
            try
            {
                foreach (var x in records)
                {
                    tasks.Add(Utilities.LimitedTaskDispatcher.Factory.StartNew<StatusEntry>(() =>
                    {
                        StageFlags objectFlags;
                        stageInformation.TryGetValue(x.CanonicalName, out objectFlags);
                        Entry snapshotRecord = null;
                        if (RestrictedPath != null)
                        {
                            if (!x.CanonicalName.StartsWith(RestrictedPath, StringComparison.Ordinal) && x.CanonicalName != RestrictedPath)
                            {
                                if (x.CanonicalName == ".vrmeta" && !string.IsNullOrEmpty(Workspace.PartialPath))
                                {
                                    if (snapshotData.TryGetValue(x.CanonicalName, out snapshotRecord))
                                    {
                                        lock (foundEntries)
                                            foundEntries.Add(snapshotRecord);
                                    }
                                }
                                else
                                    return new StatusEntry() { Code = StatusCode.Ignored, FilesystemEntry = snapshotRecord, VersionControlRecord = x, Staged = objectFlags.HasFlag(StageFlags.Recorded) };
                            }
                        }

                        if (snapshotData.TryGetValue(x.CanonicalName, out snapshotRecord) && snapshotRecord.Ignored == false)
                        {
                            lock (foundEntries)
                                foundEntries.Add(snapshotRecord);

                            if (objectFlags.HasFlag(StageFlags.Removed))
                                Printer.PrintWarning("Removed object `{0}` still in filesystem!", x.CanonicalName);

                            if (objectFlags.HasFlag(StageFlags.Renamed))
                                Printer.PrintWarning("Renamed object `{0}` still in filesystem!", x.CanonicalName);

                            if (objectFlags.HasFlag(StageFlags.Conflicted))
                                return new StatusEntry() { Code = StatusCode.Conflict, FilesystemEntry = snapshotRecord, VersionControlRecord = x, Staged = objectFlags.HasFlag(StageFlags.Recorded) };

                            bool changed = false;
                            if (snapshotRecord.Length != x.Size)
                                changed = true;
                            if (!changed && snapshotRecord.IsSymlink && snapshotRecord.SymlinkTarget != x.Fingerprint)
                                changed = true;
                            if (!changed && !snapshotRecord.IsDirectory && !snapshotRecord.IsSymlink)
                            {
                                LocalState.FileTimestamp fst = Workspace.GetReferenceTime(x.CanonicalName);
                                if (snapshotRecord.ModificationTime == x.ModificationTime || (fst.DataIdentifier == x.DataIdentifier && snapshotRecord.ModificationTime == fst.LastSeenTime))
                                    changed = false;
                                else
                                {
                                    Printer.PrintDiagnostics("Computing hash for: " + x.CanonicalName);
                                    if (snapshotRecord.Hash != x.Fingerprint)
                                        changed = true;
                                    else
                                    {
                                        System.Threading.Interlocked.Increment(ref this.UpdatedFileTimeCount);
                                        Workspace.UpdateFileTimeCache(x.CanonicalName, x, snapshotRecord.ModificationTime, false);
                                    }
                                }
                            }
                            if (changed == true)
                                return new StatusEntry() { Code = StatusCode.Modified, FilesystemEntry = snapshotRecord, VersionControlRecord = x, Staged = objectFlags.HasFlag(StageFlags.Recorded) };
                            else
                            {
                                if (objectFlags.HasFlag(StageFlags.Recorded))
                                    Printer.PrintWarning("Unchanged object `{0}` still marked as recorded in commit!", x.CanonicalName);
                                return new StatusEntry() { Code = StatusCode.Unchanged, FilesystemEntry = snapshotRecord, VersionControlRecord = x, Staged = objectFlags.HasFlag(StageFlags.Recorded) };
                            }
                        }
                        else
                        {
                            if (objectFlags.HasFlag(StageFlags.Removed))
                                return new StatusEntry() { Code = StatusCode.Deleted, FilesystemEntry = null, VersionControlRecord = x, Staged = true };

                            string parentName = x.CanonicalName;
                            bool resolved = false;
                            while (true)
                            {
                                if (!parentName.Contains('/'))
                                    break;
                                if (parentName.EndsWith("/"))
                                    parentName = parentName.Substring(0, parentName.Length - 1);
                                parentName = parentName.Substring(0, parentName.LastIndexOf('/') + 1);
                                bool ignoredInParentList = false;
                                if (parentIgnoredList.TryGetValue(parentName, out ignoredInParentList))
                                {
                                    if (ignoredInParentList)
                                        resolved = true;
                                    break;
                                }
                                else
                                {
                                    Entry parentObjectEntry = null;
                                    snapshotData.TryGetValue(parentName, out parentObjectEntry);
                                    if (parentObjectEntry != null && parentObjectEntry.Ignored == true)
                                    {
                                        parentIgnoredList[parentName] = true;
                                        resolved = true;
                                        break;
                                    }
                                }
                            }<commit_msg>Fixed race condition in status operations<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Versionr.Objects;
using System.IO;
using System.Text.RegularExpressions;

namespace Versionr
{
    public enum StatusCode
    {
        Unversioned,
        Unchanged,
        Added,
        Modified,
        Missing,
        Deleted,
        Renamed,
        Copied,
        Conflict,
		Ignored,
        Masked,

        Count
    }
    public class Status
    {
        public class StatusEntry
        {
            public StatusCode Code { get; set; }
            public bool Staged { get; set; }
            public Entry FilesystemEntry { get; set; }
            public Record VersionControlRecord { get; set; }
            public string CanonicalName
            {
                get
                {
                    if (FilesystemEntry != null)
                        return FilesystemEntry.CanonicalName;
                    return VersionControlRecord.CanonicalName;
                }
            }
            public string Name
            {
                get
                {
                    if (FilesystemEntry != null)
                        return FilesystemEntry.Name;
                    return VersionControlRecord.Name;
                }
            }
            public bool IsDirectory
            {
                get
                {
                    return FilesystemEntry != null ? FilesystemEntry.IsDirectory : VersionControlRecord.IsDirectory;
                }
            }

			public bool IsSymlink
			{
				get
				{
					return FilesystemEntry != null ? FilesystemEntry.Attributes.HasFlag(Objects.Attributes.Symlink) : VersionControlRecord.Attributes.HasFlag(Objects.Attributes.Symlink);
				}
			}

            public bool DataEquals(Record x)
            {
                if (FilesystemEntry != null)
                {
                    if (FilesystemEntry.IsDirectory)
                        return x.Fingerprint == FilesystemEntry.CanonicalName;
					if (FilesystemEntry.IsSymlink)
						return x.Fingerprint == FilesystemEntry.SymlinkTarget;
                    if (Code == StatusCode.Unchanged)
                        return x.DataEquals(VersionControlRecord);
                    return FilesystemEntry.DataEquals(x.Fingerprint, x.Size);
                }
                return false;
            }

            public string Hash
            {
                get
                {
                    if (Code == StatusCode.Unchanged)
                        return VersionControlRecord.Fingerprint;
                    return FilesystemEntry.Hash;
                }
            }

            public long Length
            {
                get
                {
                    return FilesystemEntry != null ? FilesystemEntry.Length : 0;
                }
            }

            public bool Removed
            {
                get
                {
                    return FilesystemEntry == null;
                }
            }
        }
        public List<Objects.Record> VersionControlRecords { get; set; }
        public Objects.Version CurrentVersion { get; set; }
        public Branch Branch { get; set; }
        public List<StatusEntry> Elements { get; set; }
        public List<Objects.Version> MergeInputs { get; set; }
		public Dictionary<string, StatusEntry> Map { get; set; }
		public List<LocalState.StageOperation> Stage { get; set; }
        public Area Workspace { get; set; }
        public string RestrictedPath { get; set; }
        public int Files { get; set; }
        public int Directories { get; set; }
        public int IgnoredObjects { get; set; }
        public bool HasData
        {
            get
            {
                foreach (var x in Elements)
                {
                    if (x.Staged)
                        return true;
                }
                return false;
            }
        }
        public bool HasModifications(bool requireStaging)
        {
            HashSet<string> addedFiles = new HashSet<string>(Stage.Where(x => x.Type == LocalState.StageOperationType.Add).Select(x => x.Operand1));
            foreach (var x in Elements)
            {
                if (x.Staged == true)
                    return true;
                else if (x.Code == StatusCode.Modified && !requireStaging)
                {
                    return true;
                }
            }
            return false;
        }
        [Flags]
        internal enum StageFlags
        {
            Recorded = 1,
            Removed = 2,
            Renamed = 4,
            Conflicted = 8,
            MergeInfo = 16
        }
        int UpdatedFileTimeCount = 0;
        class StatusPercentage
        {
            public FileStatus Snapshot { get; set; }
            public System.Threading.ManualResetEvent EndEvent { get; set; }
        }

        internal Status(Area workspace, WorkspaceDB db, LocalDB ldb, FileStatus currentSnapshot, string restrictedPath = null, bool updateFileTimes = true, bool findCopies = true)
        {
            if (!string.IsNullOrEmpty(restrictedPath) && !restrictedPath.EndsWith("/"))
                restrictedPath += "/";
            System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
            sw.Start();
            StatusPercentage pct = new StatusPercentage()
            {
                Snapshot = currentSnapshot,
                EndEvent = new System.Threading.ManualResetEvent(false)
            };
            var progressLog = Task.Run(() =>
            {
                Printer.InteractivePrinter ip = null;
                while (true)
                {
                    if (pct.EndEvent.WaitOne(500))
                    {
                        if (ip != null)
                            ip.End(0);
                        return;
                    }
                    if (sw.ElapsedMilliseconds > 2000)
                    {
                        if (ip == null)
                            ip = Printer.CreateSpinnerPrinter(string.Format("Computing status for {0} objects", pct.Snapshot.Entries.Count), (obj) => { return string.Empty; });
                        ip.Update(0);
                    }
                }
            });
            RestrictedPath = restrictedPath;
            Workspace = workspace;
            CurrentVersion = db.Version;
            Branch = db.Branch;
            Elements = new List<StatusEntry>();
            var tasks = new List<Task<StatusEntry>>();
            Dictionary<string, Entry> snapshotData = new Dictionary<string, Entry>();
            foreach (var x in currentSnapshot.Entries)
                snapshotData[x.CanonicalName] = x;
            HashSet<Entry> foundEntries = new HashSet<Entry>();
            var records = db.GetCachedRecords(CurrentVersion);
            var stage = ldb.StageOperations;
            Stage = stage;
            VersionControlRecords = records;
            Dictionary<string, string> caseInsensitiveNames = new Dictionary<string, string>();
            foreach (var x in records)
                caseInsensitiveNames[x.CanonicalName.ToLowerInvariant()] = x.CanonicalName;
            MergeInputs = new List<Objects.Version>();
            Dictionary<string, StageFlags> stageInformation = new Dictionary<string, StageFlags>();
            Dictionary<Record, StatusEntry> statusMap = new Dictionary<Record, StatusEntry>();
            Dictionary<string, bool> parentIgnoredList = new Dictionary<string, bool>();
            foreach (var x in stage)
            {
                if (x.Type == LocalState.StageOperationType.Merge)
                    MergeInputs.Add(Workspace.GetVersion(new Guid(x.Operand1)));
                if (!x.IsFileOperation)
                    continue;
                StageFlags ops;
                if (!stageInformation.TryGetValue(x.Operand1, out ops))
                    stageInformation[x.Operand1] = ops;
                if (x.Type == LocalState.StageOperationType.Add)
                    ops |= StageFlags.Recorded;
                if (x.Type == LocalState.StageOperationType.Conflict)
                    ops |= StageFlags.Conflicted;
                if (x.Type == LocalState.StageOperationType.Remove)
                    ops |= StageFlags.Removed;
                if (x.Type == LocalState.StageOperationType.Rename)
                    ops |= StageFlags.Renamed;
                if (x.Type == LocalState.StageOperationType.MergeRecord)
                    ops |= StageFlags.MergeInfo;
                stageInformation[x.Operand1] = ops;
            }
            try
            {
                foreach (var x in records)
                {
                    tasks.Add(Utilities.LimitedTaskDispatcher.Factory.StartNew<StatusEntry>(() =>
                    {
                        StageFlags objectFlags;
                        stageInformation.TryGetValue(x.CanonicalName, out objectFlags);
                        Entry snapshotRecord = null;
                        if (RestrictedPath != null)
                        {
                            if (!x.CanonicalName.StartsWith(RestrictedPath, StringComparison.Ordinal) && x.CanonicalName != RestrictedPath)
                            {
                                if (x.CanonicalName == ".vrmeta" && !string.IsNullOrEmpty(Workspace.PartialPath))
                                {
                                    if (snapshotData.TryGetValue(x.CanonicalName, out snapshotRecord))
                                    {
                                        lock (foundEntries)
                                            foundEntries.Add(snapshotRecord);
                                    }
                                }
                                else
                                    return new StatusEntry() { Code = StatusCode.Ignored, FilesystemEntry = snapshotRecord, VersionControlRecord = x, Staged = objectFlags.HasFlag(StageFlags.Recorded) };
                            }
                        }

                        if (snapshotData.TryGetValue(x.CanonicalName, out snapshotRecord) && snapshotRecord.Ignored == false)
                        {
                            lock (foundEntries)
                                foundEntries.Add(snapshotRecord);

                            if (objectFlags.HasFlag(StageFlags.Removed))
                                Printer.PrintWarning("Removed object `{0}` still in filesystem!", x.CanonicalName);

                            if (objectFlags.HasFlag(StageFlags.Renamed))
                                Printer.PrintWarning("Renamed object `{0}` still in filesystem!", x.CanonicalName);

                            if (objectFlags.HasFlag(StageFlags.Conflicted))
                                return new StatusEntry() { Code = StatusCode.Conflict, FilesystemEntry = snapshotRecord, VersionControlRecord = x, Staged = objectFlags.HasFlag(StageFlags.Recorded) };

                            bool changed = false;
                            if (snapshotRecord.Length != x.Size)
                                changed = true;
                            if (!changed && snapshotRecord.IsSymlink && snapshotRecord.SymlinkTarget != x.Fingerprint)
                                changed = true;
                            if (!changed && !snapshotRecord.IsDirectory && !snapshotRecord.IsSymlink)
                            {
                                LocalState.FileTimestamp fst = Workspace.GetReferenceTime(x.CanonicalName);
                                if (snapshotRecord.ModificationTime == x.ModificationTime || (fst.DataIdentifier == x.DataIdentifier && snapshotRecord.ModificationTime == fst.LastSeenTime))
                                    changed = false;
                                else
                                {
                                    Printer.PrintDiagnostics("Computing hash for: " + x.CanonicalName);
                                    if (snapshotRecord.Hash != x.Fingerprint)
                                        changed = true;
                                    else
                                    {
                                        System.Threading.Interlocked.Increment(ref this.UpdatedFileTimeCount);
                                        Workspace.UpdateFileTimeCache(x.CanonicalName, x, snapshotRecord.ModificationTime, false);
                                    }
                                }
                            }
                            if (changed == true)
                                return new StatusEntry() { Code = StatusCode.Modified, FilesystemEntry = snapshotRecord, VersionControlRecord = x, Staged = objectFlags.HasFlag(StageFlags.Recorded) };
                            else
                            {
                                if (objectFlags.HasFlag(StageFlags.Recorded))
                                    Printer.PrintWarning("Unchanged object `{0}` still marked as recorded in commit!", x.CanonicalName);
                                return new StatusEntry() { Code = StatusCode.Unchanged, FilesystemEntry = snapshotRecord, VersionControlRecord = x, Staged = objectFlags.HasFlag(StageFlags.Recorded) };
                            }
                        }
                        else
                        {
                            if (objectFlags.HasFlag(StageFlags.Removed))
                                return new StatusEntry() { Code = StatusCode.Deleted, FilesystemEntry = null, VersionControlRecord = x, Staged = true };

                            string parentName = x.CanonicalName;
                            bool resolved = false;
                            while (true)
                            {
                                if (!parentName.Contains('/'))
                                    break;
                                if (parentName.EndsWith("/"))
                                    parentName = parentName.Substring(0, parentName.Length - 1);
                                parentName = parentName.Substring(0, parentName.LastIndexOf('/') + 1);
                                bool ignoredInParentList = false;
                                lock (parentIgnoredList)
                                {
                                    if (parentIgnoredList.TryGetValue(parentName, out ignoredInParentList))
                                    {
                                        if (ignoredInParentList)
                                            resolved = true;
                                        break;
                                    }
                                    else
                                    {
                                        Entry parentObjectEntry = null;
                                        snapshotData.TryGetValue(parentName, out parentObjectEntry);
                                        if (parentObjectEntry != null && parentObjectEntry.Ignored == true)
                                        {
                                            parentIgnoredList[parentName] = true;
                                            resolved = true;
                                            break;
                                        }
                                    }
                                }
                            }
{'fraction_non_alphanumeric': 0.07432234551430152, 'fraction_numerical': 0.0019524258891672904, 'mean_word_length': 1.1261934412619343, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 85, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1166141', 'n_tokens_mistral': 6922, 'n_tokens_neox': 6515, 'n_words': 1439}