<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import { ICollapseStateChangeEvent, ITreeElement, ITreeFilter, ITreeFilterDataResult, ITreeModel, ITreeNode, TreeVisibility, ITreeModelSpliceEvent, TreeError } from 'vs/base/browser/ui/tree/tree';
import { tail2 } from 'vs/base/common/arrays';
import { Emitter, Event, EventBufferer } from 'vs/base/common/event';
import { ISequence, Iterator } from 'vs/base/common/iterator';
import { ISpliceable } from 'vs/base/common/sequence';

interface IMutableTreeNode<T, TFilterData> extends ITreeNode<T, TFilterData> {
	readonly parent: IMutableTreeNode<T, TFilterData> | undefined;
	readonly children: IMutableTreeNode<T, TFilterData>[];
	visibleChildrenCount: number;
	visibleChildIndex: number;
	collapsible: boolean;
	collapsed: boolean;
	renderNodeCount: number;
	visible: boolean;
	filterData: TFilterData | undefined;
}

export function isFilterResult<T>(obj: any): obj is ITreeFilterDataResult<T> {
	return typeof obj === 'object' && 'visibility' in obj && 'data' in obj;
}

export function getVisibleState(visibility: boolean | TreeVisibility): TreeVisibility {
	switch (visibility) {
		case true: return TreeVisibility.Visible;
		case false: return TreeVisibility.Hidden;
		default: return visibility;
	}
}

function treeNodeToElement<T>(node: IMutableTreeNode<T, any>): ITreeElement<T> {
	const { element, collapsed } = node;
	const children = Iterator.map(Iterator.fromArray(node.children), treeNodeToElement);

	return { element, children, collapsed };
}

export interface IIndexTreeModelOptions<T, TFilterData> {
	readonly collapseByDefault?: boolean; // defaults to false
	readonly filter?: ITreeFilter<T, TFilterData>;
	readonly autoExpandSingleChildren?: boolean;
}

export class IndexTreeModel<T extends Exclude<any, undefined>, TFilterData = void> implements ITreeModel<T, TFilterData, number[]> {

	readonly rootRef = [];

	private root: IMutableTreeNode<T, TFilterData>;
	private eventBufferer = new EventBufferer();

	private _onDidChangeCollapseState = new Emitter<ICollapseStateChangeEvent<T, TFilterData>>();
	readonly onDidChangeCollapseState: Event<ICollapseStateChangeEvent<T, TFilterData>> = this.eventBufferer.wrapEvent(this._onDidChangeCollapseState.event);

	private _onDidChangeRenderNodeCount = new Emitter<ITreeNode<T, TFilterData>>();
	readonly onDidChangeRenderNodeCount: Event<ITreeNode<T, TFilterData>> = this.eventBufferer.wrapEvent(this._onDidChangeRenderNodeCount.event);

	private collapseByDefault: boolean;
	private filter?: ITreeFilter<T, TFilterData>;
	private autoExpandSingleChildren: boolean;

	private _onDidSplice = new Emitter<ITreeModelSpliceEvent<T, TFilterData>>();
	readonly onDidSplice = this._onDidSplice.event;

	constructor(
		private user: string,
		private list: ISpliceable<ITreeNode<T, TFilterData>>,
		rootElement: T,
		options: IIndexTreeModelOptions<T, TFilterData> = {}
	) {
		this.collapseByDefault = typeof options.collapseByDefault === 'undefined' ? false : options.collapseByDefault;
		this.filter = options.filter;
		this.autoExpandSingleChildren = typeof options.autoExpandSingleChildren === 'undefined' ? false : options.autoExpandSingleChildren;

		this.root = {
			parent: undefined,
			element: rootElement,
			children: [],
			depth: 0,
			visibleChildrenCount: 0,
			visibleChildIndex: -1,
			collapsible: false,
			collapsed: false,
			renderNodeCount: 0,
			visible: true,
			filterData: undefined
		};
	}

	splice(
		location: number[],
		deleteCount: number,
		toInsert?: ISequence<ITreeElement<T>>,
		onDidCreateNode?: (node: ITreeNode<T, TFilterData>) => void,
		onDidDeleteNode?: (node: ITreeNode<T, TFilterData>) => void
	): Iterator<ITreeElement<T>> {
		if (location.length === 0) {
			throw new TreeError(this.user, 'Invalid tree location');
		}

		const { parentNode, listIndex, revealed, visible } = this.getParentNodeWithListIndex(location);
		const treeListElementsToInsert: ITreeNode<T, TFilterData>[] = [];
		const nodesToInsertIterator = Iterator.map(Iterator.from(toInsert), el => this.createTreeNode(el, parentNode, parentNode.visible ? TreeVisibility.Visible : TreeVisibility.Hidden, revealed, treeListElementsToInsert, onDidCreateNode));

		const lastIndex = location[location.length - 1];

		// figure out what's the visible child start index right before the
		// splice point
		let visibleChildStartIndex = 0;

		for (let i = lastIndex; i >= 0 && i < parentNode.children.length; i--) {
			const child = parentNode.children[i];

			if (child.visible) {
				visibleChildStartIndex = child.visibleChildIndex;
				break;
			}
		}

		const nodesToInsert: IMutableTreeNode<T, TFilterData>[] = [];
		let insertedVisibleChildrenCount = 0;
		let renderNodeCount = 0;

		Iterator.forEach(nodesToInsertIterator, child => {
			nodesToInsert.push(child);
			renderNodeCount += child.renderNodeCount;

			if (child.visible) {
				child.visibleChildIndex = visibleChildStartIndex + insertedVisibleChildrenCount++;
			}
		});

		const deletedNodes = parentNode.children.splice(lastIndex, deleteCount, ...nodesToInsert);

		// figure out what is the count of deleted visible children
		let deletedVisibleChildrenCount = 0;

		for (const child of deletedNodes) {
			if (child.visible) {
				deletedVisibleChildrenCount++;
			}
		}

		// and adjust for all visible children after the splice point
		if (deletedVisibleChildrenCount !== 0) {
			for (let i = lastIndex + nodesToInsert.length; i < parentNode.children.length; i++) {
				const child = parentNode.children[i];

				if (child.visible) {
					child.visibleChildIndex -= deletedVisibleChildrenCount;
				}
			}
		}

		// update parent's visible children count
		parentNode.visibleChildrenCount += insertedVisibleChildrenCount - deletedVisibleChildrenCount;

		if (revealed && visible) {
			const visibleDeleteCount = deletedNodes.reduce((r, node) => r + node.renderNodeCount, 0);

			this._updateAncestorsRenderNodeCount(parentNode, renderNodeCount - visibleDeleteCount);
			this.list.splice(listIndex, visibleDeleteCount, treeListElementsToInsert);
		}

		if (deletedNodes.length > 0 && onDidDeleteNode) {
			const visit = (node: ITreeNode<T, TFilterData>) => {
				onDidDeleteNode(node);
				node.children.forEach(visit);
			};

			deletedNodes.forEach(visit);
		}

		const result = Iterator.map(Iterator.fromArray(deletedNodes), treeNodeToElement);
		this._onDidSplice.fire({ insertedNodes: nodesToInsert, deletedNodes });
		return result;
	}

	rerender(location: number[]): void {
		if (location.length === 0) {
			throw new TreeError(this.user, 'Invalid tree location');
		}

		const { node, listIndex, revealed } = this.getTreeNodeWithListIndex(location);

		if (revealed) {
			this.list.splice(listIndex, 1, [node]);
		}
	}

	getListIndex(location: number[]): number {
		const { listIndex, visible, revealed } = this.getTreeNodeWithListIndex(location);
		return visible && revealed ? listIndex : -1;
	}

	getListRenderCount(location: number[]): number {
		return this.getTreeNode(location).renderNodeCount;
	}

	isCollapsible(location: number[]): boolean {
		return this.getTreeNode(location).collapsible;
	}

	setCollapsible(location: number[], collapsible?: boolean): boolean {
		const node = this.getTreeNode(location);

		if (typeof collapsible === 'undefined') {
			collapsible = !node.collapsible;
		} else if (node.collapsible === collapsible) {
			return false;
		}

		node.collapsible = collapsible;
		this.eventBufferer.bufferEvents(() => this._setCollapsed(location, node.collapsed));
		return true;
	}

	isCollapsed(location: number[]): boolean {
		return this.getTreeNode(location).collapsed;
	}

	setCollapsed(location: number[], collapsed?: boolean, recursive?: boolean): boolean {
		const node = this.getTreeNode(location);

		if (typeof collapsed === 'undefined') {
			collapsed = !node.collapsed;
		}

		return this.eventBufferer.bufferEvents(() => this._setCollapsed(location, collapsed!, recursive));
	}

	private _setCollapsed(location: number[], collapsed: boolean, recursive?: boolean): boolean {
		const { node, listIndex, revealed } = this.getTreeNodeWithListIndex(location);

		const result = this._setListNodeCollapsed(node, listIndex, revealed, collapsed!, recursive || false);

		if (node !== this.root && this.autoExpandSingleChildren && !collapsed! && !recursive) {
			let onlyVisibleChildIndex = -1;

			for (let i = 0; i < node.children.length; i++) {
				const child = node.children[i];

				if (child.visible) {
					if (onlyVisibleChildIndex > -1) {
						onlyVisibleChildIndex = -1;
						break;
					} else {
						onlyVisibleChildIndex = i;
					}
				}
			}

			if (onlyVisibleChildIndex > -1) {
				this._setCollapsed([...location, onlyVisibleChildIndex], false, false);
			}
		}

		return result;
	}

	private _setListNodeCollapsed(node: IMutableTreeNode<T, TFilterData>, listIndex: number, revealed: boolean, collapsed: boolean, recursive: boolean): boolean {
		const result = this._setNodeCollapsed(node, collapsed, recursive, false);

		if (!revealed || !node.visible) {
			return result;
		}

		const previousRenderNodeCount = node.renderNodeCount;
		const toInsert = this.updateNodeAfterCollapseChange(node);
		const deleteCount = previousRenderNodeCount - (listIndex === -1 ? 0 : 1);
		this.list.splice(listIndex + 1, deleteCount, toInsert.slice(1));

		return result;
	}

	private _setNodeCollapsed(node: IMutableTreeNode<T, TFilterData>, collapsed: boolean, recursive: boolean, deep: boolean): boolean {
		let result = node.collapsible && node.collapsed !== collapsed;

		if (node.collapsible) {
			node.collapsed = collapsed;

			if (result) {
				this._onDidChangeCollapseState.fire({ node, deep });
			}
		}

		if (recursive) {
			for (const child of node.children) {
				result = this._setNodeCollapsed(child, collapsed, true, true) || result;
			}
		}

		return result;
	}

	expandTo(location: number[]): void {
		this.eventBufferer.bufferEvents(() => {
			let node = this.getTreeNode(location);

			while (node.parent) {
				node = node.parent;
				location = location.slice(0, location.length - 1);

				if (node.collapsed) {
					this._setCollapsed(location, false);
				}
			}
		});
	}

	refilter(): void {
		const previousRenderNodeCount = this.root.renderNodeCount;
		const toInsert = this.updateNodeAfterFilterChange(this.root);
		this.list.splice(0, previousRenderNodeCount, toInsert);
	}

	private createTreeNode(
		treeElement: ITreeElement<T>,
		parent: IMutableTreeNode<T, TFilterData>,
		parentVisibility: TreeVisibility,
		revealed: boolean,
		treeListElements: ITreeNode<T, TFilterData>[],
		onDidCreateNode?: (node: ITreeNode<T, TFilterData>) => void
	): IMutableTreeNode<T, TFilterData> {
		const node: IMutableTreeNode<T, TFilterData> = {
			parent,
			element: treeElement.element,
			children: [],
			depth: parent.depth + 1,
			visibleChildrenCount: 0,
			visibleChildIndex: -1,
			collapsible: typeof treeElement.collapsible === 'boolean' ? treeElement.collapsible : (typeof treeElement.collapsed !== 'undefined'),<commit_msg>IndexTreeModel: complete setCollapsible()<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import { ICollapseStateChangeEvent, ITreeElement, ITreeFilter, ITreeFilterDataResult, ITreeModel, ITreeNode, TreeVisibility, ITreeModelSpliceEvent, TreeError } from 'vs/base/browser/ui/tree/tree';
import { tail2 } from 'vs/base/common/arrays';
import { Emitter, Event, EventBufferer } from 'vs/base/common/event';
import { ISequence, Iterator } from 'vs/base/common/iterator';
import { ISpliceable } from 'vs/base/common/sequence';

interface IMutableTreeNode<T, TFilterData> extends ITreeNode<T, TFilterData> {
	readonly parent: IMutableTreeNode<T, TFilterData> | undefined;
	readonly children: IMutableTreeNode<T, TFilterData>[];
	visibleChildrenCount: number;
	visibleChildIndex: number;
	collapsible: boolean;
	collapsed: boolean;
	renderNodeCount: number;
	visible: boolean;
	filterData: TFilterData | undefined;
}

export function isFilterResult<T>(obj: any): obj is ITreeFilterDataResult<T> {
	return typeof obj === 'object' && 'visibility' in obj && 'data' in obj;
}

export function getVisibleState(visibility: boolean | TreeVisibility): TreeVisibility {
	switch (visibility) {
		case true: return TreeVisibility.Visible;
		case false: return TreeVisibility.Hidden;
		default: return visibility;
	}
}

function treeNodeToElement<T>(node: IMutableTreeNode<T, any>): ITreeElement<T> {
	const { element, collapsed } = node;
	const children = Iterator.map(Iterator.fromArray(node.children), treeNodeToElement);

	return { element, children, collapsed };
}

export interface IIndexTreeModelOptions<T, TFilterData> {
	readonly collapseByDefault?: boolean; // defaults to false
	readonly filter?: ITreeFilter<T, TFilterData>;
	readonly autoExpandSingleChildren?: boolean;
}

export class IndexTreeModel<T extends Exclude<any, undefined>, TFilterData = void> implements ITreeModel<T, TFilterData, number[]> {

	readonly rootRef = [];

	private root: IMutableTreeNode<T, TFilterData>;
	private eventBufferer = new EventBufferer();

	private _onDidChangeCollapseState = new Emitter<ICollapseStateChangeEvent<T, TFilterData>>();
	readonly onDidChangeCollapseState: Event<ICollapseStateChangeEvent<T, TFilterData>> = this.eventBufferer.wrapEvent(this._onDidChangeCollapseState.event);

	private _onDidChangeRenderNodeCount = new Emitter<ITreeNode<T, TFilterData>>();
	readonly onDidChangeRenderNodeCount: Event<ITreeNode<T, TFilterData>> = this.eventBufferer.wrapEvent(this._onDidChangeRenderNodeCount.event);

	private collapseByDefault: boolean;
	private filter?: ITreeFilter<T, TFilterData>;
	private autoExpandSingleChildren: boolean;

	private _onDidSplice = new Emitter<ITreeModelSpliceEvent<T, TFilterData>>();
	readonly onDidSplice = this._onDidSplice.event;

	constructor(
		private user: string,
		private list: ISpliceable<ITreeNode<T, TFilterData>>,
		rootElement: T,
		options: IIndexTreeModelOptions<T, TFilterData> = {}
	) {
		this.collapseByDefault = typeof options.collapseByDefault === 'undefined' ? false : options.collapseByDefault;
		this.filter = options.filter;
		this.autoExpandSingleChildren = typeof options.autoExpandSingleChildren === 'undefined' ? false : options.autoExpandSingleChildren;

		this.root = {
			parent: undefined,
			element: rootElement,
			children: [],
			depth: 0,
			visibleChildrenCount: 0,
			visibleChildIndex: -1,
			collapsible: false,
			collapsed: false,
			renderNodeCount: 0,
			visible: true,
			filterData: undefined
		};
	}

	splice(
		location: number[],
		deleteCount: number,
		toInsert?: ISequence<ITreeElement<T>>,
		onDidCreateNode?: (node: ITreeNode<T, TFilterData>) => void,
		onDidDeleteNode?: (node: ITreeNode<T, TFilterData>) => void
	): Iterator<ITreeElement<T>> {
		if (location.length === 0) {
			throw new TreeError(this.user, 'Invalid tree location');
		}

		const { parentNode, listIndex, revealed, visible } = this.getParentNodeWithListIndex(location);
		const treeListElementsToInsert: ITreeNode<T, TFilterData>[] = [];
		const nodesToInsertIterator = Iterator.map(Iterator.from(toInsert), el => this.createTreeNode(el, parentNode, parentNode.visible ? TreeVisibility.Visible : TreeVisibility.Hidden, revealed, treeListElementsToInsert, onDidCreateNode));

		const lastIndex = location[location.length - 1];

		// figure out what's the visible child start index right before the
		// splice point
		let visibleChildStartIndex = 0;

		for (let i = lastIndex; i >= 0 && i < parentNode.children.length; i--) {
			const child = parentNode.children[i];

			if (child.visible) {
				visibleChildStartIndex = child.visibleChildIndex;
				break;
			}
		}

		const nodesToInsert: IMutableTreeNode<T, TFilterData>[] = [];
		let insertedVisibleChildrenCount = 0;
		let renderNodeCount = 0;

		Iterator.forEach(nodesToInsertIterator, child => {
			nodesToInsert.push(child);
			renderNodeCount += child.renderNodeCount;

			if (child.visible) {
				child.visibleChildIndex = visibleChildStartIndex + insertedVisibleChildrenCount++;
			}
		});

		const deletedNodes = parentNode.children.splice(lastIndex, deleteCount, ...nodesToInsert);

		// figure out what is the count of deleted visible children
		let deletedVisibleChildrenCount = 0;

		for (const child of deletedNodes) {
			if (child.visible) {
				deletedVisibleChildrenCount++;
			}
		}

		// and adjust for all visible children after the splice point
		if (deletedVisibleChildrenCount !== 0) {
			for (let i = lastIndex + nodesToInsert.length; i < parentNode.children.length; i++) {
				const child = parentNode.children[i];

				if (child.visible) {
					child.visibleChildIndex -= deletedVisibleChildrenCount;
				}
			}
		}

		// update parent's visible children count
		parentNode.visibleChildrenCount += insertedVisibleChildrenCount - deletedVisibleChildrenCount;

		if (revealed && visible) {
			const visibleDeleteCount = deletedNodes.reduce((r, node) => r + node.renderNodeCount, 0);

			this._updateAncestorsRenderNodeCount(parentNode, renderNodeCount - visibleDeleteCount);
			this.list.splice(listIndex, visibleDeleteCount, treeListElementsToInsert);
		}

		if (deletedNodes.length > 0 && onDidDeleteNode) {
			const visit = (node: ITreeNode<T, TFilterData>) => {
				onDidDeleteNode(node);
				node.children.forEach(visit);
			};

			deletedNodes.forEach(visit);
		}

		const result = Iterator.map(Iterator.fromArray(deletedNodes), treeNodeToElement);
		this._onDidSplice.fire({ insertedNodes: nodesToInsert, deletedNodes });
		return result;
	}

	rerender(location: number[]): void {
		if (location.length === 0) {
			throw new TreeError(this.user, 'Invalid tree location');
		}

		const { node, listIndex, revealed } = this.getTreeNodeWithListIndex(location);

		if (revealed) {
			this.list.splice(listIndex, 1, [node]);
		}
	}

	getListIndex(location: number[]): number {
		const { listIndex, visible, revealed } = this.getTreeNodeWithListIndex(location);
		return visible && revealed ? listIndex : -1;
	}

	getListRenderCount(location: number[]): number {
		return this.getTreeNode(location).renderNodeCount;
	}

	isCollapsible(location: number[]): boolean {
		return this.getTreeNode(location).collapsible;
	}

	setCollapsible(location: number[], collapsible?: boolean): boolean {
		const node = this.getTreeNode(location);

		if (typeof collapsible === 'undefined') {
			collapsible = !node.collapsible;
		}

		return this.eventBufferer.bufferEvents(() => this._setCollapseState(location, collapsible!, node.collapsed));
	}

	isCollapsed(location: number[]): boolean {
		return this.getTreeNode(location).collapsed;
	}

	setCollapsed(location: number[], collapsed?: boolean, recursive?: boolean): boolean {
		const node = this.getTreeNode(location);

		if (typeof collapsed === 'undefined') {
			collapsed = !node.collapsed;
		}

		return this.eventBufferer.bufferEvents(() => this._setCollapseState(location, node.collapsible, collapsed!, recursive));
	}

	private _setCollapseState(location: number[], collapsible: boolean, collapsed: boolean, recursive?: boolean): boolean {
		const { node, listIndex, revealed } = this.getTreeNodeWithListIndex(location);

		const result = this._setListNodeCollapseState(node, listIndex, revealed, collapsible, collapsed, recursive || false);

		if (node !== this.root && this.autoExpandSingleChildren && !collapsed! && !recursive) {
			let onlyVisibleChildIndex = -1;

			for (let i = 0; i < node.children.length; i++) {
				const child = node.children[i];

				if (child.visible) {
					if (onlyVisibleChildIndex > -1) {
						onlyVisibleChildIndex = -1;
						break;
					} else {
						onlyVisibleChildIndex = i;
					}
				}
			}

			if (onlyVisibleChildIndex > -1) {
				this._setCollapseState([...location, onlyVisibleChildIndex], false, false);
			}
		}

		return result;
	}

	private _setListNodeCollapseState(node: IMutableTreeNode<T, TFilterData>, listIndex: number, revealed: boolean, collapsible: boolean, collapsed: boolean, recursive: boolean): boolean {
		const result = this._setNodeCollapseState(node, collapsible, collapsed, recursive, false);

		if (!revealed || !node.visible) {
			return result;
		}

		const previousRenderNodeCount = node.renderNodeCount;
		const toInsert = this.updateNodeAfterCollapseChange(node);
		const deleteCount = previousRenderNodeCount - (listIndex === -1 ? 0 : 1);
		this.list.splice(listIndex + 1, deleteCount, toInsert.slice(1));

		return result;
	}

	private _setNodeCollapseState(node: IMutableTreeNode<T, TFilterData>, collapsible: boolean, collapsed: boolean, recursive: boolean, deep: boolean): boolean {
		let result = node.collapsible !== collapsible || node.collapsed !== collapsed;

		node.collapsible = collapsible;
		node.collapsed = collapsed;

		if (result) {
			this._onDidChangeCollapseState.fire({ node, deep });
		}

		if (recursive) {
			for (const child of node.children) {
				result = this._setNodeCollapseState(child, collapsible, collapsed, true, true) || result;
			}
		}

		return result;
	}

	expandTo(location: number[]): void {
		this.eventBufferer.bufferEvents(() => {
			let node = this.getTreeNode(location);

			while (node.parent) {
				node = node.parent;
				location = location.slice(0, location.length - 1);

				if (node.collapsed) {
					this._setCollapseState(location, node.collapsible, false);
				}
			}
		});
	}

	refilter(): void {
		const previousRenderNodeCount = this.root.renderNodeCount;
		const toInsert = this.updateNodeAfterFilterChange(this.root);
		this.list.splice(0, previousRenderNodeCount, toInsert);
	}

	private createTreeNode(
		treeElement: ITreeElement<T>,
		parent: IMutableTreeNode<T, TFilterData>,
		parentVisibility: TreeVisibility,
		revealed: boolean,
		treeListElements: ITreeNode<T, TFilterData>[],
		onDidCreateNode?: (node: ITreeNode<T, TFilterData>) => void
	): IMutableTreeNode<T, TFilterData> {
		const node: IMutableTreeNode<T, TFilterData> = {
			parent,
			element: treeElement.element,
			children: [],
			depth: parent.depth + 1,
			visibleChildrenCount: 0,
			visibleChildIndex: -1,
			collapsible: typeof treeElement.collapsible === 'boolean' ? treeElement.collapsible : (typeof treeElement.collapsed !== 'undefined'),
{'fraction_non_alphanumeric': 0.13238137082601054, 'fraction_numerical': 0.0029876977152899823, 'mean_word_length': 4.5111380145278455, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 123, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6604908', 'n_tokens_mistral': 7564, 'n_tokens_neox': 6719, 'n_words': 1571}