<commit_before>using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Web;
using System.Xml;
using System.Xml.Linq;
using Bloom.Api;
using Bloom.Collection;
using Bloom.Edit;
using Bloom.ToPalaso;
using L10NSharp;
using Microsoft.CSharp.RuntimeBinder;
using SIL.Code;
using SIL.Extensions;
using SIL.Linq;
using SIL.Reporting;
using SIL.Text;
using SIL.WritingSystems;
using SIL.Xml;
using TagLib;
using File = System.IO.File;

namespace Bloom.Book
{
	/// <summary>
	/// This class manages the "data-*" elements of a bloom document.
	/// </summary>
	/// <remarks>
	/// At the beginning of the document, we have a special div for holding book-wide data.
	/// It may hosts all manner of data about the book, including copyright, what languages are currently visible, etc.Here's a sample of a simple one:
	/*<div id="bloomDataDiv">
			  <div data-book="bookTitle" lang="en">Awito Builds a toilet</div>
			  <div data-book="bookTitle" lang="tpi">Awito i wokim haus</div>
			  <div data-book="coverImage" lang="*">tmpABDB.png</div>
			  <div data-book="topic" lang="tpi">Health</div>
			  <div data-book="contentLanguage1" lang="*">en</div>
			  <div data-book="contentLanguage2" lang="*">tpi</div>
			  <div data-book="copyright" lang="*">Copyright © 1994, National Department of Education</div>
			  <div data-book="licenseImage" lang="*">license.png?1348557455942</div>
			  <div data-book="licenseUrl" lang="en">http://creativecommons.org/licenses/by-nc-sa/3.0/</div>
			  <div data-book="licenseDescription" lang="en">You may not use this work for commercial purposes. You may adapt or build upon this work, but you may distribute the resulting work only under the same or similar license to this one.You must attribute the work in the manner specified by the author.</div>
			  <div data-book="originalAcknowledgments" lang="tpi">Book Development by:  Curriculum Development Division</div>
			</div>
			*/
	/// After the bloomDataDiv, elements with "data-*" attributes can occur throughout a book, for example on the cover page:
	/*    <div class="bloom-page">
		<div class="bloom-translationGroup coverTitle">
		  <div data-book="bookTitle" lang="en">Awito Builds a house</div>
		  <div data-book="bookTitle" lang="tpi">Awito i wokim haus</div>
		</div>
	*/
	/// This class must keep these in sync
	/// There is also a file meta.json which contains data that is also kept online to aid in searching for books. Some of this must also be kept
	/// in sync with data in the html, for example, metadata.volumeInfo.title should match (currently the English alternative of) the content of the
	/// bloomDataDiv bookTitle div.
	/// </remarks>
	public class BookData
	{
		/// <summary>
		/// This is the attribute name used in the data div for a rather complex data-model.
		/// Specifically, an xmatter page sets a specific page name as the value of this attribute.
		/// That value is used as a key to link the attributes stored in the data div with the attributes on the page itself.
		/// This allows us to keep the values persisted (since the xmatter is constantly being regenerated).
		///
		/// E.g.
		/// In the data div, we have a div such as
		/// &lt;div data-xmatter-page="frontCover" data-backgroundaudio="SoundTrack0.mp3" data-backgroundaudiovolume="0.5717869999999999"&gt;&lt;/div&gt;
		///
		/// The actual xmatter page will also have these same attributes (along with all the rest it has).
		/// &lt;div class="bloom-page cover coverColor bloom-frontMatter frontCover Device16x9Landscape layout-style-Default side-right"
		/// id="76ed5d5b-c178-4db1-8be1-4a2f63eccaa4"
		/// data-xmatter-page="frontCover" data-backgroundaudio="SoundTrack0.mp3" data-backgroundaudiovolume="0.5717869999999999"
		/// lang=""&gt;
		/// </summary>
		private const string kDataXmatterPage = "data-xmatter-page";

		private readonly HtmlDom _dom;
		private readonly Action<XmlElement> _updateImgNode;
		internal readonly CollectionSettings CollectionSettings;
		private readonly DataSet _dataset;
		private XmlElement _dataDiv;
		private Object thisLock = new Object();
		private XmlString _cachedMcl2;
		private XmlString _cachedMcl3;
		private bool _gotMclCache;

		//URLs are encoded in a certain way for the src attributes
		//If they have certain symbols (namely &), they need to be encoded differently
		//when saved as xml in the data-div. Ideally, we might use an abstraction over
		//string which know what encoding things are in. Or we could work hard to always
		//keep strings unencoded in this class and DataSet in then encode them as they
		//get written out to various places. But either of those is too disruptive at this
		//point so this is a simple solution. Ref BL-3235.
		public HashSet<string> KeysOfVariablesThatAreUrlEncoded = new HashSet<string>();

		/// <param name="dom">Set this parameter to, say, a page that the user just edited, to limit reading to it, so its values don't get overriden by previous pages.
		///   Supply the whole dom if nothing has priority (which will mean the data-div will win, because it is first)</param>
		/// <param name="collectionSettings"> </param>
		/// <param name="updateImgNodeCallback">This is a callback so as not to introduce dependencies on ImageUpdater & the current folder path</param>
		public BookData(HtmlDom dom, CollectionSettings collectionSettings, Action<XmlElement> updateImgNodeCallback)
		{
			_dom = dom;
			_updateImgNode = updateImgNodeCallback;
			CollectionSettings = collectionSettings;
			GetOrCreateDataDiv();
			_dataset = GatherDataItemsFromCollectionSettings(CollectionSettings);
			GatherDataItemsFromXElement(_dataset,_dom.RawDom);
			MigrateData(_dataset);
		}

		/// <summary>
		/// For bilingual or trilingual books, this is the second language to show, after the vernacular.
		/// </summary>
		public XmlString MultilingualContentLanguage2
		{
			get
			{
				if (!_gotMclCache)
					CacheMclData();
				return _cachedMcl2;
			}
		}

		/// <summary>
		/// Cache values for MultilingualContentLanguage2 and MultilingualContentLanguage3
		/// It is safe to cache this because we reload everything when changing languages.
		/// It is important because these properties are heavily used, especially when saving pages.
		/// </summary>
		private void CacheMclData()
		{
			GatherDataItemsFromXElement(_dataset, _dom.RawDom);
			_cachedMcl2 = GetVariableOrNull("contentLanguage2", "*");
			_cachedMcl3 = GetVariableOrNull("contentLanguage3", "*");
			_gotMclCache = true;
		}

		/// <summary>
		/// For trilingual books, this is the third language to show
		/// </summary>
		public XmlString MultilingualContentLanguage3
		{
			get
			{
				if (!_gotMclCache)
					CacheMclData();
				return _cachedMcl3;
			}
		}

		/// <summary>
		/// A book-level style number sequence
		/// </summary>
		public int StyleNumberSequence
		{
			get
			{
				lock(thisLock)
				{
					GatherDataItemsFromXElement(_dataset, _dom.RawDom);
					string curSeqStr = GetVariableOrNull("styleNumberSequence", "*").Unencoded;
					int curSeq;
					int nextSeq = 1;
					if (Int32.TryParse(curSeqStr, out curSeq))
						nextSeq = curSeq + 1;
					Set("styleNumberSequence", XmlString.FromUnencoded(nextSeq.ToString(CultureInfo.InvariantCulture)),
						false);
					return nextSeq;
				}
			}
		}


		public void UpdateVariablesAndDataDivThroughDOM(BookInfo info = null)
		{
			// For some reason, cleaning up the anchor elements in the div#bloomDataDiv doesn't work
			// when called from HtmlDom.ProcessPageAfterEditing.  Doing it here works.
			var div = _dom.Body.SelectSingleNode("div[@id='bloomDataDiv']") as XmlElement;
			if (div != null)
				HtmlDom.CleanupAnchorElements(div);
			UpdateVariablesAndDataDiv(_dom.RawDom.FirstChild, info);
		}


		/// <summary>
		/// Create or update the data div with all the data-book values in the document
		/// </summary>
		/// <param name="dom">This is either the whole document, or a page div that we just edited and want to read from.</param>
		public void SuckInDataFromEditedDom(HtmlDom dom, BookInfo info = null)
		{
			UpdateVariablesAndDataDiv(dom.RawDom, info);
		}

		public void SynchronizeDataItemsThroughoutDOM()
		{
			var itemsToDelete = new HashSet<Tuple<string, string>>();
			SynchronizeDataItemsFromContentsOfElement(_dom.Body, itemsToDelete);
		}

		/// <summary>
		/// Create or update the data div with all the data-book values in the document
		/// </summary>
		/// <param name="elementToReadFrom">This is either the whole document, or a page div that we just edited and want to read from.</param>
		private void UpdateVariablesAndDataDiv(XmlNode elementToReadFrom, BookInfo info = null)
		{
			//Debug.WriteLine("before update: " + _dataDiv.OuterXml);

			var itemsToDelete = new HashSet<Tuple<string, string>>();
			DataSet incomingData = SynchronizeDataItemsFromContentsOfElement(elementToReadFrom, itemsToDelete);
			UpdateToolRelatedDataFromBookInfo(info, incomingData, itemsToDelete);
			incomingData.UpdateGenericLanguageString("contentLanguage1", XmlString.FromUnencoded(Language1.Iso639Code), false);
			incomingData.UpdateGenericLanguageString("contentLanguage2",
											 XmlString.IsNullOrEmpty(MultilingualContentLanguage2)
												 ? null
												 : MultilingualContentLanguage2, false);
			incomingData.UpdateGenericLanguageString("contentLanguage3",
											 XmlString.IsNullOrEmpty(MultilingualContentLanguage3)
												 ? null
												 : MultilingualContentLanguage3, false);

			//Debug.WriteLine("xyz: " + _dataDiv.OuterXml);
			foreach (var v in incomingData.TextVariables)
			{
				if (!v.Value.IsCollectionValue)
					UpdateSingleTextVariableInDataDiv(v.Key,v.Value);
			}
			foreach (var tuple in itemsToDelete)
				UpdateSingleTextVariableInDataDiv(tuple.Item1, tuple.Item2, XmlString.Empty);
			foreach (var attributeSet in incomingData.XmatterPageDataAttributeSets)
				PushXmatterPageAttributesIntoDataDiv(attributeSet);
			//Debug.WriteLine("after update: " + _dataDiv.OuterXml);

			UpdateTitle(info);//this may change our "bookTitle" variable if the title is based on a template that reads other variables (e.g. "Primer Term2-Week3")
			UpdateIsbn(info);
			if (info != null)
				UpdateBookInfoTags(info);
			UpdateCredits(info);
		}

		private void UpdateToolRelatedDataFromBookInfo(BookInfo info, DataSet incomingData, HashSet<Tuple<string, string>> itemsToDelete)
		{
			if (info == null)
				return; // only in tests
			var tools = info.Tools;
			var bookClass = _dom.Body.Attributes["class"]?.Value ?? "";

			if (!bookClass.Contains("leveled-reader") && !bookClass.Contains("decodable-reader"))
			{
				incomingData.UpdateGenericLanguageString("levelOrStageNumber", XmlString.Empty, false);
				itemsToDelete.Add(new Tuple<string, string>("levelOrStageNumber", "*"));
				return;
			}

			var levelTool = tools.FirstOrDefault(t => t.ToolId == "leveledReader");
			if (levelTool != null && bookClass.Contains("leveled-reader"))
			{
				var level = levelTool.State;
				incomingData.UpdateGenericLanguageString("levelOrStageNumber", XmlString.FromUnencoded(level), false);
				itemsToDelete.RemoveWhere(item => item.Item1 == "levelOrStageNumber");
			}

			var decodableTool = tools.FirstOrDefault(t => t.ToolId == "decodableReader");
			if (decodableTool != null && bookClass.Contains("decodable-reader"))
			{
				var stageString = decodableTool.State.Split(';').FirstOrDefault()?.Split(':').Skip(1).FirstOrDefault();
				if (!string.IsNullOrEmpty(stageString))
				{
					incomingData.UpdateGenericLanguageString("levelOrStageNumber", XmlString.FromUnencoded(stageString), false);
					itemsToDelete.RemoveWhere(item => item.Item1 == "levelOrStageNumber");
				}

				int stage;
				if (int.TryParse(stageString, out stage) && CollectionSettings != null)
				{
					var settingsPath = DecodableReaderToolSettings.GetReaderToolsSettingsFilePath(CollectionSettings);
					if (File.Exists(settingsPath))
					{
						try
						{
							var settingsJson = File.ReadAllText(settingsPath, Encoding.UTF8);
							var settings = DynamicJson.Parse(settingsJson);
							var stages = settings.stages;
							var allLetters = "";
							for (var i=0; i<stage;i++)<commit_msg>Fix a null pointer issue (BL-9978)<commit_after>using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Web;
using System.Xml;
using System.Xml.Linq;
using Bloom.Api;
using Bloom.Collection;
using Bloom.Edit;
using Bloom.ToPalaso;
using L10NSharp;
using Microsoft.CSharp.RuntimeBinder;
using SIL.Code;
using SIL.Extensions;
using SIL.Linq;
using SIL.Reporting;
using SIL.Text;
using SIL.WritingSystems;
using SIL.Xml;
using TagLib;
using File = System.IO.File;

namespace Bloom.Book
{
	/// <summary>
	/// This class manages the "data-*" elements of a bloom document.
	/// </summary>
	/// <remarks>
	/// At the beginning of the document, we have a special div for holding book-wide data.
	/// It may hosts all manner of data about the book, including copyright, what languages are currently visible, etc.Here's a sample of a simple one:
	/*<div id="bloomDataDiv">
			  <div data-book="bookTitle" lang="en">Awito Builds a toilet</div>
			  <div data-book="bookTitle" lang="tpi">Awito i wokim haus</div>
			  <div data-book="coverImage" lang="*">tmpABDB.png</div>
			  <div data-book="topic" lang="tpi">Health</div>
			  <div data-book="contentLanguage1" lang="*">en</div>
			  <div data-book="contentLanguage2" lang="*">tpi</div>
			  <div data-book="copyright" lang="*">Copyright © 1994, National Department of Education</div>
			  <div data-book="licenseImage" lang="*">license.png?1348557455942</div>
			  <div data-book="licenseUrl" lang="en">http://creativecommons.org/licenses/by-nc-sa/3.0/</div>
			  <div data-book="licenseDescription" lang="en">You may not use this work for commercial purposes. You may adapt or build upon this work, but you may distribute the resulting work only under the same or similar license to this one.You must attribute the work in the manner specified by the author.</div>
			  <div data-book="originalAcknowledgments" lang="tpi">Book Development by:  Curriculum Development Division</div>
			</div>
			*/
	/// After the bloomDataDiv, elements with "data-*" attributes can occur throughout a book, for example on the cover page:
	/*    <div class="bloom-page">
		<div class="bloom-translationGroup coverTitle">
		  <div data-book="bookTitle" lang="en">Awito Builds a house</div>
		  <div data-book="bookTitle" lang="tpi">Awito i wokim haus</div>
		</div>
	*/
	/// This class must keep these in sync
	/// There is also a file meta.json which contains data that is also kept online to aid in searching for books. Some of this must also be kept
	/// in sync with data in the html, for example, metadata.volumeInfo.title should match (currently the English alternative of) the content of the
	/// bloomDataDiv bookTitle div.
	/// </remarks>
	public class BookData
	{
		/// <summary>
		/// This is the attribute name used in the data div for a rather complex data-model.
		/// Specifically, an xmatter page sets a specific page name as the value of this attribute.
		/// That value is used as a key to link the attributes stored in the data div with the attributes on the page itself.
		/// This allows us to keep the values persisted (since the xmatter is constantly being regenerated).
		///
		/// E.g.
		/// In the data div, we have a div such as
		/// &lt;div data-xmatter-page="frontCover" data-backgroundaudio="SoundTrack0.mp3" data-backgroundaudiovolume="0.5717869999999999"&gt;&lt;/div&gt;
		///
		/// The actual xmatter page will also have these same attributes (along with all the rest it has).
		/// &lt;div class="bloom-page cover coverColor bloom-frontMatter frontCover Device16x9Landscape layout-style-Default side-right"
		/// id="76ed5d5b-c178-4db1-8be1-4a2f63eccaa4"
		/// data-xmatter-page="frontCover" data-backgroundaudio="SoundTrack0.mp3" data-backgroundaudiovolume="0.5717869999999999"
		/// lang=""&gt;
		/// </summary>
		private const string kDataXmatterPage = "data-xmatter-page";

		private readonly HtmlDom _dom;
		private readonly Action<XmlElement> _updateImgNode;
		internal readonly CollectionSettings CollectionSettings;
		private readonly DataSet _dataset;
		private XmlElement _dataDiv;
		private Object thisLock = new Object();
		private XmlString _cachedMcl2;
		private XmlString _cachedMcl3;
		private bool _gotMclCache;

		//URLs are encoded in a certain way for the src attributes
		//If they have certain symbols (namely &), they need to be encoded differently
		//when saved as xml in the data-div. Ideally, we might use an abstraction over
		//string which know what encoding things are in. Or we could work hard to always
		//keep strings unencoded in this class and DataSet in then encode them as they
		//get written out to various places. But either of those is too disruptive at this
		//point so this is a simple solution. Ref BL-3235.
		public HashSet<string> KeysOfVariablesThatAreUrlEncoded = new HashSet<string>();

		/// <param name="dom">Set this parameter to, say, a page that the user just edited, to limit reading to it, so its values don't get overriden by previous pages.
		///   Supply the whole dom if nothing has priority (which will mean the data-div will win, because it is first)</param>
		/// <param name="collectionSettings"> </param>
		/// <param name="updateImgNodeCallback">This is a callback so as not to introduce dependencies on ImageUpdater & the current folder path</param>
		public BookData(HtmlDom dom, CollectionSettings collectionSettings, Action<XmlElement> updateImgNodeCallback)
		{
			_dom = dom;
			_updateImgNode = updateImgNodeCallback;
			CollectionSettings = collectionSettings;
			GetOrCreateDataDiv();
			_dataset = GatherDataItemsFromCollectionSettings(CollectionSettings);
			GatherDataItemsFromXElement(_dataset,_dom.RawDom);
			MigrateData(_dataset);
		}

		/// <summary>
		/// For bilingual or trilingual books, this is the second language to show, after the vernacular.
		/// </summary>
		public XmlString MultilingualContentLanguage2
		{
			get
			{
				if (!_gotMclCache)
					CacheMclData();
				return _cachedMcl2;
			}
		}

		/// <summary>
		/// Cache values for MultilingualContentLanguage2 and MultilingualContentLanguage3
		/// It is safe to cache this because we reload everything when changing languages.
		/// It is important because these properties are heavily used, especially when saving pages.
		/// </summary>
		private void CacheMclData()
		{
			GatherDataItemsFromXElement(_dataset, _dom.RawDom);
			_cachedMcl2 = GetVariableOrNull("contentLanguage2", "*");
			_cachedMcl3 = GetVariableOrNull("contentLanguage3", "*");
			_gotMclCache = true;
		}

		/// <summary>
		/// For trilingual books, this is the third language to show
		/// </summary>
		public XmlString MultilingualContentLanguage3
		{
			get
			{
				if (!_gotMclCache)
					CacheMclData();
				return _cachedMcl3;
			}
		}

		/// <summary>
		/// A book-level style number sequence
		/// </summary>
		public int StyleNumberSequence
		{
			get
			{
				lock(thisLock)
				{
					GatherDataItemsFromXElement(_dataset, _dom.RawDom);
					string curSeqStr = GetVariableOrNull("styleNumberSequence", "*").Unencoded;
					int curSeq;
					int nextSeq = 1;
					if (Int32.TryParse(curSeqStr, out curSeq))
						nextSeq = curSeq + 1;
					Set("styleNumberSequence", XmlString.FromUnencoded(nextSeq.ToString(CultureInfo.InvariantCulture)),
						false);
					return nextSeq;
				}
			}
		}


		public void UpdateVariablesAndDataDivThroughDOM(BookInfo info = null)
		{
			// For some reason, cleaning up the anchor elements in the div#bloomDataDiv doesn't work
			// when called from HtmlDom.ProcessPageAfterEditing.  Doing it here works.
			var div = _dom.Body.SelectSingleNode("div[@id='bloomDataDiv']") as XmlElement;
			if (div != null)
				HtmlDom.CleanupAnchorElements(div);
			UpdateVariablesAndDataDiv(_dom.RawDom.FirstChild, info);
		}


		/// <summary>
		/// Create or update the data div with all the data-book values in the document
		/// </summary>
		/// <param name="dom">This is either the whole document, or a page div that we just edited and want to read from.</param>
		public void SuckInDataFromEditedDom(HtmlDom dom, BookInfo info = null)
		{
			UpdateVariablesAndDataDiv(dom.RawDom, info);
		}

		public void SynchronizeDataItemsThroughoutDOM()
		{
			var itemsToDelete = new HashSet<Tuple<string, string>>();
			SynchronizeDataItemsFromContentsOfElement(_dom.Body, itemsToDelete);
		}

		/// <summary>
		/// Create or update the data div with all the data-book values in the document
		/// </summary>
		/// <param name="elementToReadFrom">This is either the whole document, or a page div that we just edited and want to read from.</param>
		private void UpdateVariablesAndDataDiv(XmlNode elementToReadFrom, BookInfo info = null)
		{
			//Debug.WriteLine("before update: " + _dataDiv.OuterXml);

			var itemsToDelete = new HashSet<Tuple<string, string>>();
			DataSet incomingData = SynchronizeDataItemsFromContentsOfElement(elementToReadFrom, itemsToDelete);
			UpdateToolRelatedDataFromBookInfo(info, incomingData, itemsToDelete);
			incomingData.UpdateGenericLanguageString("contentLanguage1", XmlString.FromUnencoded(Language1.Iso639Code), false);
			incomingData.UpdateGenericLanguageString("contentLanguage2",
											 XmlString.IsNullOrEmpty(MultilingualContentLanguage2)
												 ? null
												 : MultilingualContentLanguage2, false);
			incomingData.UpdateGenericLanguageString("contentLanguage3",
											 XmlString.IsNullOrEmpty(MultilingualContentLanguage3)
												 ? null
												 : MultilingualContentLanguage3, false);

			//Debug.WriteLine("xyz: " + _dataDiv.OuterXml);
			foreach (var v in incomingData.TextVariables)
			{
				if (!v.Value.IsCollectionValue)
					UpdateSingleTextVariableInDataDiv(v.Key,v.Value);
			}
			foreach (var tuple in itemsToDelete)
				UpdateSingleTextVariableInDataDiv(tuple.Item1, tuple.Item2, XmlString.Empty);
			foreach (var attributeSet in incomingData.XmatterPageDataAttributeSets)
				PushXmatterPageAttributesIntoDataDiv(attributeSet);
			//Debug.WriteLine("after update: " + _dataDiv.OuterXml);

			UpdateTitle(info);//this may change our "bookTitle" variable if the title is based on a template that reads other variables (e.g. "Primer Term2-Week3")
			UpdateIsbn(info);
			if (info != null)
				UpdateBookInfoTags(info);
			UpdateCredits(info);
		}

		private void UpdateToolRelatedDataFromBookInfo(BookInfo info, DataSet incomingData, HashSet<Tuple<string, string>> itemsToDelete)
		{
			if (info == null)
				return; // only in tests
			var tools = info.Tools;
			var bookClass = _dom.Body.Attributes["class"]?.Value ?? "";

			if (!bookClass.Contains("leveled-reader") && !bookClass.Contains("decodable-reader"))
			{
				incomingData.UpdateGenericLanguageString("levelOrStageNumber", XmlString.Empty, false);
				itemsToDelete.Add(new Tuple<string, string>("levelOrStageNumber", "*"));
				return;
			}

			var levelTool = tools.FirstOrDefault(t => t.ToolId == "leveledReader");
			if (levelTool != null && bookClass.Contains("leveled-reader"))
			{
				var level = levelTool.State;
				incomingData.UpdateGenericLanguageString("levelOrStageNumber", XmlString.FromUnencoded(level), false);
				itemsToDelete.RemoveWhere(item => item.Item1 == "levelOrStageNumber");
			}

			var decodableTool = tools.FirstOrDefault(t => t.ToolId == "decodableReader");
			if (decodableTool != null && bookClass.Contains("decodable-reader"))
			{
				var stageString = decodableTool.State?.Split(';').FirstOrDefault()?.Split(':').Skip(1).FirstOrDefault();
				if (!string.IsNullOrEmpty(stageString))
				{
					incomingData.UpdateGenericLanguageString("levelOrStageNumber", XmlString.FromUnencoded(stageString), false);
					itemsToDelete.RemoveWhere(item => item.Item1 == "levelOrStageNumber");
				}

				int stage;
				if (int.TryParse(stageString, out stage) && CollectionSettings != null)
				{
					var settingsPath = DecodableReaderToolSettings.GetReaderToolsSettingsFilePath(CollectionSettings);
					if (File.Exists(settingsPath))
					{
						try
						{
							var settingsJson = File.ReadAllText(settingsPath, Encoding.UTF8);
							var settings = DynamicJson.Parse(settingsJson);
							var stages = settings.stages;
							var allLetters = "";
							for (var i=0; i<stage;i++)
{'fraction_non_alphanumeric': 0.10791190864600327, 'fraction_numerical': 0.009951060358890701, 'mean_word_length': 3.984956291929254, 'pattern_counts': {'":': 0, '<': 145, '<?xml version=': 0, '>': 151, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3841317', 'n_tokens_mistral': 8361, 'n_tokens_neox': 7101, 'n_words': 2154}