<commit_before>﻿using System;
using System.IO;
using System.Text;
using System.Collections.Generic;

namespace LightJson
{
	using ErrorCode = JsonSerializationException.ErrorCode;

	public sealed class JsonWriter
	{
		private int indent;
		private bool isNewLine;
		private TextWriter writer;
		private HashSet<JsonValue> renderedValues;

		public string IndentString { get; set; }
		public string SpacingString { get; set; }
		public string NewLineString { get; set; }

		public JsonWriter(bool pretty = false)
		{
			if (pretty)
			{
				this.IndentString = "\t";
				this.SpacingString = " ";
				this.NewLineString = "\n";
			}
		}

		private void Initialize()
		{
			this.indent = 0;
			this.isNewLine = true;
			this.writer = new StringWriter();
			this.renderedValues = new HashSet<JsonValue>();
		}

		private void Write(string text)
		{
			if (this.isNewLine)
			{
				this.isNewLine = false;
				WriteIndentation();
			}

			writer.Write(text);
		}

		private void WriteEncodedString(string text)
		{
			Write(EncodeStringValue(text));
		}

		private void WriteIndentation()
		{
			for (var i = 0; i < this.indent; i += 1)
			{
				Write(this.IndentString);
			}
		}

		private void WriteSpacing()
		{
			Write(this.SpacingString);
		}

		private void WriteLine()
		{
			Write(this.NewLineString);
			this.isNewLine = true;
		}

		private void WriteLine(string line)
		{
			Write(line);
			WriteLine();
		}

		private void AddRenderedValue(JsonValue value)
		{
			if (renderedValues.Contains(value))
			{
				throw new JsonSerializationException(ErrorCode.CircularReference, value);
			}
			else
			{
				renderedValues.Add(value);
			}
		}

		private void Render(JsonValue value)
		{
			switch (value.Type)
			{
				case JsonValueType.Null:
				case JsonValueType.Boolean:
					Write(value.ToString());
					break;

				case JsonValueType.Number:
					if (IsValidNumber((double)value))
					{
						Write(value.ToString());
					}
					else
					{
						throw new JsonSerializationException(ErrorCode.InvalidNumber, value);
					}
					break;

				case JsonValueType.String:
					WriteEncodedString((string)value);
					break;

				case JsonValueType.Object:
					Render((JsonObject)value);
					break;

				case JsonValueType.Array:
					Render((JsonArray)value);
					break;

				default:
					throw new JsonSerializationException(ErrorCode.InvalidValueType, value);
			}
		}

		private void Render(JsonArray value)
		{
			AddRenderedValue(value);

			WriteLine("[");

			indent += 1;

			using (var enumerator = value.GetEnumerator())
			{
				var hasNext = enumerator.MoveNext();

				while (hasNext)
				{
					Render(enumerator.Current);

					hasNext = enumerator.MoveNext();

					if (hasNext)
					{
						WriteLine(",");
					}
				}
			}

			indent -= 1;

			WriteLine();
			Write("]");
		}

		private void Render(JsonObject value)
		{
			AddRenderedValue(value);

			WriteLine("{");

			indent += 1;

			using(var enumerator = value.GetEnumerator())
			{
				var hasNext = enumerator.MoveNext();

				while (hasNext)
				{
					WriteEncodedString(enumerator.Current.Key);
					Write(":");
					WriteSpacing();
					Render(enumerator.Current.Value);

					hasNext = enumerator.MoveNext();

					if (hasNext)
					{
						WriteLine(",");
					}
				}
			}

			indent -= 1;

			WriteLine();
			Write("}");
		}

		public string Serialize(JsonObject jsonObject)
		{
			Initialize();

			Render(jsonObject);

			return writer.ToString();
		}

		private static string EncodeStringValue(string value)
		{
			var builder = new StringBuilder(value);

			builder.Replace("\\", "\\\\"); // \ -> \\
			builder.Replace("\"", "\\\""); // " -> \"
			builder.Replace("/", "\\/");   // / -> \/
			builder.Replace("\b", "\\b");
			builder.Replace("\f", "\\f");
			builder.Replace("\n", "\\n");
			builder.Replace("\r", "\\r");
			builder.Replace("\t", "\\t");

			return string.Format("\"{0}\"", builder.ToString());
		}

		private static bool IsValidNumber(double number)
		{
			return !(double.IsNaN(number) || double.IsInfinity(number));
		}
	}
}
<commit_msg>Adding serialization for JSON arrays.<commit_after>﻿using System;
using System.IO;
using System.Text;
using System.Collections.Generic;

namespace LightJson
{
	using ErrorCode = JsonSerializationException.ErrorCode;

	public sealed class JsonWriter
	{
		private int indent;
		private bool isNewLine;
		private TextWriter writer;
		private HashSet<JsonValue> renderedValues;

		public string IndentString { get; set; }
		public string SpacingString { get; set; }
		public string NewLineString { get; set; }

		public JsonWriter(bool pretty = false)
		{
			if (pretty)
			{
				this.IndentString = "\t";
				this.SpacingString = " ";
				this.NewLineString = "\n";
			}
		}

		private void Initialize()
		{
			this.indent = 0;
			this.isNewLine = true;
			this.writer = new StringWriter();
			this.renderedValues = new HashSet<JsonValue>();
		}

		private void Write(string text)
		{
			if (this.isNewLine)
			{
				this.isNewLine = false;
				WriteIndentation();
			}

			writer.Write(text);
		}

		private void WriteEncodedString(string text)
		{
			Write(EncodeStringValue(text));
		}

		private void WriteIndentation()
		{
			for (var i = 0; i < this.indent; i += 1)
			{
				Write(this.IndentString);
			}
		}

		private void WriteSpacing()
		{
			Write(this.SpacingString);
		}

		private void WriteLine()
		{
			Write(this.NewLineString);
			this.isNewLine = true;
		}

		private void WriteLine(string line)
		{
			Write(line);
			WriteLine();
		}

		private void AddRenderedValue(JsonValue value)
		{
			if (renderedValues.Contains(value))
			{
				throw new JsonSerializationException(ErrorCode.CircularReference, value);
			}
			else
			{
				renderedValues.Add(value);
			}
		}

		private void Render(JsonValue value)
		{
			switch (value.Type)
			{
				case JsonValueType.Null:
				case JsonValueType.Boolean:
					Write(value.ToString());
					break;

				case JsonValueType.Number:
					if (IsValidNumber((double)value))
					{
						Write(value.ToString());
					}
					else
					{
						throw new JsonSerializationException(ErrorCode.InvalidNumber, value);
					}
					break;

				case JsonValueType.String:
					WriteEncodedString((string)value);
					break;

				case JsonValueType.Object:
					Render((JsonObject)value);
					break;

				case JsonValueType.Array:
					Render((JsonArray)value);
					break;

				default:
					throw new JsonSerializationException(ErrorCode.InvalidValueType, value);
			}
		}

		private void Render(JsonArray value)
		{
			AddRenderedValue(value);

			WriteLine("[");

			indent += 1;

			using (var enumerator = value.GetEnumerator())
			{
				var hasNext = enumerator.MoveNext();

				while (hasNext)
				{
					Render(enumerator.Current);

					hasNext = enumerator.MoveNext();

					if (hasNext)
					{
						WriteLine(",");
					}
				}
			}

			indent -= 1;

			WriteLine();
			Write("]");
		}

		private void Render(JsonObject value)
		{
			AddRenderedValue(value);

			WriteLine("{");

			indent += 1;

			using(var enumerator = value.GetEnumerator())
			{
				var hasNext = enumerator.MoveNext();

				while (hasNext)
				{
					WriteEncodedString(enumerator.Current.Key);
					Write(":");
					WriteSpacing();
					Render(enumerator.Current.Value);

					hasNext = enumerator.MoveNext();

					if (hasNext)
					{
						WriteLine(",");
					}
				}
			}

			indent -= 1;

			WriteLine();
			Write("}");
		}

		public string Serialize(JsonObject jsonObject)
		{
			Initialize();

			Render(jsonObject);

			return writer.ToString();
		}

		public string Serialize(JsonArray jsonArray)
		{
			Initialize();

			Render(jsonArray);

			return writer.ToString();
		}

		private static string EncodeStringValue(string value)
		{
			var builder = new StringBuilder(value);

			builder.Replace("\\", "\\\\"); // \ -> \\
			builder.Replace("\"", "\\\""); // " -> \"
			builder.Replace("/", "\\/");   // / -> \/
			builder.Replace("\b", "\\b");
			builder.Replace("\f", "\\f");
			builder.Replace("\n", "\\n");
			builder.Replace("\r", "\\r");
			builder.Replace("\t", "\\t");

			return string.Format("\"{0}\"", builder.ToString());
		}

		private static bool IsValidNumber(double number)
		{
			return !(double.IsNaN(number) || double.IsInfinity(number));
		}
	}
}

{'fraction_non_alphanumeric': 0.14139096836512918, 'fraction_numerical': 0.001931900507123883, 'mean_word_length': 2.7804655408489274, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6320796', 'n_tokens_mistral': 3487, 'n_tokens_neox': 2628, 'n_words': 532}