<commit_before>﻿/// <summary>
/// Serpent, a Python literal expression serializer/deserializer
/// (a.k.a. Python's ast.literal_eval in .NET)
///
/// Copyright <NAME> (<EMAIL>)
/// Software license: "MIT software license". See http://opensource.org/licenses/MIT
/// </summary>

using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Reflection;
using System.Runtime.Serialization;
using System.Text;
using System.Xml;

namespace Razorvine.Serpent
{
	/// <summary>
	/// Serialize an object tree to a byte stream.
	/// It is not thread-safe: make sure you're not making changes to the object tree that is being serialized.
	/// </summary>
	public class Serializer
	{
		/// <summary>
		/// indent output?
		/// </summary>
		public bool Indent;
		
		/// <summary>
		/// use set literals?
		/// </summary>
		public bool SetLiterals;
		
		/// <summary>
		/// include namespace prefix for classes that are serialized to dict?
		/// </summary>
		public bool NamespaceInClassName;
		
		/// <summary>
		/// The maximum nesting level of the object graphs that you want to serialize.
		/// This limit has been set to avoid troublesome stack overflow errors.
		/// (If it is reached, an IllegalArgumentException is thrown instead with a clear message) 
		/// </summary>
		public int MaximumLevel = 800;     // avoids stackoverflow errors
		
		private static IDictionary<Type, Func<object, IDictionary>> classToDictRegistry = new Dictionary<Type, Func<object, IDictionary>>();
		

		/// <summary>
		/// Initialize the serializer.
		/// </summary>
		/// <param name="indent">indent the output over multiple lines (default=false)</param>
		/// <param name="setLiterals">use set-literals or not (set to False if you need compatibility with Python < 3.2)</param>
		/// <param name="namespaceInClassName">include namespace prefix for class names or only use the class name itself</param>
		public Serializer(bool indent=false, bool setLiterals=true, bool namespaceInClassName=false)
		{
			this.Indent = indent;
			this.SetLiterals = setLiterals;
			this.NamespaceInClassName = namespaceInClassName;
		}
		
		/// <summary>
		/// Register a custom class-to-dict converter.
		/// </summary>
		public static void RegisterClass(Type clazz, Func<object, IDictionary> converter)
		{
			classToDictRegistry[clazz] = converter;
		}

		/// <summary>
		/// Serialize the object tree to bytes.
		/// </summary>
		public byte[] Serialize(object obj)
		{
			using(MemoryStream ms = new MemoryStream())	// XXX StringWriter?
			using(TextWriter tw = new StreamWriter(ms, new UTF8Encoding(false)))			// don't write BOM
			{
				if(this.SetLiterals)
					tw.Write("# serpent utf-8 python3.2\n");  //set-literals require python 3.2+ to deserialize (ast.literal_eval limitation)
				else
					tw.Write("# serpent utf-8 python2.6\n");
				Serialize(obj, tw, 0);
				tw.Flush();
				return ms.ToArray();
			}
		}
		
		protected void Serialize(object obj, TextWriter tw, int level)
		{<commit_msg>dotnet: optimized serializer io a bit<commit_after>﻿/// <summary>
/// Serpent, a Python literal expression serializer/deserializer
/// (a.k.a. Python's ast.literal_eval in .NET)
///
/// Copyright <NAME> (<EMAIL>)
/// Software license: "MIT software license". See http://opensource.org/licenses/MIT
/// </summary>

using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Reflection;
using System.Runtime.Serialization;
using System.Text;
using System.Xml;

namespace Razorvine.Serpent
{
	/// <summary>
	/// Serialize an object tree to a byte stream.
	/// It is not thread-safe: make sure you're not making changes to the object tree that is being serialized.
	/// </summary>
	public class Serializer
	{
		/// <summary>
		/// indent output?
		/// </summary>
		public bool Indent;
		
		/// <summary>
		/// use set literals?
		/// </summary>
		public bool SetLiterals;
		
		/// <summary>
		/// include namespace prefix for classes that are serialized to dict?
		/// </summary>
		public bool NamespaceInClassName;
		
		/// <summary>
		/// The maximum nesting level of the object graphs that you want to serialize.
		/// This limit has been set to avoid troublesome stack overflow errors.
		/// (If it is reached, an IllegalArgumentException is thrown instead with a clear message) 
		/// </summary>
		public int MaximumLevel = 800;     // avoids stackoverflow errors
		
		private static IDictionary<Type, Func<object, IDictionary>> classToDictRegistry = new Dictionary<Type, Func<object, IDictionary>>();
		

		/// <summary>
		/// Initialize the serializer.
		/// </summary>
		/// <param name="indent">indent the output over multiple lines (default=false)</param>
		/// <param name="setLiterals">use set-literals or not (set to False if you need compatibility with Python < 3.2)</param>
		/// <param name="namespaceInClassName">include namespace prefix for class names or only use the class name itself</param>
		public Serializer(bool indent=false, bool setLiterals=true, bool namespaceInClassName=false)
		{
			this.Indent = indent;
			this.SetLiterals = setLiterals;
			this.NamespaceInClassName = namespaceInClassName;
		}
		
		/// <summary>
		/// Register a custom class-to-dict converter.
		/// </summary>
		public static void RegisterClass(Type clazz, Func<object, IDictionary> converter)
		{
			classToDictRegistry[clazz] = converter;
		}

		/// <summary>
		/// Serialize the object tree to bytes.
		/// </summary>
		public byte[] Serialize(object obj)
		{
			using(StringWriter tw = new StringWriter())
			{
				if(this.SetLiterals)
					tw.Write("# serpent utf-8 python3.2\n");  //set-literals require python 3.2+ to deserialize (ast.literal_eval limitation)
				else
					tw.Write("# serpent utf-8 python2.6\n");
				Serialize(obj, tw, 0);
				tw.Flush();
				return Encoding.UTF8.GetBytes(tw.ToString());
			}
		}
		
		protected void Serialize(object obj, TextWriter tw, int level)
		{
{'fraction_non_alphanumeric': 0.1274787535410765, 'fraction_numerical': 0.004999166805532411, 'mean_word_length': 3.271886120996441, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5741280', 'n_tokens_mistral': 1966, 'n_tokens_neox': 1709, 'n_words': 578}