<commit_before>/* part of Pyrolite, by <NAME> (<EMAIL>) */

using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Reflection;
using System.Text;

namespace Razorvine.Pickle
{
	
/// <summary>
/// Pickle an object graph into a Python-compatible pickle stream. For
/// simplicity, the only supported pickle protocol at this time is protocol 2. 
/// See README.txt for a table with the type mapping.
/// This class is NOT threadsafe! (Don't use the same pickler from different threads)
/// </summary>
public class Pickler : IDisposable {

	public static int HIGHEST_PROTOCOL = 2;

	private const int MAX_RECURSE_DEPTH = 200;
	private Stream outs;
	private int recurse = 0;	// recursion level
	private int PROTOCOL = 2;
	private static IDictionary<Type, IObjectPickler> customPicklers = new Dictionary<Type, IObjectPickler>();
	private bool useMemo=true;
	private IDictionary<object, int> memo;		// maps objects to memo index
	
	/**
	 * Create a Pickler.
	 */
	public Pickler() : this(true) {
	}

	/**
	 * Create a Pickler. Specify if it is to use a memo table or not.
	 * The memo table is NOT reused across different calls.
	 * If you use a memo table, you can only pickle objects that are hashable.
	 */
	public Pickler(bool useMemo) {
		this.useMemo=useMemo;
	}
	
	/**
	 * Close the pickler stream, discard any internal buffers.
	 */
	public void close() {
		memo = null;
		outs.Flush();
		outs.Close();
	}

	/**
	 * Register additional object picklers for custom classes.
	 */
	public static void registerCustomPickler(Type clazz, IObjectPickler pickler) {
		customPicklers[clazz]=pickler;
	}
	
	/**
	 * Pickle a given object graph, returning the result as a byte array.
	 */
	public byte[] dumps(object o) {
		MemoryStream bo = new MemoryStream();
		dump(o, bo);
		bo.Flush();
		return bo.ToArray();
	}

	/**
	 * Pickle a given object graph, writing the result to the output stream.
	 */
	public void dump(object o, Stream stream) {
		outs = stream;
		recurse = 0;
		if(useMemo)
			memo = new Dictionary<object, int>();
		outs.WriteByte(Opcodes.PROTO);
		outs.WriteByte((byte)PROTOCOL);
		save(o);
		memo = null;  // get rid of the memo table
		outs.WriteByte(Opcodes.STOP);
		outs.Flush();
		if(recurse!=0)  // sanity check
			throw new PickleException("recursive structure error, please report this problem");
	}

	/**
	 * Pickle a single object and write its pickle representation to the output stream.
	 * Normally this is used internally by the pickler, but you can also utilize it from
	 * within custom picklers. This is handy if as part of the custom pickler, you need
	 * to write a couple of normal objects such as strings or ints, that are already
	 * supported by the pickler.
	 * This method can be called recursively to output sub-objects.
	 */
	public void save(object o) {
		recurse++;
		if(recurse>MAX_RECURSE_DEPTH)
			throw new StackOverflowException("recursion too deep in Pickler.save (>"+MAX_RECURSE_DEPTH+")");

		// null type?
		if(o==null) {
			outs.WriteByte(Opcodes.NONE);
			recurse--;
			return;
		}


		Type t=o.GetType();
		
		// check the memo table, otherwise simply dispatch
		if(LookupMemo(t, o) || dispatch(t, o)){
			recurse--;
			return;
		}

		throw new PickleException("couldn't pickle object of type "+t);
	}
	
	/**
	 * Write the object to the memo table and output a memo write opcode
	 * Only works for hashable objects
	 */
	private void WriteMemo(object obj) {
		if(!this.useMemo)
			return;
		if(!memo.ContainsKey(obj))
		{
			int memo_index = memo.Count;
			memo[obj] = memo_index;
			if(memo_index<=0xFF)
			{
				outs.WriteByte(Opcodes.BINPUT);
				outs.WriteByte((byte)memo_index);
			}
			else
			{
				outs.WriteByte(Opcodes.LONG_BINPUT);
				byte[] index_bytes = PickleUtils.integer_to_bytes(memo_index);<commit_msg>pickler better subclassable<commit_after>/* part of Pyrolite, by <NAME> (<EMAIL>) */

using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Reflection;
using System.Text;

namespace Razorvine.Pickle
{
	
/// <summary>
/// Pickle an object graph into a Python-compatible pickle stream. For
/// simplicity, the only supported pickle protocol at this time is protocol 2. 
/// See README.txt for a table with the type mapping.
/// This class is NOT threadsafe! (Don't use the same pickler from different threads)
/// </summary>
public class Pickler : IDisposable {

	public static int HIGHEST_PROTOCOL = 2;

	protected const int MAX_RECURSE_DEPTH = 200;
	protected Stream outs;
	protected int recurse = 0;	// recursion level
	protected int PROTOCOL = 2;
	protected static IDictionary<Type, IObjectPickler> customPicklers = new Dictionary<Type, IObjectPickler>();
	protected bool useMemo=true;
	protected IDictionary<object, int> memo;		// maps objects to memo index
	
	/**
	 * Create a Pickler.
	 */
	public Pickler() : this(true) {
	}

	/**
	 * Create a Pickler. Specify if it is to use a memo table or not.
	 * The memo table is NOT reused across different calls.
	 * If you use a memo table, you can only pickle objects that are hashable.
	 */
	public Pickler(bool useMemo) {
		this.useMemo=useMemo;
	}
	
	/**
	 * Close the pickler stream, discard any internal buffers.
	 */
	public void close() {
		memo = null;
		outs.Flush();
		outs.Close();
	}

	/**
	 * Register additional object picklers for custom classes.
	 */
	public static void registerCustomPickler(Type clazz, IObjectPickler pickler) {
		customPicklers[clazz]=pickler;
	}
	
	/**
	 * Pickle a given object graph, returning the result as a byte array.
	 */
	public byte[] dumps(object o) {
		MemoryStream bo = new MemoryStream();
		dump(o, bo);
		bo.Flush();
		return bo.ToArray();
	}

	/**
	 * Pickle a given object graph, writing the result to the output stream.
	 */
	public void dump(object o, Stream stream) {
		outs = stream;
		recurse = 0;
		if(useMemo)
			memo = new Dictionary<object, int>();
		outs.WriteByte(Opcodes.PROTO);
		outs.WriteByte((byte)PROTOCOL);
		save(o);
		memo = null;  // get rid of the memo table
		outs.WriteByte(Opcodes.STOP);
		outs.Flush();
		if(recurse!=0)  // sanity check
			throw new PickleException("recursive structure error, please report this problem");
	}

	/**
	 * Pickle a single object and write its pickle representation to the output stream.
	 * Normally this is used internally by the pickler, but you can also utilize it from
	 * within custom picklers. This is handy if as part of the custom pickler, you need
	 * to write a couple of normal objects such as strings or ints, that are already
	 * supported by the pickler.
	 * This method can be called recursively to output sub-objects.
	 */
	public void save(object o) {
		recurse++;
		if(recurse>MAX_RECURSE_DEPTH)
			throw new StackOverflowException("recursion too deep in Pickler.save (>"+MAX_RECURSE_DEPTH+")");

		// null type?
		if(o==null) {
			outs.WriteByte(Opcodes.NONE);
			recurse--;
			return;
		}


		Type t=o.GetType();
		
		// check the memo table, otherwise simply dispatch
		if(LookupMemo(t, o) || dispatch(t, o)){
			recurse--;
			return;
		}

		throw new PickleException("couldn't pickle object of type "+t);
	}
	
	/**
	 * Write the object to the memo table and output a memo write opcode
	 * Only works for hashable objects
	 */
	protected void WriteMemo(object obj) {
		if(!this.useMemo)
			return;
		if(!memo.ContainsKey(obj))
		{
			int memo_index = memo.Count;
			memo[obj] = memo_index;
			if(memo_index<=0xFF)
			{
				outs.WriteByte(Opcodes.BINPUT);
				outs.WriteByte((byte)memo_index);
			}
			else
			{
				outs.WriteByte(Opcodes.LONG_BINPUT);
				byte[] index_bytes = PickleUtils.integer_to_bytes(memo_index);
{'fraction_non_alphanumeric': 0.10639399430494434, 'fraction_numerical': 0.0025886616619207868, 'mean_word_length': 3.5533294048320565, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '951774', 'n_tokens_mistral': 2732, 'n_tokens_neox': 2434, 'n_words': 845}