<commit_before>package com.judax.webgl2opengl;

import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.ShortBuffer;
import java.util.HashSet;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.opengl.Matrix;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import android.util.Base64;
import android.util.SparseIntArray;

public class WebGLMessage implements Runnable
{
	private static final boolean VERBOSE = false;
		
	private static final int BYTES_PER_FLOAT = 4;
	private static final int BYTES_PER_INT = 4;
	private static final int BYTES_PER_SHORT = 2;
	
  private static SparseIntArray jsIdsToNativeIds = new SparseIntArray();
  
  // Configuration data
  private static HashSet<String> projectionMatrixUniformNames = new HashSet<String>();
  private static HashSet<Integer> projectionMatrixUniformJSIds = new HashSet<Integer>();
  private static HashSet<String> modelViewMatrixUniformNames = new HashSet<String>();
  private static HashSet<Integer> modelViewMatrixUniformsJSIds = new HashSet<Integer>();
 
  // Some default names of matrix uniforms
  static 
  {
  	// PROJECTION
  	
  	// WebGLLessons
  	projectionMatrixUniformNames.add("uProjectionMatrix");
  	projectionMatrixUniformNames.add("uPMatrix");

  	// ThreeJS
  	projectionMatrixUniformNames.add("projectionMatrix");
  	
  	// PlayCanvas
  	projectionMatrixUniformNames.add("matrix_projection"); 
  	projectionMatrixUniformNames.add("matrix_viewProjection");
  	
  	// Sketchfab
  	projectionMatrixUniformNames.add("ProjectionMatrix");
  	
  	// Goo
  	projectionMatrixUniformNames.add("projectionMatrix"); 
  	projectionMatrixUniformNames.add("viewProjectionMatrix"); 


  	// MODELVIEW
  	
  	// WebGLLessons
  	modelViewMatrixUniformNames.add("uMVMatrix");
  	
  	// ThreeJS
  	modelViewMatrixUniformNames.add("modelViewMatrix"); 
  	
  	// PlayCanvas
  	modelViewMatrixUniformNames.add("matrix_view"); 
  	modelViewMatrixUniformNames.add("matrix_model");
  	
  	// Sketchfab
  	modelViewMatrixUniformNames.add("ModelViewMatrix"); 
  	
  	// Goo
  	modelViewMatrixUniformNames.add("viewMatrix"); 
  	modelViewMatrixUniformNames.add("worldMatrix"); 
  }
  
	private static float[] projectionMatrix = new float[16];
	private static float[] modelViewMatrix = new float[16];
	
  // These variables represent OpenGL states modifiable by a WebGL command
	private static boolean unpackPremultiplyAlpha = false;
	private static boolean unpackFlipY = false;
	
	private static float[] fromObjectToFloatArray(Object object) throws JSONException
	{
		float[] values = null;
		if (object instanceof JSONObject)
		{
			JSONObject jsonObject = (JSONObject)object;
			values = new float[jsonObject.length()];
		}
		else if (object instanceof JSONArray)
		{
			JSONArray jsonArray = (JSONArray)object;
			values = new float[jsonArray.length()];
		}
		else 
		{
			throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
		}
		fromObjectToFloatArray(object, values);
		return values;
	}

	private static void fromObjectToFloatArray(Object object, float[] values) throws JSONException
	{
		for (int i = 0; i < values.length; i++) 
		{
			Object o = null;
			if (object instanceof JSONObject)
			{
				JSONObject jsonObject = (JSONObject)object;
				o = jsonObject.get("" + i);
			}
			else if (object instanceof JSONArray)
			{
				JSONArray jsonArray = (JSONArray)object;
				o = jsonArray.get(i);
			}
			else 
			{
				throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
			}
			float value = 0;
			if (o instanceof Number)
			{
				value = ((Number)o).floatValue();
			}
			values[i] = value;
		}
		// =========================================
//		String s = "[";
//		for (int i = 0; i < values.length; i++) 
//		{
//			s += values[i] + (i < values.length - 1 ? ", " : "");
//		}
//		s += "]";
//		System.out.println("JUDAX: " + s);
		// =========================================
	}
	
	private static short[] fromObjectToShortArray(Object object) throws JSONException
	{
		short[] values = null;
		if (object instanceof JSONObject)
		{
			JSONObject jsonObject = (JSONObject)object;
			values = new short[jsonObject.length()];
		}
		else if (object instanceof JSONArray)
		{
			JSONArray jsonArray = (JSONArray)object;
			values = new short[jsonArray.length()];
		}
		else 
		{
			throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
		}
		fromObjectToShortArray(object, values);
		return values;
	}
	
	private static void fromObjectToShortArray(Object object, short[] values) throws JSONException
	{
		for (int i = 0; i < values.length; i++) 
		{
			Object o = null;
			if (object instanceof JSONObject)
			{
				JSONObject jsonObject = (JSONObject)object;
				o = jsonObject.get("" + i);
			}
			else if (object instanceof JSONArray)
			{
				JSONArray jsonArray = (JSONArray)object;
				o = jsonArray.get(i);
			}
			else 
			{
				throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
			}
			short value = 0;
			if (o instanceof Number)
			{
				value = ((Number)o).shortValue();
			}
			values[i] = value;
		}
	}
	
	private static int[] fromObjectToIntArray(Object object) throws JSONException
	{
		int[] values = null;
		if (object instanceof JSONObject)
		{
			JSONObject jsonObject = (JSONObject)object;
			values = new int[jsonObject.length()];
		}
		else if (object instanceof JSONArray)
		{
			JSONArray jsonArray = (JSONArray)object;
			values = new int[jsonArray.length()];
		}
		else 
		{
			throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
		}
		fromObjectToIntArray(object, values);
		return values;
	}
	
	private static void fromObjectToIntArray(Object object, int[] values) throws JSONException
	{
		for (int i = 0; i < values.length; i++) 
		{
			Object o = null;
			if (object instanceof JSONObject)
			{
				JSONObject jsonObject = (JSONObject)object;
				o = jsonObject.get("" + i);
			}
			else if (object instanceof JSONArray)
			{
				JSONArray jsonArray = (JSONArray)object;
				o = jsonArray.get(i);
			}
			else 
			{
				throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
			}
			int value = 0;
			if (o instanceof Number)
			{
				value = ((Number)o).intValue();
			}
			values[i] = value;
		}
	}
	
	private static byte[] fromObjectToByteArray(Object object) throws JSONException
	{
		byte[] values = null;
		if (object instanceof JSONObject)
		{
			JSONObject jsonObject = (JSONObject)object;
			values = new byte[jsonObject.length()];
		}
		else if (object instanceof JSONArray)
		{
			JSONArray jsonArray = (JSONArray)object;
			values = new byte[jsonArray.length()];
		}
		else 
		{
			throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
		}
		fromObjectToByteArray(object, values);
		return values;
	}

	private static void fromObjectToByteArray(Object object, byte[] values) throws JSONException
	{
		for (int i = 0; i < values.length; i++) 
		{
			Object o = null;
			if (object instanceof JSONObject)
			{
				JSONObject jsonObject = (JSONObject)object;
				o = jsonObject.get("" + i);
			}
			else if (object instanceof JSONArray)
			{
				JSONArray jsonArray = (JSONArray)object;
				o = jsonArray.get(i);
			}
			else 
			{
				throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
			}
			byte value = 0;
			if (o instanceof Number)
			{
				value = ((Number)o).byteValue();
			}
			values[i] = value;
		}
		// =========================================
//		String s = "[";
//		for (int i = 0; i < values.length; i++) 
//		{
//			s += values[i] + (i < values.length - 1 ? ", " : "");
//		}
//		s += "]";
//		System.out.println("JUDAX: " + s);
		// =========================================
	}
		
	private static String fromWebGLNameToOpenGLName(String webGLFunctionName)
	{
		return "gl" + Character.toUpperCase(webGLFunctionName.charAt(0)) + webGLFunctionName.substring(1);	
	}
	
	// The original WebGL message and the JSON conversion of the message object
	private String message = null;
	private JSONObject messageJSON = null;
	private String webGLFunctionName = null;
	private JSONArray webGLFunctionArgs = null;
	private JSONArray supportedExtensions = null;

	private void getSupportedExtensions()
	{
		supportedExtensions = new JSONArray();
		String supportedExtensionsString = GLES20.glGetString(GLES20.GL_EXTENSIONS);
		String[] supportedExtensionsStringArray = supportedExtensionsString.split("[\t ]");
		for (int i = 0; i < supportedExtensionsStringArray.length; i++)
		{
			supportedExtensions.put(supportedExtensionsStringArray[i]);
//			System.out.println("JUDAX: Extension " + supportedExtensionsStringArray[i] + " supported");
		}
	}
	
	public WebGLMessage(String message) throws Exception
	{
		this.message = message;
		this.messageJSON = new JSONObject(message);
		this.webGLFunctionName = messageJSON.getString("name");
		this.webGLFunctionArgs = messageJSON.getJSONArray("args");
	}
	
	public boolean isSynchronous()
	{
//		return false;
		return 
				webGLFunctionName.equals("getParameter") ||
				webGLFunctionName.equals("getActiveAttrib") ||
				webGLFunctionName.equals("getActiveUniform") ||
				webGLFunctionName.equals("getAttribLocation") || // This is a special case. The WebGL spec forces it to return a GLint so let's comply with the spec in case an engine is expecting that value
				webGLFunctionName.equals("getProgramParameter") ||
				webGLFunctionName.equals("getBufferParameter") ||
				webGLFunctionName.equals("getShaderPrecisionFormat") ||
				webGLFunctionName.equals("getShaderInfoLog") ||
				webGLFunctionName.equals("getShaderParameter");
	}
	
	public String fromWebGL2OpenGL()
	{
		String resultString = "";
		try 
		{
			// =========================================
			if (VERBOSE)
			{
				System.out.println("JUDAX: " + message);
			}
			// =========================================
			
			// These functions are not webgl messages directly but messages to be able to configure some aspects of the whole WebGL2OpenGL conversion
			if (webGLFunctionName.equals("configure"))
			{
				JSONObject webGL2OpenGLConfig = webGLFunctionArgs.getJSONObject(0);
				JSONArray projectionMatrixUniformNamesJSONArray = webGL2OpenGLConfig.getJSONArray("projectionMatrixUniformNames");
				for (int i = 0; i < projectionMatrixUniformNamesJSONArray.length(); i++)
				{
					projectionMatrixUniformNames.add(projectionMatrixUniformNamesJSONArray.getString(i));
				}
				JSONArray modelViewMatrixUniformNamesJSONArray = webGL2OpenGLConfig.getJSONArray("modelViewMatrixUniformNames");
				for (int i = 0; i < modelViewMatrixUniformNamesJSONArray.length(); i++)
				{
					modelViewMatrixUniformNames.add(modelViewMatrixUniformNamesJSONArray.getString(i));
				}
				return resultString;<commit_msg>Added a mechanism for some messages to be only processed once: createBuffer, createTexture, ... why execute them for both eyes?<commit_after>package com.judax.webgl2opengl;

import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.ShortBuffer;
import java.util.HashSet;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.opengl.Matrix;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import android.util.Base64;
import android.util.SparseIntArray;

public class WebGLMessage implements Runnable
{
	private static final boolean VERBOSE = false;
		
	private static final int BYTES_PER_FLOAT = 4;
	private static final int BYTES_PER_INT = 4;
	private static final int BYTES_PER_SHORT = 2;
	
  private static SparseIntArray jsIdsToNativeIds = new SparseIntArray();
  
  // Configuration data
  private static HashSet<String> projectionMatrixUniformNames = new HashSet<String>();
  private static HashSet<Integer> projectionMatrixUniformJSIds = new HashSet<Integer>();
  private static HashSet<String> modelViewMatrixUniformNames = new HashSet<String>();
  private static HashSet<Integer> modelViewMatrixUniformsJSIds = new HashSet<Integer>();
 
  // Some default names of matrix uniforms
  static 
  {
  	// PROJECTION
  	
  	// WebGLLessons
  	projectionMatrixUniformNames.add("uProjectionMatrix");
  	projectionMatrixUniformNames.add("uPMatrix");

  	// ThreeJS
  	projectionMatrixUniformNames.add("projectionMatrix");
  	
  	// PlayCanvas
  	projectionMatrixUniformNames.add("matrix_projection"); 
  	projectionMatrixUniformNames.add("matrix_viewProjection");
  	
  	// Sketchfab
  	projectionMatrixUniformNames.add("ProjectionMatrix");
  	
  	// Goo
  	projectionMatrixUniformNames.add("projectionMatrix"); 
  	projectionMatrixUniformNames.add("viewProjectionMatrix"); 


  	// MODELVIEW
  	
  	// WebGLLessons
  	modelViewMatrixUniformNames.add("uMVMatrix");
  	
  	// ThreeJS
  	modelViewMatrixUniformNames.add("modelViewMatrix"); 
  	
  	// PlayCanvas
  	modelViewMatrixUniformNames.add("matrix_view"); 
  	modelViewMatrixUniformNames.add("matrix_model");
  	
  	// Sketchfab
  	modelViewMatrixUniformNames.add("ModelViewMatrix"); 
  	
  	// Goo
  	modelViewMatrixUniformNames.add("viewMatrix"); 
  	modelViewMatrixUniformNames.add("worldMatrix"); 
  }
  
	private static float[] projectionMatrix = new float[16];
	private static float[] modelViewMatrix = new float[16];
	
  // These variables represent OpenGL states modifiable by a WebGL command
	private static boolean unpackPremultiplyAlpha = false;
	private static boolean unpackFlipY = false;
	
	private static float[] fromObjectToFloatArray(Object object) throws JSONException
	{
		float[] values = null;
		if (object instanceof JSONObject)
		{
			JSONObject jsonObject = (JSONObject)object;
			values = new float[jsonObject.length()];
		}
		else if (object instanceof JSONArray)
		{
			JSONArray jsonArray = (JSONArray)object;
			values = new float[jsonArray.length()];
		}
		else 
		{
			throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
		}
		fromObjectToFloatArray(object, values);
		return values;
	}

	private static void fromObjectToFloatArray(Object object, float[] values) throws JSONException
	{
		for (int i = 0; i < values.length; i++) 
		{
			Object o = null;
			if (object instanceof JSONObject)
			{
				JSONObject jsonObject = (JSONObject)object;
				o = jsonObject.get("" + i);
			}
			else if (object instanceof JSONArray)
			{
				JSONArray jsonArray = (JSONArray)object;
				o = jsonArray.get(i);
			}
			else 
			{
				throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
			}
			float value = 0;
			if (o instanceof Number)
			{
				value = ((Number)o).floatValue();
			}
			values[i] = value;
		}
		// =========================================
//		String s = "[";
//		for (int i = 0; i < values.length; i++) 
//		{
//			s += values[i] + (i < values.length - 1 ? ", " : "");
//		}
//		s += "]";
//		System.out.println("JUDAX: " + s);
		// =========================================
	}
	
	private static short[] fromObjectToShortArray(Object object) throws JSONException
	{
		short[] values = null;
		if (object instanceof JSONObject)
		{
			JSONObject jsonObject = (JSONObject)object;
			values = new short[jsonObject.length()];
		}
		else if (object instanceof JSONArray)
		{
			JSONArray jsonArray = (JSONArray)object;
			values = new short[jsonArray.length()];
		}
		else 
		{
			throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
		}
		fromObjectToShortArray(object, values);
		return values;
	}
	
	private static void fromObjectToShortArray(Object object, short[] values) throws JSONException
	{
		for (int i = 0; i < values.length; i++) 
		{
			Object o = null;
			if (object instanceof JSONObject)
			{
				JSONObject jsonObject = (JSONObject)object;
				o = jsonObject.get("" + i);
			}
			else if (object instanceof JSONArray)
			{
				JSONArray jsonArray = (JSONArray)object;
				o = jsonArray.get(i);
			}
			else 
			{
				throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
			}
			short value = 0;
			if (o instanceof Number)
			{
				value = ((Number)o).shortValue();
			}
			values[i] = value;
		}
	}
	
	private static int[] fromObjectToIntArray(Object object) throws JSONException
	{
		int[] values = null;
		if (object instanceof JSONObject)
		{
			JSONObject jsonObject = (JSONObject)object;
			values = new int[jsonObject.length()];
		}
		else if (object instanceof JSONArray)
		{
			JSONArray jsonArray = (JSONArray)object;
			values = new int[jsonArray.length()];
		}
		else 
		{
			throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
		}
		fromObjectToIntArray(object, values);
		return values;
	}
	
	private static void fromObjectToIntArray(Object object, int[] values) throws JSONException
	{
		for (int i = 0; i < values.length; i++) 
		{
			Object o = null;
			if (object instanceof JSONObject)
			{
				JSONObject jsonObject = (JSONObject)object;
				o = jsonObject.get("" + i);
			}
			else if (object instanceof JSONArray)
			{
				JSONArray jsonArray = (JSONArray)object;
				o = jsonArray.get(i);
			}
			else 
			{
				throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
			}
			int value = 0;
			if (o instanceof Number)
			{
				value = ((Number)o).intValue();
			}
			values[i] = value;
		}
	}
	
	private static byte[] fromObjectToByteArray(Object object) throws JSONException
	{
		byte[] values = null;
		if (object instanceof JSONObject)
		{
			JSONObject jsonObject = (JSONObject)object;
			values = new byte[jsonObject.length()];
		}
		else if (object instanceof JSONArray)
		{
			JSONArray jsonArray = (JSONArray)object;
			values = new byte[jsonArray.length()];
		}
		else 
		{
			throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
		}
		fromObjectToByteArray(object, values);
		return values;
	}

	private static void fromObjectToByteArray(Object object, byte[] values) throws JSONException
	{
		for (int i = 0; i < values.length; i++) 
		{
			Object o = null;
			if (object instanceof JSONObject)
			{
				JSONObject jsonObject = (JSONObject)object;
				o = jsonObject.get("" + i);
			}
			else if (object instanceof JSONArray)
			{
				JSONArray jsonArray = (JSONArray)object;
				o = jsonArray.get(i);
			}
			else 
			{
				throw new IllegalArgumentException("JUDAX: Could not identify the object nor as a JSONObject nor a JSONArray.");
			}
			byte value = 0;
			if (o instanceof Number)
			{
				value = ((Number)o).byteValue();
			}
			values[i] = value;
		}
		// =========================================
//		String s = "[";
//		for (int i = 0; i < values.length; i++) 
//		{
//			s += values[i] + (i < values.length - 1 ? ", " : "");
//		}
//		s += "]";
//		System.out.println("JUDAX: " + s);
		// =========================================
	}
		
	private static String fromWebGLNameToOpenGLName(String webGLFunctionName)
	{
		return "gl" + Character.toUpperCase(webGLFunctionName.charAt(0)) + webGLFunctionName.substring(1);	
	}
	
	// The original WebGL message and the JSON conversion of the message object
	private String message = null;
	private JSONObject messageJSON = null;
	private String webGLFunctionName = null;
	private JSONArray webGLFunctionArgs = null;
	private JSONArray supportedExtensions = null;
	
	private boolean processed = false;

	private void getSupportedExtensions()
	{
		supportedExtensions = new JSONArray();
		String supportedExtensionsString = GLES20.glGetString(GLES20.GL_EXTENSIONS);
		String[] supportedExtensionsStringArray = supportedExtensionsString.split("[\t ]");
		for (int i = 0; i < supportedExtensionsStringArray.length; i++)
		{
			supportedExtensions.put(supportedExtensionsStringArray[i]);
//			System.out.println("JUDAX: Extension " + supportedExtensionsStringArray[i] + " supported");
		}
	}
	
	public WebGLMessage(String message) throws Exception
	{
		this.message = message;
		this.messageJSON = new JSONObject(message);
		this.webGLFunctionName = messageJSON.getString("name");
		this.webGLFunctionArgs = messageJSON.getJSONArray("args");
	}
	
	public boolean isSynchronous()
	{
//		return false;
		return 
				webGLFunctionName.equals("getParameter") ||
				webGLFunctionName.equals("getActiveAttrib") ||
				webGLFunctionName.equals("getActiveUniform") ||
				webGLFunctionName.equals("getAttribLocation") || // This is a special case. The WebGL spec forces it to return a GLint so let's comply with the spec in case an engine is expecting that value
				webGLFunctionName.equals("getProgramParameter") ||
				webGLFunctionName.equals("getBufferParameter") ||
				webGLFunctionName.equals("getShaderPrecisionFormat") ||
				webGLFunctionName.equals("getShaderInfoLog") ||
				webGLFunctionName.equals("getShaderParameter");
	}
	
	public String fromWebGL2OpenGL()
	{
		String resultString = "";
		
		if (processed && ( 
				webGLFunctionName.equals("createBuffer") || 
				webGLFunctionName.equals("createTexture") || 
				webGLFunctionName.equals("createFramebuffer") || 
				webGLFunctionName.equals("createRenderbuffer") ||
				webGLFunctionName.equals("createShader") || 
				webGLFunctionName.equals("createProgram") ||
				webGLFunctionName.equals("texImage2D") ||
				webGLFunctionName.equals("getUniformLocation")
				))
		{
			return resultString;
		}
		
		processed = true;
		
		try 
		{
			// =========================================
			if (VERBOSE)
			{
				System.out.println("JUDAX: " + message);
			}
			// =========================================
			
			// These functions are not webgl messages directly but messages to be able to configure some aspects of the whole WebGL2OpenGL conversion
			if (webGLFunctionName.equals("configure"))
			{
				JSONObject webGL2OpenGLConfig = webGLFunctionArgs.getJSONObject(0);
				JSONArray projectionMatrixUniformNamesJSONArray = webGL2OpenGLConfig.getJSONArray("projectionMatrixUniformNames");
				for (int i = 0; i < projectionMatrixUniformNamesJSONArray.length(); i++)
				{
					projectionMatrixUniformNames.add(projectionMatrixUniformNamesJSONArray.getString(i));
				}
				JSONArray modelViewMatrixUniformNamesJSONArray = webGL2OpenGLConfig.getJSONArray("modelViewMatrixUniformNames");
				for (int i = 0; i < modelViewMatrixUniformNamesJSONArray.length(); i++)
				{
					modelViewMatrixUniformNames.add(modelViewMatrixUniformNamesJSONArray.getString(i));
				}
				return resultString;
{'fraction_non_alphanumeric': 0.1266839378238342, 'fraction_numerical': 0.0034110535405872193, 'mean_word_length': 3.98193159819316, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '253474', 'n_tokens_mistral': 7939, 'n_tokens_neox': 6919, 'n_words': 1729}