<commit_before>package org.reprap.comms;

import javax.swing.JFileChooser;
import javax.swing.filechooser.FileFilter;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.OutputStream;
import java.io.PrintStream;

import gnu.io.CommPortIdentifier;
import gnu.io.NoSuchPortException;
import gnu.io.PortInUseException;
import gnu.io.SerialPort;
import gnu.io.UnsupportedCommOperationException;

import org.reprap.utilities.Debug;
import org.reprap.utilities.RrGraphics;
import org.reprap.utilities.ExtensionFileFilter;
import org.reprap.Main;
import org.reprap.Preferences;
import org.reprap.geometry.LayerRules;

public class GCodeReaderAndWriter
{
	// Codes for responses from the machine
	// A positive number returned is a request for that line number
	// to be resent.
	
	private static final long shutDown = -3;
	//private static final long startOrNullResponse = -2;
	private static final long allSentOK = -1;
	private double eTemp;
	private double bTemp;
	private double x, y, z, e;
	private RrGraphics simulationPlot = null;
	/**
	 * Stop sending a file (if we are).
	 */
	private boolean paused = false;
	
	private boolean iAmPaused = false;
	
	/**
	 * Not quite sure why this is needed...
	 */
	private boolean alreadyReversed = false;
	
	/**
	 * The name of the port talking to the RepRap machine
	 */
	String portName;
	
	/**
	* this is if we need to talk over serial
	*/
	private SerialPort port;
	
	/**
	 * Flag to tell it we've finished
	 */
	private boolean exhaustBuffer = false;
	
	/**
	* this is for doing easy serial writes
	*/
	private PrintStream serialOutStream = null;
	
	/**
	 * this is our read handle on the serial port
	 */
	private InputStream serialInStream = null;
	
	/**
	 * The root file name for output (without ".gcode" on the end)
	 */
	private String opFileName;
	
	/**
	 * List of file names - used to reverse layer order when layers are done top-down
	 */
	private String[] opFileArray;
	
	/**
	 * Index into opFileArray
	 */
	private int opFileIndex;
	
	/**
	 * How does the first file name in a multiple set end?
	 */
	private static final String gcodeExtension = ".gcode";
	
	/**
	 * How does the first file name in a multiple set end?
	 */
	private static final String firstEnding = "_prologue";
	
	/**
	 * How does the last file name in a multiple set end?
	 */
	private static final String lastEnding = "_epilogue";
	
	/**
	 * Flag for temporary files
	 */
	private static final String tmpString = "_TeMpOrArY_";
		
	/**
	 * This is used for file input
	 */
	private BufferedReader fileInStream = null;
	
	/**
	 * How long is our G-Code input file (if any).
	 */
	private long fileInStreamLength = -1;
	
	/**
	 * This is for file output
	 */
	private PrintStream fileOutStream = null;
	
	/**
	 * The last command sent
	 */
	//private String lastCommand;
	
	/**
	 * The current linenumber
	 */
	private long lineNumber;
	
	/**
	 * The ring buffer that stores the commands for 
	 * possible resend requests.
	 */
	private int head, tail;
	private static final int buflen = 10; // No too long, or pause doesn't work well
	private String[] ringBuffer;
	private long[] ringLines;
	
	/**
	 * The transmission to the RepRap machine is handled by
	 * a separate thread.  These control that.
	 */
	//private boolean threadLock = false;
	private Thread bufferThread = null;
	private int myPriority;
	
	/**
	 * Some commands (at the moment just M105 - get temperature and M114 - get coords) generate
	 * a response.  Return that as a string.
	 */
	private int responsesExpected = 0;
//	private boolean responseAvailable = false;
//	private String response;
	
	//private boolean sendFileToMachine = false;
		
	public GCodeReaderAndWriter()
	{
		init();
	}
	
	/**
	 * constructor for when we definitely want to send GCodes to a known file
	 * @param fos
	 */
	public GCodeReaderAndWriter(PrintStream fos)
	{
		init();
		fileOutStream = fos;
	}
	
	private void init()
	{
		resetReceived();
		paused = false;
		iAmPaused = false;
		alreadyReversed = false;
		ringBuffer = new String[buflen];
		ringLines = new long[buflen];
		head = 0;
		tail = 0;
		lineNumber = 0;
		//threadLock = false;
		exhaustBuffer = false;
		responsesExpected = 0;
//		responseAvailable = false;
//		response = "0000";
		opFileIndex = -1;
		try
		{
			portName = Preferences.loadGlobalString("Port(name)");
		} catch (Exception ex)
		{
			System.err.println("Cannot load preference Port(name).");
			portName = "stdout";
		}

		openSerialConnection(portName);

		myPriority = Thread.currentThread().getPriority();

		bufferThread = null;
	}

	
	public boolean buildingFromFile()
	{
		return fileInStream != null;
	}
	
	public boolean savingToFile()
	{
		return fileOutStream != null;
	}

	/**
	 * Stop the printer building.
	 * This _shouldn't_ also stop it being controlled interactively.
	 */
	public void pause()
	{
		paused = true;
//		while(!bufferEmpty())
//		{
//			//System.err.println("Waiting for buffer to empty.");
//			sleep (131);
//		}
	}
	
	/**
	 * Resume building.
	 *
	 */
	public void resume()
	{
		paused = false;
	}
	
	/**
	 * Start the production run
	 * (as opposed to driving the machine interactively).
	 */
	public void startRun()
	{
		if(fileOutStream != null)
		{
			// Exhause buffer before we start.
			if(bufferThread != null)
			{
				exhaustBuffer = true;
				while(exhaustBuffer) sleep(200);
			}
			bufferThread = null;
			head = 0;
			tail = 0;
		}
	}
	
	/**
	 * Are we paused?
	 * @return
	 */
	public boolean iAmPaused()
	{
		return iAmPaused;
	}
	
	/**
	 * Send a GCode file to the machine if that's what we have to do, and
	 * return true.  Otherwise return false.
	 *
	 */
	public boolean filePlay()
	{
		if(fileInStream == null)
		{
			// Not playing a file...
			return false;
		}
		
		simulationPlot = null;
		try 
		{
			if(Preferences.loadGlobalBool("DisplaySimulation"))
				simulationPlot = new RrGraphics("RepRap building simulation");
		} catch (IOException e1) {
			e1.printStackTrace();
		}
//		if(bufferThread == null)
//		{
//			System.err.println("GCodeWriter: attempt to write to non-existent buffer.");
//			return true;
//		}			
		
		// Launch a thread to run through the file, so we can return control
		// to the user
		
		Thread playFile = new Thread() 
		{
			public void run() 
			{
				Thread.currentThread().setName("GCode file printer");
				String line;
				long bytes = 0;
				double fractionDone = 0;
				try 
				{
					while ((line = fileInStream.readLine()) != null) 
					{
						bufferQueue(line, 0);
						bytes += line.length();
						fractionDone = (double)bytes/(double)fileInStreamLength;
						setFractionDone(fractionDone, -1, -1);
						while(paused)
						{
							iAmPaused = true;
							//System.err.println("Waiting for pause to end.");
							sleep(239);
						}
						iAmPaused = false;
					}
					fileInStream.close();
				} catch (Exception e) 
				{  
					Debug.e("Error printing file: " + e.toString());
					e.printStackTrace();
				}
			}
		};
		
		playFile.start();

	    return true;
	}
	
	public void setFractionDone(double fractionDone, int layer, int outOf)
	{
		org.reprap.gui.botConsole.BotConsoleFrame.getBotConsoleFrame().setFractionDone(fractionDone, layer, outOf);
	}
	
	/**
	 * Wrapper for Thread.sleep()
	 * @param millis
	 */
	public void sleep(int millis)
	{
		try
		{
			Thread.sleep(millis);
		} catch (Exception ex)
		{}		
	}
	
	/**
	 * All done.
	 *
	 */
	public void finish()
	{
		Debug.d("disposing of GCodeReaderAndWriter.");
		
//		// Wait for the ring buffer to be exhausted
//		if(fileOutStream == null && bufferThread != null)
//		{
//			exhaustBuffer = true;
//			while(exhaustBuffer) sleep(200);
//		}
		
		try
		{
			if (serialInStream != null)
				serialInStream.close();

			if (serialOutStream != null)
				serialOutStream.close();
			
			if (fileInStream != null)
				fileInStream.close();
			
			if (fileOutStream != null)
				fileOutStream.close();
			
		} catch (Exception e) {}
		
	}
	
	/**
	 * Anything in the buffer?  (NB this still works if we aren't
	 * using the buffer as then head == tail == 0 always).
	 * @return
	 */
//	public boolean bufferEmpty()
//	{
//		return head == tail;
//	}
	
	/**
	 * Between layers nothing will be queued.  Use the next two
	 * functions to slow and speed the buffer's spinning.
	 *
	 */
	public void slowBufferThread()
	{
		if(bufferThread != null)
			bufferThread.setPriority(1);
	}
	
	public void speedBufferThread()
	{
		if(bufferThread != null)		
			bufferThread.setPriority(myPriority);
	}
	
	/**
	 * Compute the checksum of a GCode string.
	 * @param cmd
	 * @return
	 */
	private String checkSum(String cmd)
	{
		int cs = 0;
		for(int i = 0; i < cmd.length(); i++)
			cs = cs ^ cmd.charAt(i);
		cs &= 0xff;
		return "*" + cs;
	}
	
	private void ringAdd(long ln, String cmd)
	{
		head++;
		if(head >= ringBuffer.length)
			head = 0;
		ringBuffer[head] = cmd;
		ringLines[head] = ln;
	}
	
	private String ringGet(long ln)
	{
		int h = head;
		do
		{
			if(ringLines[h] == ln)
				return ringBuffer[h];
			h--;
			if(h < 0)
				h = ringBuffer.length - 1;
		} while(h != head);
		Debug.e("ringGet: line " + ln + " not stored");
		return "";
	}
	
	/**
	 * Send a command to the machine.  Return true if a response is expected; 
	 * false if not.
	 * @param cmd
	 * @return
	 */
	private boolean sendLine(String cmd)
	{
		int com = cmd.indexOf(';');
		if(com > 0)
			cmd = cmd.substring(0, com);
		if(com != 0)
		{
			cmd = cmd.trim();
			if(cmd.length() > 0)
			{
				ringAdd(lineNumber, cmd);
				cmd = "N" + lineNumber + " " + cmd + " ";
				cmd += checkSum(cmd);
				serialOutStream.print(cmd + "\n");				
				serialOutStream.flush();
				Debug.c("G-code: " + cmd + " dequeued and sent");
				return true;
			}
			return false;
		}
	
		Debug.c("G-code: " + cmd + " not sent");
		if(cmd.startsWith(";#!LAYER:"))
		{
			int l = Integer.parseInt(cmd.substring(cmd.indexOf(" ") + 1, cmd.indexOf("/")));
			int o = Integer.parseInt(cmd.substring(cmd.indexOf("/") + 1));
			setFractionDone(-1, l, o+1);
		}
		return false;
	}
	
	/**
	 * Queue a command.  
	 */
	private void bufferQueue(String cmd, int retries) throws Exception
	{
		if(simulationPlot != null)
			simulationPlot.add(cmd);
		
		if(serialOutStream == null)
		{
			Debug.d("bufferQueue: attempt to queue: " + cmd + " to a non-running output buffer.");
			return;
		}
		if(retries > 3)
		{
			Debug.e("bufferQueue(): too many retries (" + retries + ") sending " + cmd);
			return;			
		}
		if(sendLine(cmd))
		{
			long resp = waitForResponse();
			if(resp == shutDown)
			{
				throw new Exception("The RepRap machine has flagged a hard error!");
			} else if (resp == allSentOK)
			{
				lineNumber++;
			} else // Must be a re-send line number
			{
				long gotTo = lineNumber;
				lineNumber = resp;
				while(lineNumber <= gotTo)
				{
					String rCmd = ringGet(lineNumber);
					bufferQueue(rCmd, retries+1);  // Is recursion clever; or stupid?
				}
			}<commit_msg>Fixed infinite loop in re-sending a GCode when the ring buffer doesn't have the one to send available.<commit_after>package org.reprap.comms;

import javax.swing.JFileChooser;
import javax.swing.filechooser.FileFilter;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.OutputStream;
import java.io.PrintStream;

import gnu.io.CommPortIdentifier;
import gnu.io.NoSuchPortException;
import gnu.io.PortInUseException;
import gnu.io.SerialPort;
import gnu.io.UnsupportedCommOperationException;

import org.reprap.utilities.Debug;
import org.reprap.utilities.RrGraphics;
import org.reprap.utilities.ExtensionFileFilter;
import org.reprap.Main;
import org.reprap.Preferences;
import org.reprap.geometry.LayerRules;

public class GCodeReaderAndWriter
{
	// Codes for responses from the machine
	// A positive number returned is a request for that line number
	// to be resent.
	
	private static final long shutDown = -3;
	//private static final long startOrNullResponse = -2;
	private static final long allSentOK = -1;
	private double eTemp;
	private double bTemp;
	private double x, y, z, e;
	private RrGraphics simulationPlot = null;
	/**
	 * Stop sending a file (if we are).
	 */
	private boolean paused = false;
	
	private boolean iAmPaused = false;
	
	/**
	 * Not quite sure why this is needed...
	 */
	private boolean alreadyReversed = false;
	
	/**
	 * The name of the port talking to the RepRap machine
	 */
	String portName;
	
	/**
	* this is if we need to talk over serial
	*/
	private SerialPort port;
	
	/**
	 * Flag to tell it we've finished
	 */
	private boolean exhaustBuffer = false;
	
	/**
	* this is for doing easy serial writes
	*/
	private PrintStream serialOutStream = null;
	
	/**
	 * this is our read handle on the serial port
	 */
	private InputStream serialInStream = null;
	
	/**
	 * The root file name for output (without ".gcode" on the end)
	 */
	private String opFileName;
	
	/**
	 * List of file names - used to reverse layer order when layers are done top-down
	 */
	private String[] opFileArray;
	
	/**
	 * Index into opFileArray
	 */
	private int opFileIndex;
	
	/**
	 * How does the first file name in a multiple set end?
	 */
	private static final String gcodeExtension = ".gcode";
	
	/**
	 * How does the first file name in a multiple set end?
	 */
	private static final String firstEnding = "_prologue";
	
	/**
	 * How does the last file name in a multiple set end?
	 */
	private static final String lastEnding = "_epilogue";
	
	/**
	 * Flag for temporary files
	 */
	private static final String tmpString = "_TeMpOrArY_";
		
	/**
	 * This is used for file input
	 */
	private BufferedReader fileInStream = null;
	
	/**
	 * How long is our G-Code input file (if any).
	 */
	private long fileInStreamLength = -1;
	
	/**
	 * This is for file output
	 */
	private PrintStream fileOutStream = null;
	
	/**
	 * The last command sent
	 */
	//private String lastCommand;
	
	/**
	 * The current linenumber
	 */
	private long lineNumber;
	
	/**
	 * The ring buffer that stores the commands for 
	 * possible resend requests.
	 */
	private int head, tail;
	private static final int buflen = 10; // No too long, or pause doesn't work well
	private String[] ringBuffer;
	private long[] ringLines;
	
	/**
	 * The transmission to the RepRap machine is handled by
	 * a separate thread.  These control that.
	 */
	//private boolean threadLock = false;
	private Thread bufferThread = null;
	private int myPriority;
	
	/**
	 * Some commands (at the moment just M105 - get temperature and M114 - get coords) generate
	 * a response.  Return that as a string.
	 */
	private int responsesExpected = 0;
//	private boolean responseAvailable = false;
//	private String response;
	
	//private boolean sendFileToMachine = false;
		
	public GCodeReaderAndWriter()
	{
		init();
	}
	
	/**
	 * constructor for when we definitely want to send GCodes to a known file
	 * @param fos
	 */
	public GCodeReaderAndWriter(PrintStream fos)
	{
		init();
		fileOutStream = fos;
	}
	
	private void init()
	{
		resetReceived();
		paused = false;
		iAmPaused = false;
		alreadyReversed = false;
		ringBuffer = new String[buflen];
		ringLines = new long[buflen];
		head = 0;
		tail = 0;
		lineNumber = 0;
		//threadLock = false;
		exhaustBuffer = false;
		responsesExpected = 0;
//		responseAvailable = false;
//		response = "0000";
		opFileIndex = -1;
		try
		{
			portName = Preferences.loadGlobalString("Port(name)");
		} catch (Exception ex)
		{
			System.err.println("Cannot load preference Port(name).");
			portName = "stdout";
		}

		openSerialConnection(portName);

		myPriority = Thread.currentThread().getPriority();

		bufferThread = null;
	}

	
	public boolean buildingFromFile()
	{
		return fileInStream != null;
	}
	
	public boolean savingToFile()
	{
		return fileOutStream != null;
	}

	/**
	 * Stop the printer building.
	 * This _shouldn't_ also stop it being controlled interactively.
	 */
	public void pause()
	{
		paused = true;
//		while(!bufferEmpty())
//		{
//			//System.err.println("Waiting for buffer to empty.");
//			sleep (131);
//		}
	}
	
	/**
	 * Resume building.
	 *
	 */
	public void resume()
	{
		paused = false;
	}
	
	/**
	 * Start the production run
	 * (as opposed to driving the machine interactively).
	 */
	public void startRun()
	{
		if(fileOutStream != null)
		{
			// Exhause buffer before we start.
			if(bufferThread != null)
			{
				exhaustBuffer = true;
				while(exhaustBuffer) sleep(200);
			}
			bufferThread = null;
			head = 0;
			tail = 0;
		}
	}
	
	/**
	 * Are we paused?
	 * @return
	 */
	public boolean iAmPaused()
	{
		return iAmPaused;
	}
	
	/**
	 * Send a GCode file to the machine if that's what we have to do, and
	 * return true.  Otherwise return false.
	 *
	 */
	public boolean filePlay()
	{
		if(fileInStream == null)
		{
			// Not playing a file...
			return false;
		}
		
		simulationPlot = null;
		try 
		{
			if(Preferences.loadGlobalBool("DisplaySimulation"))
				simulationPlot = new RrGraphics("RepRap building simulation");
		} catch (IOException e1) {
			e1.printStackTrace();
		}
//		if(bufferThread == null)
//		{
//			System.err.println("GCodeWriter: attempt to write to non-existent buffer.");
//			return true;
//		}			
		
		// Launch a thread to run through the file, so we can return control
		// to the user
		
		Thread playFile = new Thread() 
		{
			public void run() 
			{
				Thread.currentThread().setName("GCode file printer");
				String line;
				long bytes = 0;
				double fractionDone = 0;
				try 
				{
					while ((line = fileInStream.readLine()) != null) 
					{
						bufferQueue(line, 0);
						bytes += line.length();
						fractionDone = (double)bytes/(double)fileInStreamLength;
						setFractionDone(fractionDone, -1, -1);
						while(paused)
						{
							iAmPaused = true;
							//System.err.println("Waiting for pause to end.");
							sleep(239);
						}
						iAmPaused = false;
					}
					fileInStream.close();
				} catch (Exception e) 
				{  
					Debug.e("Error printing file: " + e.toString());
					e.printStackTrace();
				}
			}
		};
		
		playFile.start();

	    return true;
	}
	
	public void setFractionDone(double fractionDone, int layer, int outOf)
	{
		org.reprap.gui.botConsole.BotConsoleFrame.getBotConsoleFrame().setFractionDone(fractionDone, layer, outOf);
	}
	
	/**
	 * Wrapper for Thread.sleep()
	 * @param millis
	 */
	public void sleep(int millis)
	{
		try
		{
			Thread.sleep(millis);
		} catch (Exception ex)
		{}		
	}
	
	/**
	 * All done.
	 *
	 */
	public void finish()
	{
		Debug.d("disposing of GCodeReaderAndWriter.");
		
//		// Wait for the ring buffer to be exhausted
//		if(fileOutStream == null && bufferThread != null)
//		{
//			exhaustBuffer = true;
//			while(exhaustBuffer) sleep(200);
//		}
		
		try
		{
			if (serialInStream != null)
				serialInStream.close();

			if (serialOutStream != null)
				serialOutStream.close();
			
			if (fileInStream != null)
				fileInStream.close();
			
			if (fileOutStream != null)
				fileOutStream.close();
			
		} catch (Exception e) {}
		
	}
	
	/**
	 * Anything in the buffer?  (NB this still works if we aren't
	 * using the buffer as then head == tail == 0 always).
	 * @return
	 */
//	public boolean bufferEmpty()
//	{
//		return head == tail;
//	}
	
	/**
	 * Between layers nothing will be queued.  Use the next two
	 * functions to slow and speed the buffer's spinning.
	 *
	 */
	public void slowBufferThread()
	{
		if(bufferThread != null)
			bufferThread.setPriority(1);
	}
	
	public void speedBufferThread()
	{
		if(bufferThread != null)		
			bufferThread.setPriority(myPriority);
	}
	
	/**
	 * Compute the checksum of a GCode string.
	 * @param cmd
	 * @return
	 */
	private String checkSum(String cmd)
	{
		int cs = 0;
		for(int i = 0; i < cmd.length(); i++)
			cs = cs ^ cmd.charAt(i);
		cs &= 0xff;
		return "*" + cs;
	}
	
	private void ringAdd(long ln, String cmd)
	{
		head++;
		if(head >= ringBuffer.length)
			head = 0;
		ringBuffer[head] = cmd;
		ringLines[head] = ln;
	}
	
	private String ringGet(long ln)
	{
		int h = head;
		do
		{
			if(ringLines[h] == ln)
				return ringBuffer[h];
			h--;
			if(h < 0)
				h = ringBuffer.length - 1;
		} while(h != head);
		Debug.e("ringGet: line " + ln + " not stored");
		return "";
	}
	
	/**
	 * Send a command to the machine.  Return true if a response is expected; 
	 * false if not.
	 * @param cmd
	 * @return
	 */
	private boolean sendLine(String cmd)
	{
		int com = cmd.indexOf(';');
		if(com > 0)
			cmd = cmd.substring(0, com);
		if(com != 0)
		{
			cmd = cmd.trim();
			if(cmd.length() > 0)
			{
				ringAdd(lineNumber, cmd);
				cmd = "N" + lineNumber + " " + cmd + " ";
				cmd += checkSum(cmd);
				serialOutStream.print(cmd + "\n");				
				serialOutStream.flush();
				Debug.c("G-code: " + cmd + " dequeued and sent");
				return true;
			}
			return false;
		}
	
		Debug.c("G-code: " + cmd + " not sent");
		if(cmd.startsWith(";#!LAYER:"))
		{
			int l = Integer.parseInt(cmd.substring(cmd.indexOf(" ") + 1, cmd.indexOf("/")));
			int o = Integer.parseInt(cmd.substring(cmd.indexOf("/") + 1));
			setFractionDone(-1, l, o+1);
		}
		return false;
	}
	
	/**
	 * Queue a command.  
	 */
	private void bufferQueue(String cmd, int retries) throws Exception
	{
		if(simulationPlot != null)
			simulationPlot.add(cmd);
		
		if(serialOutStream == null)
		{
			Debug.d("bufferQueue: attempt to queue: " + cmd + " to a non-running output buffer.");
			return;
		}
		if(retries > 3)
		{
			Debug.e("bufferQueue(): too many retries (" + retries + ") sending " + cmd);
			return;			
		}
		if(sendLine(cmd))
		{
			long resp = waitForResponse();
			if(resp == shutDown)
			{
				throw new Exception("The RepRap machine has flagged a hard error!");
			} else if (resp == allSentOK)
			{
				lineNumber++;
			} else // Must be a re-send line number
			{
				long gotTo = lineNumber;
				lineNumber = resp;
				String rCmd = " ";
				while(lineNumber <= gotTo && !rCmd.contentEquals(""))
				{
					rCmd = ringGet(lineNumber);
					bufferQueue(rCmd, retries+1);  // Is recursion clever; or stupid?
				}
			}
{'fraction_non_alphanumeric': 0.12545976495918185, 'fraction_numerical': 0.0054723243922131515, 'mean_word_length': 2.8261541101767635, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5135798', 'n_tokens_mistral': 8544, 'n_tokens_neox': 7160, 'n_words': 2157}