<commit_before>/*
 * FRC 4931 (http://www.evilletech.com)
 *
 * Open source software. Licensed under the FIRST BSD license file in the
 * root directory of this project's Git repository.
 */
package org.frc4931.utils;
import java.io.BufferedInputStream;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;

/**
 * Utility to process data logs generated by strongback into human readable csv format. The path to the input 
 * file should be specified as the first command line argument. The path to the output is optional, if it is not
 * specified, the output will be saved in the current directory, with the same filename as the input, and a .csv
 * extension. <br>
 * {@code Usage: LogDecoder <input_path> [output_path]}
 * 
 * <h1>Binary file format:</h1>
 * <p>
 * The first three bytes are the ASCII log ({@code 6C 6F 67}). The next byte is the number of elements that have 
 * been logged {@code n}. The next {@code n} bytes are the number of bytes in each data point, followed by 
 * {@code n} repetitions of the length of the name of each data point and the name itself in ASCII bytes.
 * <p>
 * Now the data is recorded with respect to the number of bytes in each element. 
 * When Logger is stopped it will finish writing the current record followed by the terminator: {@code FF FF FF FF}.
 * 
 * <h1>Example binary file:</h1>
 * <i>Demonstrates the raw binary output of the logger. New lines included only for clarity, 
 * and are not part of the file format</i>
 * <pre> [l o g] 
 * [3] 
 * [4] [2] [2] 
 * [4][T i m e] [3][F o o] [3][B a r]
 * [00 00 00 00] [00 52] [00 37]
 * [00 00 00 0A] [04 D5] [23 AF]
 * [00 00 00 14] [3F 00] [12 34]
 * [FF FF FF FF]
 * </pre>
 * 
 * <h1>Output CSV format:</h1>
 * <p>
 * The first row contains the names of the elements as encoded into the log file, delimited by a comma. The end
 * of the row is delimited by a newline character. Each following line lists the integer value of the data encoded,
 * to the precision specified in the log file.
 * 
 * <h1>Example csv file:</h1>
 * <pre> Time, Foo, Bar
 * 0, 82, 55,
 * 10, 1237, 9135,
 * 20, 16128, 4660
 * </pre>
 * 
 * @author <NAME>
 *
 */
public class LogDecoder {
    public static void main(String[] args) {
        // Check arguments
        if(args.length != 1 && args.length != 2) {
            System.err.println("Usage: LogDecoder <input_path> [output_path]");
            System.exit(3);
        }
        // Resolve home directory in path name
        if(args[0].charAt(0)=='~')
            args[0] = System.getProperty("user.home")+args[0].substring(1);
        
        if(args.length==2)
            if(args[1].charAt(0)=='~')
                args[1] = System.getProperty("user.home")+args[1].substring(1);
        
        try {
            // Resolve input file
            File file = new File(args[0]).getAbsoluteFile();
            DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(file)));
            
            // Resolve output file
            File out;
            if(args.length==1) {
                // Strips file extension
                out = new File(file.getName().substring(0, 
                               file.getName().contains(".") 
                             ? file.getName().lastIndexOf('.') 
                             : file.getName().length()
                        ) + ".csv");
            } else {
                assert args.length == 2;
                out = new File(args[1]).getAbsoluteFile();
            }
            BufferedWriter writer = new BufferedWriter(new FileWriter(out));
            
            // Verify Header
            byte[] header = new byte[3];
            in.read(header);
            if(!Arrays.equals(header, "log".getBytes())){
                System.err.println("File format not recognized");
                System.exit(2);
            }
            
            // Get the number of elements
            int numElements = in.read();
            
            // Get the size of each element
            int[] elementSizes = new int[numElements];
            for(int i = 0; i< elementSizes.length; i++)
                elementSizes[i] = in.read();
            
            // Write the name of each element
            for(int i = 0; i< numElements; i++) {
                int nameSize = in.read();
                byte[] b = new byte[nameSize];
                in.read(b);
                writer.write(new String(b) + ", ");
            }
            writer.newLine();
            
            // Read each record
            in.mark(4);
            while(in.readInt()!=0xFFFFFFFF){
                in.reset();
                for(int i = 0; i < numElements; i++){
                    if(elementSizes[i]==4){
                        writer.write(in.readInt() + ", ");
                    }
                    else if(elementSizes[i]==2)
                        writer.write(in.readShort() + ", ");
                }
                writer.newLine();
                in.mark(4);
            }
            writer.close();
            in.close();
            System.out.println("Success, output saved to: "+out.getAbsolutePath());
        } catch (FileNotFoundException e) {
            System.err.println("Can not open file: " + e.getLocalizedMessage());
            System.exit(1);
        } catch (IOException e) {
            System.err.println(e.getLocalizedMessage());
        }
    }
}<commit_msg>Added unexpected end of file handling to 'LogDecoder'<commit_after>/*
 * FRC 4931 (http://www.evilletech.com)
 *
 * Open source software. Licensed under the FIRST BSD license file in the
 * root directory of this project's Git repository.
 */
package org.frc4931.utils;
import java.io.BufferedInputStream;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;

/**
 * Utility to process data logs generated by strongback into human readable csv format. The path to the input 
 * file should be specified as the first command line argument. The path to the output is optional, if it is not
 * specified, the output will be saved in the current directory, with the same filename as the input, and a .csv
 * extension. <br>
 * {@code Usage: LogDecoder <input_path> [output_path]}
 * 
 * <h1>Binary file format:</h1>
 * <p>
 * The first three bytes are the ASCII log ({@code 6C 6F 67}). The next byte is the number of elements that have 
 * been logged {@code n}. The next {@code n} bytes are the number of bytes in each data point, followed by 
 * {@code n} repetitions of the length of the name of each data point and the name itself in ASCII bytes.
 * <p>
 * Now the data is recorded with respect to the number of bytes in each element. 
 * When Logger is stopped it will finish writing the current record followed by the terminator: {@code FF FF FF FF}.
 * If the Logger was interrupted and was unable to finish writing the log, the decoder will recover as many records
 * as possible. (Because the Logger is unbuffered, everything up to the point of the crash should be recovered.)
 * 
 * <h1>Example binary file:</h1>
 * <i>Demonstrates the raw binary output of the logger. New lines included only for clarity, 
 * and are not part of the file format</i>
 * <pre> [l o g] 
 * [3] 
 * [4] [2] [2] 
 * [4][T i m e] [3][F o o] [3][B a r]
 * [00 00 00 00] [00 52] [00 37]
 * [00 00 00 0A] [04 D5] [23 AF]
 * [00 00 00 14] [3F 00] [12 34]
 * [FF FF FF FF]
 * </pre>
 * 
 * <h1>Output CSV format:</h1>
 * <p>
 * The first row contains the names of the elements as encoded into the log file, delimited by a comma. The end
 * of the row is delimited by a newline character. Each following line lists the integer value of the data encoded,
 * to the precision specified in the log file.
 * 
 * <h1>Example csv file:</h1>
 * <pre> Time, Foo, Bar
 * 0, 82, 55,
 * 10, 1237, 9135,
 * 20, 16128, 4660
 * </pre>
 * 
 * @author <NAME>
 *
 */
public class LogDecoder {
    public static void main(String[] args) {
        // Check arguments
        if(args.length != 1 && args.length != 2) {
            System.err.println("Usage: LogDecoder <input_path> [output_path]");
            System.exit(3);
        }
        // Resolve home directory in path name
        if(args[0].charAt(0)=='~')
            args[0] = System.getProperty("user.home")+args[0].substring(1);
        
        if(args.length==2)
            if(args[1].charAt(0)=='~')
                args[1] = System.getProperty("user.home")+args[1].substring(1);
        
        try {
            // Resolve input file
            File file = new File(args[0]).getAbsoluteFile();
            DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(file)));
            
            // Resolve output file
            File out;
            if(args.length==1) {
                // Strips file extension
                out = new File(file.getName().substring(0, 
                               file.getName().contains(".") 
                             ? file.getName().lastIndexOf('.') 
                             : file.getName().length()
                        ) + ".csv");
            } else {
                assert args.length == 2;
                out = new File(args[1]).getAbsoluteFile();
            }
            BufferedWriter writer = new BufferedWriter(new FileWriter(out));
            
            // Verify Header
            byte[] header = new byte[3];
            in.read(header);
            if(!Arrays.equals(header, "log".getBytes())){
                System.err.println("File format not recognized");
                System.exit(2);
            }
            
            // Get the number of elements
            int numElements = in.read();
            
            // Get the size of each element
            int[] elementSizes = new int[numElements];
            for(int i = 0; i< elementSizes.length; i++)
                elementSizes[i] = in.read();
            
            // Write the name of each element
            for(int i = 0; i< numElements; i++) {
                int nameSize = in.read();
                byte[] b = new byte[nameSize];
                in.read(b);
                writer.write(new String(b) + ", ");
            }
            writer.newLine();
            
            // Read each record
            try {
                in.mark(4);
                while(in.readInt()!=0xFFFFFFFF){
                    in.reset();
                    for(int i = 0; i < numElements; i++){
                        if(elementSizes[i]==4){
                            writer.write(in.readInt() + ", ");
                        }
                        else if(elementSizes[i]==2)
                            writer.write(in.readShort() + ", ");
                    }
                    writer.newLine();
                    in.mark(4);
                }
            } catch (EOFException e) {
                System.err.println("Unexpected end of file, did robot crash?");
            } finally {
                writer.close();
                in.close();
                System.out.println("Output saved to: "+out.getAbsolutePath());
            }
        } catch (FileNotFoundException e) {
            System.err.println("Can not open file: " + e.getLocalizedMessage());
            System.exit(1);
        } catch (IOException e) {
            System.err.println(e.getLocalizedMessage());
        }
    }
}
{'fraction_non_alphanumeric': 0.11804255319148936, 'fraction_numerical': 0.021446808510638297, 'mean_word_length': 1.839091568011597, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5628220', 'n_tokens_mistral': 3471, 'n_tokens_neox': 3204, 'n_words': 1120}