<commit_before><commit_msg>I forgot to add FHT class<commit_after>package csci599;

import java.io.*;
import java.util.*;
import java.util.function.*;
import static java.util.stream.Collectors.*;

public class FHT {
    public static void analyze(String[] fileNames) {
        // create sparse matrix
        final int H = 16;
        int[][] matrix = new int[H][256];
        int numFiles = 0; // don't use fileNames.length in case can't open some files
        for (String fileName : fileNames) {
            try (FileInputStream in = new FileInputStream(fileName)) {
                numFiles++;
                for (int j = 0; j < H; j++) {
                    int b = in.read();
                    if (b < 0) break;
                    matrix[j][b]++;
                }
            } catch (IOException ex) {
                System.err.println("Error reading file: " + fileName);
            }
        }
        if (numFiles == 0) {
            System.out.println("Must specify files.");
            return;
        }
        // helper function to convert matrix row into string
        final double finalNumFiles = numFiles;
        Function<int[], String> rowString = row -> Arrays.stream(row).mapToObj(c -> "" + c / finalNumFiles).collect(joining(","));
        // write csv file
        try (Writer out = new OutputStreamWriter(new FileOutputStream("fht.csv"), "UTF-8")) {
            for (int[] row : matrix) {
                out.write(rowString.apply(row) + "\n");
            }
        } catch (IOException ex) {
            System.err.println("Error writing fht.csv");
        }
        // write json file
        try (Writer out = new OutputStreamWriter(new FileOutputStream("fht.json"), "UTF-8")) {
            out.write("[\n");
            for (int[] row : matrix) {
                out.write("    [" + rowString.apply(row) + "],\n");
            }
            out.write("]\n");
        } catch (IOException ex) {
            System.err.println("Error writing fht.json");
        }
    }
}
{'fraction_non_alphanumeric': 0.1370887337986042, 'fraction_numerical': 0.006979062811565304, 'mean_word_length': 1.708502024291498, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4098115', 'n_tokens_mistral': 565, 'n_tokens_neox': 544, 'n_words': 152}