<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gov.nasa.jpl.memex.pooledtimeseries;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.logging.Logger;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfPoint2f;
import org.opencv.core.Point;
import org.opencv.core.Size;
import org.opencv.highgui.VideoCapture;
import org.opencv.imgproc.Imgproc;
import org.opencv.video.Video;

/**
 * 
 * Pooled Time Series Similarity Metric.
 * 
 */
@SuppressWarnings({ "static-access", "deprecation" })
public class PoT {

  public static int frame_width = 320;
  public static int frame_height = 240;

  private static String outputFile = "similarity.txt";

  private static enum OUTPUT_FORMATS {TXT, JSON}
  private static OUTPUT_FORMATS outputFormat = OUTPUT_FORMATS.TXT;

  private static final Logger LOG = Logger.getLogger(PoT.class.getName());

  public static void main(String[] args) {
    System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
    Option dirOpt = OptionBuilder.withArgName("directory").hasArg()
        .withLongOpt("dir")
        .withDescription("A directory with image files in it").create('d');

    Option helpOpt = OptionBuilder.withLongOpt("help")
        .withDescription("Print this message.").create('h');

    Option pathFileOpt = OptionBuilder
        .withArgName("path file")
        .hasArg()
        .withLongOpt("pathfile")
        .withDescription(
            "A file containing full absolute paths to videos. Previous default was memex-index_temp.txt")
        .create('p');

    Option outputFileOpt = OptionBuilder
        .withArgName("output file")
        .withLongOpt("outputfile")
        .hasArg()
        .withDescription("File containing similarity results. Defaults to ./similarity.txt")
        .create('o');

    Option jsonOutputFlag = OptionBuilder
        .withArgName("json output")
        .withLongOpt("json")
        .withDescription("Set similarity output format to JSON. Defaults to .txt")
        .create('j');

    Options options = new Options();
    options.addOption(dirOpt);
    options.addOption(pathFileOpt);
    options.addOption(helpOpt);
    options.addOption(outputFileOpt);
    options.addOption(jsonOutputFlag);

    // create the parser
    CommandLineParser parser = new DefaultParser();

    try {
      // parse the command line arguments
      CommandLine line = parser.parse(options, args);
      String directoryPath = null;
      String pathFile = null;
      ArrayList<Path> videoFiles = null;

      if (line.hasOption("dir")) {
        directoryPath = line.getOptionValue("dir");
      }

      if (line.hasOption("pathfile")) {
        pathFile = line.getOptionValue("pathfile");
      }

      if (line.hasOption("outputfile")) {
          outputFile = line.getOptionValue("outputfile");
      }

      if (line.hasOption("json")) {
          outputFormat = OUTPUT_FORMATS.JSON;
      }

      if (line.hasOption("help")
          || (line.getOptions() == null || (line.getOptions() != null && line
              .getOptions().length == 0))
          || (directoryPath != null && pathFile != null
              && !directoryPath.equals("") && !pathFile.equals(""))) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("pooled_time_series", options);
        System.exit(1);
      }

      if (directoryPath != null) {
        File dir = new File(directoryPath);
        List<File> files = (List<File>) FileUtils.listFiles(dir,
            TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE);
        videoFiles = new ArrayList<Path>(files.size());

        for (File file : files) {
          String filePath = file.toString();

          // When given a directory to load videos from we need to ensure that we
          // don't try to load the of.txt and hog.txt intermediate result files
          // that results from previous processing runs.
          if (!filePath.contains(".txt")) {
            videoFiles.add(file.toPath());
          }
        }

        LOG.info("Added " + videoFiles.size() + " video files from "
            + directoryPath);

      }

      if (pathFile != null) {
        Path list_file = Paths.get(pathFile);
        videoFiles = loadFiles(list_file);
        LOG.info("Loaded " + videoFiles.size() + " video files from "
            + pathFile);
      }

      evaluateSimilarity(videoFiles, 0);
      LOG.info("done.");

    } catch (ParseException exp) {
      // oops, something went wrong
      System.err.println("Parsing failed.  Reason: " + exp.getMessage());
    }

  }

  public static void evaluateSimilarity(ArrayList<Path> files, int save_mode) {
    // PoT level set
    ArrayList<double[]> tws = getTemporalWindows(4);

    // computing feature vectors
    ArrayList<FeatureVector> fv_list = new ArrayList<FeatureVector>();

    for (int k = 0; k < files.size(); k++) {
      LOG.fine(files.get(k).toString());

      ArrayList<double[][]> multi_series = new ArrayList<double[][]>();
      Path file = files.get(k);

      // optical flow descriptors
      String series_name1 = file.toString() + ".of.txt";
      Path series_path1 = Paths.get(series_name1);
      double[][] series1;

      if (save_mode == 0) {
        series1 = getOpticalTimeSeries(file, 5, 5, 8);
        saveVectors(series1, series_path1);
      } else {
        series1 = loadTimeSeries(series_path1);
      }

      multi_series.add(series1);

      // gradients descriptors
      String series_name2 = file.toString() + ".hog.txt";
      Path series_path2 = Paths.get(series_name2);
      double[][] series2;

      if (save_mode == 0) {
        series2 = getGradientTimeSeries(file, 5, 5, 8);
        saveVectors(series2, series_path2);
      } else {
        series2 = loadTimeSeries(series_path2);
      }

      multi_series.add(series2);

      // computing features from series of descriptors
      FeatureVector fv = new FeatureVector();

      for (int i = 0; i < multi_series.size(); i++) {
        fv.feature.add(computeFeaturesFromSeries(multi_series.get(i), tws, 1));
        fv.feature.add(computeFeaturesFromSeries(multi_series.get(i), tws, 2));
        fv.feature.add(computeFeaturesFromSeries(multi_series.get(i), tws, 5));
      }
      System.out.println("");
      fv_list.add(fv);
    }

    double[][] similarities = calculateSimilarities(fv_list);
    writeSimilarityOutput(files, similarities);
  }

  private static double[][] calculateSimilarities(ArrayList<FeatureVector> fv_list) {
    // feature vector similarity measure
    double[] mean_dists = new double[fv_list.get(0).numDim()];
    for (int i = 0; i < fv_list.get(0).numDim(); i++)
      mean_dists[i] = meanChiSquareDistances(fv_list, i);

    for (int i = 0; i < fv_list.get(0).numDim(); i++)
      System.out.format("%f ", mean_dists[i]);
    System.out.println("");

    double[][] sims = new double[fv_list.size()][fv_list.size()];
    for (int i = 0; i < fv_list.size(); i++) {
      for (int j = 0; j < fv_list.size(); j++) {
        sims[i][j] = kernelDistance(fv_list.get(i), fv_list.get(j), mean_dists);
      }
    }

    return sims;
  }

  private static void writeSimilarityOutput(ArrayList<Path> files, double[][] similarities) {
    if (outputFormat == OUTPUT_FORMATS.TXT) {
      writeSimilarityToTextFile(similarities);
    } else if (outputFormat == OUTPUT_FORMATS.JSON) {
      writeSimilarityToJSONFile(files, similarities);
    } else {
      LOG.severe("Invalid output format. Skipping similarity dump.");
    }
  }

  private static void writeSimilarityToTextFile(double[][] similarities) {
    try {
      FileOutputStream fos = new FileOutputStream(outputFile);
      BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fos));

      for (int i = 0; i < similarities.length; i++) {
        for (int j = 0; j < similarities[0].length; j++) {
          writer.write(String.format("%f,", similarities[i][j]));
        }
        writer.newLine();
      }

      writer.close();
      fos.close();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  private static void writeSimilarityToJSONFile(ArrayList<Path> files, double[][] similarities) {
    JSONObject root_json_obj = new JSONObject();

    for (int i = 0; i < similarities.length; i++) {
      JSONObject fileJsonObj = new JSONObject();

      for (int j = 0; j < similarities[0].length; j++) {
        fileJsonObj.put(files.get(j).getFileName(), similarities[i][j]);
      }

      root_json_obj.put(files.get(i).getFileName(), fileJsonObj);
    }

    try {
      outputFile = outputFile.substring(0, outputFile.lastIndexOf('.')) + ".json";
      FileWriter file = new FileWriter(outputFile);
      file.write(root_json_obj.toJSONString());
      file.flush();
      file.close();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  public static ArrayList<Path> loadFiles(Path list_file) {
    ArrayList<Path> filenames = new ArrayList<Path>();

    try (InputStream in = Files.newInputStream(list_file);
        BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
      String line = null;
      while ((line = reader.readLine()) != null) {
        filenames.add(Paths.get(line));
      }
    } catch (IOException x) {
      System.err.println(x);
    }

    return filenames;
  }

  public static double[][] getOpticalTimeSeries(Path filename, int w_d,
      int h_d, int o_d) {
    ArrayList<double[][][]> hists = getOpticalHistograms(filename, w_d, h_d,
        o_d);
    double[][] vectors = new double[hists.size()][];

    for (int i = 0; i < hists.size(); i++) {
      vectors[i] = histogramToVector(hists.get(i));
    }

    return vectors;
  }

  static double[] histogramToVector(double[][][] hist) {
    int d1 = hist.length;
    int d2 = hist[0].length;
    int d3 = hist[0][0].length;
    double[] vector = new double[d1 * d2 * d3];

    for (int i = 0; i < d1; i++) {
      for (int j = 0; j < d2; j++) {
        for (int k = 0; k < d3; k++) {
          vector[d3 * d2 * i + d3 * j + k] = hist[i][j][k];
        }
      }
    }

    return vector;
  }

  static ArrayList<double[][][]> getOpticalHistograms(Path filename, int w_d,
      int h_d, int o_d) {
    ArrayList<double[][][]> histograms = new ArrayList<double[][][]>();

    VideoCapture capture = new VideoCapture(filename.toString());

      LOG.warning("video file not opened.");
      
      double[][][] hist = new double[w_d][h_d][o_d];
      histograms.add(hist);
    }
    else {
	    // variables for processing images
	    Mat original_frame = new Mat();
	<commit_msg>Bug fix in open video check<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gov.nasa.jpl.memex.pooledtimeseries;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.logging.Logger;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfPoint2f;
import org.opencv.core.Point;
import org.opencv.core.Size;
import org.opencv.highgui.VideoCapture;
import org.opencv.imgproc.Imgproc;
import org.opencv.video.Video;

/**
 * 
 * Pooled Time Series Similarity Metric.
 * 
 */
@SuppressWarnings({ "static-access", "deprecation" })
public class PoT {

  public static int frame_width = 320;
  public static int frame_height = 240;

  private static String outputFile = "similarity.txt";

  private static enum OUTPUT_FORMATS {TXT, JSON}
  private static OUTPUT_FORMATS outputFormat = OUTPUT_FORMATS.TXT;

  private static final Logger LOG = Logger.getLogger(PoT.class.getName());

  public static void main(String[] args) {
    System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
    Option dirOpt = OptionBuilder.withArgName("directory").hasArg()
        .withLongOpt("dir")
        .withDescription("A directory with image files in it").create('d');

    Option helpOpt = OptionBuilder.withLongOpt("help")
        .withDescription("Print this message.").create('h');

    Option pathFileOpt = OptionBuilder
        .withArgName("path file")
        .hasArg()
        .withLongOpt("pathfile")
        .withDescription(
            "A file containing full absolute paths to videos. Previous default was memex-index_temp.txt")
        .create('p');

    Option outputFileOpt = OptionBuilder
        .withArgName("output file")
        .withLongOpt("outputfile")
        .hasArg()
        .withDescription("File containing similarity results. Defaults to ./similarity.txt")
        .create('o');

    Option jsonOutputFlag = OptionBuilder
        .withArgName("json output")
        .withLongOpt("json")
        .withDescription("Set similarity output format to JSON. Defaults to .txt")
        .create('j');

    Options options = new Options();
    options.addOption(dirOpt);
    options.addOption(pathFileOpt);
    options.addOption(helpOpt);
    options.addOption(outputFileOpt);
    options.addOption(jsonOutputFlag);

    // create the parser
    CommandLineParser parser = new DefaultParser();

    try {
      // parse the command line arguments
      CommandLine line = parser.parse(options, args);
      String directoryPath = null;
      String pathFile = null;
      ArrayList<Path> videoFiles = null;

      if (line.hasOption("dir")) {
        directoryPath = line.getOptionValue("dir");
      }

      if (line.hasOption("pathfile")) {
        pathFile = line.getOptionValue("pathfile");
      }

      if (line.hasOption("outputfile")) {
          outputFile = line.getOptionValue("outputfile");
      }

      if (line.hasOption("json")) {
          outputFormat = OUTPUT_FORMATS.JSON;
      }

      if (line.hasOption("help")
          || (line.getOptions() == null || (line.getOptions() != null && line
              .getOptions().length == 0))
          || (directoryPath != null && pathFile != null
              && !directoryPath.equals("") && !pathFile.equals(""))) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("pooled_time_series", options);
        System.exit(1);
      }

      if (directoryPath != null) {
        File dir = new File(directoryPath);
        List<File> files = (List<File>) FileUtils.listFiles(dir,
            TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE);
        videoFiles = new ArrayList<Path>(files.size());

        for (File file : files) {
          String filePath = file.toString();

          // When given a directory to load videos from we need to ensure that we
          // don't try to load the of.txt and hog.txt intermediate result files
          // that results from previous processing runs.
          if (!filePath.contains(".txt")) {
            videoFiles.add(file.toPath());
          }
        }

        LOG.info("Added " + videoFiles.size() + " video files from "
            + directoryPath);

      }

      if (pathFile != null) {
        Path list_file = Paths.get(pathFile);
        videoFiles = loadFiles(list_file);
        LOG.info("Loaded " + videoFiles.size() + " video files from "
            + pathFile);
      }

      evaluateSimilarity(videoFiles, 0);
      LOG.info("done.");

    } catch (ParseException exp) {
      // oops, something went wrong
      System.err.println("Parsing failed.  Reason: " + exp.getMessage());
    }

  }

  public static void evaluateSimilarity(ArrayList<Path> files, int save_mode) {
    // PoT level set
    ArrayList<double[]> tws = getTemporalWindows(4);

    // computing feature vectors
    ArrayList<FeatureVector> fv_list = new ArrayList<FeatureVector>();

    for (int k = 0; k < files.size(); k++) {
      LOG.fine(files.get(k).toString());

      ArrayList<double[][]> multi_series = new ArrayList<double[][]>();
      Path file = files.get(k);

      // optical flow descriptors
      String series_name1 = file.toString() + ".of.txt";
      Path series_path1 = Paths.get(series_name1);
      double[][] series1;

      if (save_mode == 0) {
        series1 = getOpticalTimeSeries(file, 5, 5, 8);
        saveVectors(series1, series_path1);
      } else {
        series1 = loadTimeSeries(series_path1);
      }

      multi_series.add(series1);

      // gradients descriptors
      String series_name2 = file.toString() + ".hog.txt";
      Path series_path2 = Paths.get(series_name2);
      double[][] series2;

      if (save_mode == 0) {
        series2 = getGradientTimeSeries(file, 5, 5, 8);
        saveVectors(series2, series_path2);
      } else {
        series2 = loadTimeSeries(series_path2);
      }

      multi_series.add(series2);

      // computing features from series of descriptors
      FeatureVector fv = new FeatureVector();

      for (int i = 0; i < multi_series.size(); i++) {
        fv.feature.add(computeFeaturesFromSeries(multi_series.get(i), tws, 1));
        fv.feature.add(computeFeaturesFromSeries(multi_series.get(i), tws, 2));
        fv.feature.add(computeFeaturesFromSeries(multi_series.get(i), tws, 5));
      }
      System.out.println("");
      fv_list.add(fv);
    }

    double[][] similarities = calculateSimilarities(fv_list);
    writeSimilarityOutput(files, similarities);
  }

  private static double[][] calculateSimilarities(ArrayList<FeatureVector> fv_list) {
    // feature vector similarity measure
    double[] mean_dists = new double[fv_list.get(0).numDim()];
    for (int i = 0; i < fv_list.get(0).numDim(); i++)
      mean_dists[i] = meanChiSquareDistances(fv_list, i);

    for (int i = 0; i < fv_list.get(0).numDim(); i++)
      System.out.format("%f ", mean_dists[i]);
    System.out.println("");

    double[][] sims = new double[fv_list.size()][fv_list.size()];
    for (int i = 0; i < fv_list.size(); i++) {
      for (int j = 0; j < fv_list.size(); j++) {
        sims[i][j] = kernelDistance(fv_list.get(i), fv_list.get(j), mean_dists);
      }
    }

    return sims;
  }

  private static void writeSimilarityOutput(ArrayList<Path> files, double[][] similarities) {
    if (outputFormat == OUTPUT_FORMATS.TXT) {
      writeSimilarityToTextFile(similarities);
    } else if (outputFormat == OUTPUT_FORMATS.JSON) {
      writeSimilarityToJSONFile(files, similarities);
    } else {
      LOG.severe("Invalid output format. Skipping similarity dump.");
    }
  }

  private static void writeSimilarityToTextFile(double[][] similarities) {
    try {
      FileOutputStream fos = new FileOutputStream(outputFile);
      BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fos));

      for (int i = 0; i < similarities.length; i++) {
        for (int j = 0; j < similarities[0].length; j++) {
          writer.write(String.format("%f,", similarities[i][j]));
        }
        writer.newLine();
      }

      writer.close();
      fos.close();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  private static void writeSimilarityToJSONFile(ArrayList<Path> files, double[][] similarities) {
    JSONObject root_json_obj = new JSONObject();

    for (int i = 0; i < similarities.length; i++) {
      JSONObject fileJsonObj = new JSONObject();

      for (int j = 0; j < similarities[0].length; j++) {
        fileJsonObj.put(files.get(j).getFileName(), similarities[i][j]);
      }

      root_json_obj.put(files.get(i).getFileName(), fileJsonObj);
    }

    try {
      outputFile = outputFile.substring(0, outputFile.lastIndexOf('.')) + ".json";
      FileWriter file = new FileWriter(outputFile);
      file.write(root_json_obj.toJSONString());
      file.flush();
      file.close();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  public static ArrayList<Path> loadFiles(Path list_file) {
    ArrayList<Path> filenames = new ArrayList<Path>();

    try (InputStream in = Files.newInputStream(list_file);
        BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
      String line = null;
      while ((line = reader.readLine()) != null) {
        filenames.add(Paths.get(line));
      }
    } catch (IOException x) {
      System.err.println(x);
    }

    return filenames;
  }

  public static double[][] getOpticalTimeSeries(Path filename, int w_d,
      int h_d, int o_d) {
    ArrayList<double[][][]> hists = getOpticalHistograms(filename, w_d, h_d,
        o_d);
    double[][] vectors = new double[hists.size()][];

    for (int i = 0; i < hists.size(); i++) {
      vectors[i] = histogramToVector(hists.get(i));
    }

    return vectors;
  }

  static double[] histogramToVector(double[][][] hist) {
    int d1 = hist.length;
    int d2 = hist[0].length;
    int d3 = hist[0][0].length;
    double[] vector = new double[d1 * d2 * d3];

    for (int i = 0; i < d1; i++) {
      for (int j = 0; j < d2; j++) {
        for (int k = 0; k < d3; k++) {
          vector[d3 * d2 * i + d3 * j + k] = hist[i][j][k];
        }
      }
    }

    return vector;
  }

  static ArrayList<double[][][]> getOpticalHistograms(Path filename, int w_d,
      int h_d, int o_d) {
    ArrayList<double[][][]> histograms = new ArrayList<double[][][]>();

    VideoCapture capture = new VideoCapture(filename.toString());

    if (!capture.isOpened()) {
      LOG.warning("video file not opened.");
      
      double[][][] hist = new double[w_d][h_d][o_d];
      histograms.add(hist);
    }
    else {
	    // variables for processing images
	    Mat original_frame = new Mat();
	
{'fraction_non_alphanumeric': 0.13088493050177313, 'fraction_numerical': 0.006603350589002568, 'mean_word_length': 3.0048971596474043, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5626497', 'n_tokens_mistral': 7606, 'n_tokens_neox': 7287, 'n_words': 1817}