<commit_before>/**
 * @license
 * Copyright 2018 The FOAM Authors. All Rights Reserved.
 * http://www.apache.org/licenses/LICENSE-2.0
 */

package foam.dao;

import foam.core.*;
import foam.lib.json.ExprParser;
import foam.lib.json.JSONParser;
import foam.lib.json.Outputter;
import foam.lib.json.OutputterMode;
import foam.lib.parse.*;
import foam.mlang.order.Comparator;
import foam.mlang.predicate.Predicate;
import foam.nanos.auth.User;
import foam.nanos.logger.*;
import foam.util.SafetyUtil;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import java.util.TimeZone;

public abstract class AbstractJDAO
  extends ProxyDAO
{
  protected Pattern COMMENT = Pattern.compile("(/\\*([^*]|[\\r\\n]|(\\*+([^*/]|[\\r\\n])))*\\*+/)|(//.*)");
  protected static final ThreadLocal<SimpleDateFormat> sdf = new ThreadLocal<SimpleDateFormat>() {
    @Override
    protected SimpleDateFormat initialValue() {
      SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
      sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
      return sdf;
    }
  };

  protected final File           outFile_;
  protected final BufferedWriter out_;
  protected       Logger         logger_ = new StdoutLogger();

  public AbstractJDAO(foam.core.X x, ClassInfo classInfo, String filename) {
    this(x, new MapDAO(classInfo), filename);
    setOf(classInfo);
  }

  public AbstractJDAO(foam.core.X x, DAO delegate, String filename){
    setX(x);
    setOf(delegate.getOf());
    setDelegate(delegate);
    Logger logger = logger_;

    if ( x != null ) {
      logger = (Logger) x.get("logger");

      if ( logger == null ) logger = logger_;
    }

    logger_ = new PrefixLogger(new Object[] { "[JDAO]", filename }, logger);

    try {
      String file = filename + ".0";
      logger_.log("Loading file: " + file);
      //get repo entries in filename.0 journal first
      File inFile = getX().get(foam.nanos.fs.Storage.class).get(file);
      //load repo entries into DAO
      if ( inFile.exists() ) {
        int validEntries = loadJournal(inFile);
        logger_.log("Success reading " + validEntries + " entries from file: " + file);
      } else {
        logger_.warning("Can not find file: " + file);
      }
      //get runtime journal
      file = filename;
      logger_.log("Loading file: " + file);
      //get output journal
      outFile_ = getX().get(foam.nanos.fs.Storage.class).get(file);
      //if output journal does not existing, create one
      if ( ! outFile_.exists() ) {
        logger_.warning("Can not find file: " + file);
        //if output journal does not exist, create one
        File dir = outFile_.getAbsoluteFile().getParentFile();
        if ( !dir.exists() ) {
          logger_.log("Create dir: " + dir.getAbsolutePath());
          dir.mkdirs();
        }
        logger_.log("Create file: " + file);
        outFile_.getAbsoluteFile().createNewFile();
      } else {
        //if output journal file exists, load entries into DAO
        int validEntries = loadJournal(outFile_);
        logger_.log("Success reading " + validEntries + " entries from file: " + file);
      }
      //link output journal file to BufferedWriter
      out_ = new BufferedWriter(new FileWriter(outFile_, true));
      out_.newLine();
    } catch ( IOException e ) {
      logger_.error(e);
      throw new RuntimeException(e);
    }
  }

  protected abstract Outputter getOutputter();

  protected int loadJournal(File file)
      throws IOException
  {
    //recoding success reading entries
    int successReading = 0;
    JSONParser parser = getX().create(JSONParser.class);
    BufferedReader br = new BufferedReader(new FileReader(file));

    for ( String line ; ( line = br.readLine() ) != null ; ) {
      // skip empty lines & comment lines
      if ( SafetyUtil.isEmpty(line) ) continue;
      if ( COMMENT.matcher(line).matches() ) continue;

      try {
        char operation = line.charAt(0);
        // remove first two characters and last character
        line = line.trim().substring(2, line.trim().length() - 1);

        FObject object = parser.parseString(line);
        if ( object == null ) {<commit_msg>Add PM-ing of journal loading.<commit_after>/**
 * @license
 * Copyright 2018 The FOAM Authors. All Rights Reserved.
 * http://www.apache.org/licenses/LICENSE-2.0
 */

package foam.dao;

import foam.core.*;
import foam.lib.json.ExprParser;
import foam.lib.json.JSONParser;
import foam.lib.json.Outputter;
import foam.lib.json.OutputterMode;
import foam.lib.parse.*;
import foam.mlang.order.Comparator;
import foam.mlang.predicate.Predicate;
import foam.nanos.auth.User;
import foam.nanos.logger.*;
import foam.nanos.pm.PM;
import foam.util.SafetyUtil;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import java.util.TimeZone;

public abstract class AbstractJDAO
  extends ProxyDAO
{
  protected Pattern COMMENT = Pattern.compile("(/\\*([^*]|[\\r\\n]|(\\*+([^*/]|[\\r\\n])))*\\*+/)|(//.*)");
  protected static final ThreadLocal<SimpleDateFormat> sdf = new ThreadLocal<SimpleDateFormat>() {
    @Override
    protected SimpleDateFormat initialValue() {
      SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
      sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
      return sdf;
    }
  };

  protected final File           outFile_;
  protected final BufferedWriter out_;
  protected       Logger         logger_ = new StdoutLogger();

  public AbstractJDAO(foam.core.X x, ClassInfo classInfo, String filename) {
    this(x, new MapDAO(classInfo), filename);
    setOf(classInfo);
  }

  public AbstractJDAO(foam.core.X x, DAO delegate, String filename){
    setX(x);
    setOf(delegate.getOf());
    setDelegate(delegate);
    Logger logger = logger_;

    if ( x != null ) {
      logger = (Logger) x.get("logger");

      if ( logger == null ) logger = logger_;
    }

    logger_ = new PrefixLogger(new Object[] { "[JDAO]", filename }, logger);

    try {
      String file = filename + ".0";
      logger_.log("Loading file: " + file);
      //get repo entries in filename.0 journal first
      File inFile = getX().get(foam.nanos.fs.Storage.class).get(file);
      //load repo entries into DAO
      if ( inFile.exists() ) {
        int validEntries = loadJournal(inFile);
        logger_.log("Success reading " + validEntries + " entries from file: " + file);
      } else {
        logger_.warning("Can not find file: " + file);
      }
      //get runtime journal
      file = filename;
      logger_.log("Loading file: " + file);
      //get output journal
      outFile_ = getX().get(foam.nanos.fs.Storage.class).get(file);
      //if output journal does not existing, create one
      if ( ! outFile_.exists() ) {
        logger_.warning("Can not find file: " + file);
        //if output journal does not exist, create one
        File dir = outFile_.getAbsoluteFile().getParentFile();
        if ( !dir.exists() ) {
          logger_.log("Create dir: " + dir.getAbsolutePath());
          dir.mkdirs();
        }
        logger_.log("Create file: " + file);
        outFile_.getAbsoluteFile().createNewFile();
      } else {
        //if output journal file exists, load entries into DAO
        int validEntries = loadJournal(outFile_);
        logger_.log("Success reading " + validEntries + " entries from file: " + file);
      }
      //link output journal file to BufferedWriter
      out_ = new BufferedWriter(new FileWriter(outFile_, true));
      out_.newLine();
    } catch ( IOException e ) {
      logger_.error(e);
      throw new RuntimeException(e);
    }
  }

  protected abstract Outputter getOutputter();

  protected int loadJournal(File file)
    throws IOException
  {
    PM pm = new PM(this.getClass(), "loadJournal:" + file);
    try {
      return loadJournal_(file);
    } finally {
      pm.log(getX());
    }
  }


  protected int loadJournal_(File file)
    throws IOException
  {
    // recoding success reading entries
    int successReading = 0;
    JSONParser parser = getX().create(JSONParser.class);
    BufferedReader br = new BufferedReader(new FileReader(file));

    for ( String line ; ( line = br.readLine() ) != null ; ) {
      // skip empty lines & comment lines
      if ( SafetyUtil.isEmpty(line) ) continue;
      if ( COMMENT.matcher(line).matches() ) continue;

      try {
        char operation = line.charAt(0);
        // remove first two characters and last character
        line = line.trim().substring(2, line.trim().length() - 1);

        FObject object = parser.parseString(line);
        if ( object == null ) {
{'fraction_non_alphanumeric': 0.13417548651416866, 'fraction_numerical': 0.0027313076135199728, 'mean_word_length': 2.8442694663167103, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5501216', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2598, 'n_words': 673}