<commit_before>package eu.datacron.in_situ_processing.flink.utils;

import java.io.BufferedReader;
import java.io.FileReader;

import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.log4j.Logger;
import org.json.JSONObject;

/**
 * File lines stream source function that read the message from text file
 * 
 * @author <NAME>
 * 
 */
public class FileLinesStreamSource implements SourceFunction<String> {

  private static final long serialVersionUID = 2174904787118597072L;
  static Logger logger = Logger.getLogger(FileLinesStreamSource.class.getName());
  private String dataFilePath;

  boolean running = true;
  int i = 0;
  private int timeStampIndex;
  private String delimiter;

  public FileLinesStreamSource() {}

  public FileLinesStreamSource(String dataFilePath, String parsingJsonConfigsStr) {
    this.dataFilePath = dataFilePath;
    JSONObject parsingJsonObject = new JSONObject(parsingJsonConfigsStr);
    this.timeStampIndex = parsingJsonObject.getInt("timestamp");
    this.delimiter = parsingJsonObject.getString("delimiter");
  }


  @Override
  public void run(SourceContext<String> ctx) throws Exception {

    Thread.sleep(10000);
    long oldTimeStamp = 0;
    while (running) {

      try (BufferedReader br = new BufferedReader(new FileReader(dataFilePath))) {
        String messageLine;
        while ((messageLine = br.readLine()) != null) {
          i++;

          long newTime = Long.parseLong(messageLine.split(delimiter)[timeStampIndex]);
          ctx.collectWithTimestamp(messageLine, newTime);
          long delay = 0;
          if (oldTimeStamp != 0) {
            // System.out.println(delay);
            delay = newTime - oldTimeStamp;
          }
          oldTimeStamp = newTime;
          if (delay > 0) {
            if (delay > 0) {
              // Thread.sleep(delay);
            }
          }

//          if (i % 1000 == 0) {
//            break;
//          }
        }
      } catch (Exception e) {

        logger.error(e.getMessage(), e);
        break;
      }

      Thread.sleep(1000000);
      running = false;
    }
  }

  @Override
  public void cancel() {
    running = false;
  }

}<commit_msg>Added an option to have a warm up waiting time<commit_after>package eu.datacron.in_situ_processing.flink.utils;

import java.io.BufferedReader;
import java.io.FileReader;

import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * File lines stream source function that read the message from text file
 * 
 * @author <NAME>
 * 
 */
public class FileLinesStreamSource implements SourceFunction<String> {

  private static final long serialVersionUID = 2174904787118597072L;
  static Logger logger = LoggerFactory.getLogger(FileLinesStreamSource.class.getName());
  private String dataFilePath;

  boolean running = true;
  int i = 0;
  private int timeStampIndex;
  private String delimiter;
  boolean warmupWait = false;

  public FileLinesStreamSource() {}

  public FileLinesStreamSource(String dataFilePath, String parsingJsonConfigsStr) {
    initFields(dataFilePath, parsingJsonConfigsStr, warmupWait);
  }

  public FileLinesStreamSource(String dataFilePath, String parsingJsonConfigsStr, boolean warmupWait) {
    initFields(dataFilePath, parsingJsonConfigsStr, false);
  }

  private void initFields(String dataFilePath, String parsingJsonConfigsStr, boolean warmupWait) {
    this.dataFilePath = dataFilePath;
    JSONObject parsingJsonObject = new JSONObject(parsingJsonConfigsStr);
    this.timeStampIndex = parsingJsonObject.getInt("timestamp");
    this.delimiter = parsingJsonObject.getString("delimiter");
    this.warmupWait = warmupWait;
  }


  @Override
  public void run(SourceContext<String> ctx) throws Exception {

    Thread.sleep(1000);
    long oldTimeStamp = 0;
    while (running) {

      try (BufferedReader br = new BufferedReader(new FileReader(dataFilePath))) {
        String messageLine;
        while ((messageLine = br.readLine()) != null) {
          i++;
          long newTime = 0;
          try {
            newTime = Long.parseLong(messageLine.split(delimiter)[timeStampIndex]);
          } catch (NumberFormatException e) {
            logger.error(e.getMessage() + " for line:" + messageLine);
          }
          ctx.collectWithTimestamp(messageLine, newTime);

          if (i == 1 && warmupWait) {
            // warm up waiting time
            Thread.sleep(300000);
          }
          long delay = 0;
          if (oldTimeStamp != 0) {
            // System.out.println(delay);
            delay = newTime - oldTimeStamp;
          }
          oldTimeStamp = newTime;
          if (delay > 0) {
            if (delay > 0) {
              // Thread.sleep(delay);
            }
          }
        }
      } catch (Exception e) {

        logger.error(e.getMessage(), e);
        break;
      }

      Thread.sleep(600000);
      running = false;
    }
  }

  @Override
  public void cancel() {
    running = false;
  }

}
{'fraction_non_alphanumeric': 0.09952700039416634, 'fraction_numerical': 0.017343318880567598, 'mean_word_length': 2.756476683937824, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6651387', 'n_tokens_mistral': 1534, 'n_tokens_neox': 1431, 'n_words': 337}