<commit_before>package org.apache.hadoop.hive.ql.exec.tez.monitoring;

import org.apache.hadoop.hive.common.log.InPlaceUpdate;
import org.apache.hadoop.hive.common.log.ProgressMonitor;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.Progress;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.StringWriter;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;

class RenderStrategy {

  interface UpdateFunction {
    void update(DAGStatus status, Map<String, Progress> vertexProgressMap);
  }

  private abstract static class BaseUpdateFunction implements UpdateFunction {
    private static final int PRINT_INTERVAL = 3000;

    final TezJobMonitor monitor;
    private final PerfLogger perfLogger;

    private long lastPrintTime = 0L;
    private String lastReport = null;

    BaseUpdateFunction(TezJobMonitor monitor) {
      this.monitor = monitor;
      perfLogger = SessionState.getPerfLogger();
    }

    @Override
    public void update(DAGStatus status, Map<String, Progress> vertexProgressMap) {
      renderProgress(monitor.progressMonitor(status, vertexProgressMap));
      String report = getReport(vertexProgressMap);
      if (showReport(report)) {
        renderReport(report);
        lastReport = report;
        lastPrintTime = System.currentTimeMillis();
      }
    }

    private boolean showReport(String report) {
      return !report.equals(lastReport)
          || System.currentTimeMillis() >= lastPrintTime + PRINT_INTERVAL;
    }

    /*
       This is used to print the progress information as pure text , a sample is as below:
          Map 1: 0/1	Reducer 2: 0/1
          Map 1: 0(+1)/1	Reducer 2: 0/1
          Map 1: 1/1	Reducer 2: 0(+1)/1
          Map 1: 1/1	Reducer 2: 1/1
     */
    private String getReport(Map<String, Progress> progressMap) {
      StringWriter reportBuffer = new StringWriter();

      SortedSet<String> keys = new TreeSet<>(progressMap.keySet());
      for (String s : keys) {
        Progress progress = progressMap.get(s);
        final int complete = progress.getSucceededTaskCount();
        final int total = progress.getTotalTaskCount();
        final int running = progress.getRunningTaskCount();
        final int failed = progress.getFailedTaskAttemptCount();
        if (total <= 0) {
          reportBuffer.append(String.format("%s: -/-\t", s));
        } else {
          if (complete == total) {
          /*
           * We may have missed the start of the vertex due to the 3 seconds interval
           */
            if (!perfLogger.startTimeHasMethod(PerfLogger.TEZ_RUN_VERTEX + s)) {
              perfLogger.PerfLogBegin(TezJobMonitor.CLASS_NAME, PerfLogger.TEZ_RUN_VERTEX + s);
            }

            perfLogger.PerfLogEnd(TezJobMonitor.CLASS_NAME, PerfLogger.TEZ_RUN_VERTEX + s);
          }
          if (complete < total && (complete > 0 || running > 0 || failed > 0)) {

            if (!perfLogger.startTimeHasMethod(PerfLogger.TEZ_RUN_VERTEX + s)) {<commit_msg>HIVE-16436: Response times in 'Task Execution Summary' at the end of the job is not correct (<NAME> reviewed by Gopal V)<commit_after>package org.apache.hadoop.hive.ql.exec.tez.monitoring;

import org.apache.hadoop.hive.common.log.InPlaceUpdate;
import org.apache.hadoop.hive.common.log.ProgressMonitor;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.Progress;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.StringWriter;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;

class RenderStrategy {

  interface UpdateFunction {
    void update(DAGStatus status, Map<String, Progress> vertexProgressMap);
  }

  private abstract static class BaseUpdateFunction implements UpdateFunction {
    private static final int PRINT_INTERVAL = 3000;

    final TezJobMonitor monitor;
    private final PerfLogger perfLogger;

    private long lastPrintTime = 0L;
    private String lastReport = null;

    BaseUpdateFunction(TezJobMonitor monitor) {
      this.monitor = monitor;
      perfLogger = SessionState.getPerfLogger();
    }

    @Override
    public void update(DAGStatus status, Map<String, Progress> vertexProgressMap) {
      renderProgress(monitor.progressMonitor(status, vertexProgressMap));
      String report = getReport(vertexProgressMap);
      if (showReport(report)) {
        renderReport(report);
        lastReport = report;
        lastPrintTime = System.currentTimeMillis();
      }
    }

    private boolean showReport(String report) {
      return !report.equals(lastReport)
          || System.currentTimeMillis() >= lastPrintTime + PRINT_INTERVAL;
    }

    /*
       This is used to print the progress information as pure text , a sample is as below:
          Map 1: 0/1	Reducer 2: 0/1
          Map 1: 0(+1)/1	Reducer 2: 0/1
          Map 1: 1/1	Reducer 2: 0(+1)/1
          Map 1: 1/1	Reducer 2: 1/1
     */
    private String getReport(Map<String, Progress> progressMap) {
      StringWriter reportBuffer = new StringWriter();

      SortedSet<String> keys = new TreeSet<>(progressMap.keySet());
      for (String s : keys) {
        Progress progress = progressMap.get(s);
        final int complete = progress.getSucceededTaskCount();
        final int total = progress.getTotalTaskCount();
        final int running = progress.getRunningTaskCount();
        final int failed = progress.getFailedTaskAttemptCount();
        if (total <= 0) {
          reportBuffer.append(String.format("%s: -/-\t", s));
        } else {
          if (complete == total) {
          /*
           * We may have missed the start of the vertex due to the 3 seconds interval
           */
            if (!perfLogger.startTimeHasMethod(PerfLogger.TEZ_RUN_VERTEX + s)) {
              perfLogger.PerfLogBegin(TezJobMonitor.CLASS_NAME, PerfLogger.TEZ_RUN_VERTEX + s);
            }

            if (!perfLogger.endTimeHasMethod(PerfLogger.TEZ_RUN_VERTEX + s)) {
              perfLogger.PerfLogEnd(TezJobMonitor.CLASS_NAME, PerfLogger.TEZ_RUN_VERTEX + s);
            }
          }
          if (complete < total && (complete > 0 || running > 0 || failed > 0)) {

            if (!perfLogger.startTimeHasMethod(PerfLogger.TEZ_RUN_VERTEX + s)) {
{'fraction_non_alphanumeric': 0.10094253572514442, 'fraction_numerical': 0.0123137731833384, 'mean_word_length': 2.9776299879081014, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3023394', 'n_tokens_mistral': 2016, 'n_tokens_neox': 1927, 'n_words': 469}