<commit_before>///////////////////////////////////////////////////////////////////////////////
//  metering.scala
//
//  Copyright (C) 2011-2014 <NAME>, The University of Texas at Austin
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
///////////////////////////////////////////////////////////////////////////////

package opennlp.textgrounder
package util

import scala.util.control.Breaks._
import scala.collection.{GenTraversable, GenTraversableOnce, GenTraversableLike}
import scala.collection.generic.CanBuildFrom

import collection.{InterruptibleIterator, SideEffectIterator}
import os._
import numeric.format_double
import print.errprint
import string.pluralize
import time._

/////////////////////////////////////////////////////////////////////////////
//                             Metered Tasks                               //
/////////////////////////////////////////////////////////////////////////////

protected class MeteringPackage {
  protected var started = false
  protected var first_time: Double = _
  protected var last_time: Double = _

  class Timer(action: String, verbose: Boolean = false,
      secs_between_output: Double = 15) {
    def start() {
      if (verbose)
        errprint("--------------------------------------------------------")
      first_time = curtimesecs
      last_time = first_time
      errprint("Beginning %s at %s.", action, humandate_full(first_time))
      started = true
    }

    def print_elapsed_time(curtime: Double) {
      val total_elapsed_secs = curtime - first_time
      errprint("Elapsed time: %s (since start: %s)",
               format_minutes_seconds(total_elapsed_secs,
                 include_hours = false),
               format_minutes_seconds(get_program_time_usage))
    }

    def finish() {
      val curtime = curtimesecs
      if (verbose || curtime - first_time >= secs_between_output) {
        errprint("Finished %s at %s.", action, humandate_full(curtime))
        // Don't include timestamp since we already output it.
        print_elapsed_time(curtime)
      }
      if (verbose)
        output_resource_usage(omit_time = true)
    }
  }

  /**
   * Time an action, displaying the elapsed time.
   *
   * @param action String describing action.
   * @param verbose If true, display more detailed messages, including
   *    amount of memory used at end, etc. Otherwise, be minimal.
   * @param fn Action, expressed as a body of code, similar to the
   *   body of a while statement.
   */
  def time_action[T](action: String, verbose: Boolean = false)(fn: => T) = {
    {
      val timer = new Timer(action, verbose = verbose)
      timer.start()
      val retval = fn
      timer.finish()
      retval
    }
  }

  /**
   * Class for tracking number of items processed in a long task, and
   * reporting periodic status messages concerning how many items
   * processed and how much real time involved.  Use in any of the following
   * ways:
   *
   * (1) Call `start` to begin processing, `item_processed` every time you have
   *     processed an item, and `finish` to end processing.
   * (1) Call `foreach` to process each item in a Traversable (either an
   *     Iterable or Iterator), automatically calling `start, `item_processed`
   *     and `end` as appropriate so that progress messages are output.
   * (3) Use `iterate` to wrap an iterator in such a way that progress messages
   *     will be output as items are fetched from the iterator.  This ensures
   *     that `start` and `finish` are executed when iteration starts and ends,
   *     not immediately -- important since the iterator may not actually be
   *     processed until some indefinite time in the future.
   *
   * @param verb Transitive verb in its -ing form indicating what is being
   *    done to the items being processed; used for status messages.  If there
   *    is a %s in the text, the item name will be substituted, else appended.
   * @param item_name Generic name of the items being processed; used for
   *    status messages.
   * @param verbose If true, display more detailed messages, including
   *    items/sec, amount of memory used at end, etc. Otherwise, be minimal.
   * @param secs_between_output Number of elapsed seconds between successive
   *    periodic status messages.
   * @param maxtime Maximum amount of time allowed for execution, in seconds.
   *    If 0.0 (the default), no maximum. This affects the return value from
   *    `item_processed`, which will be `true` if further processing should
   *    stop. This will be respected by `foreach` and `iterate`, which will
   *    stop further processing.
   * @param maxitems Maximum number of items allowed to be processed.  If 0
   *    (the default), no maximum. This works similarly to `maxtime`.
   */
  class Meter(verb: String, item_name: String,
      verbose: Boolean = false,
      secs_between_output: Double = 15, maxtime: Double = 0.0,
      maxitems: Int = 0) {
    val plural_item_name = pluralize(item_name)
    protected var items_processed = 0
    // Whether we've already printed stats after the most recent item
    // processed
    protected var printed_stats = false

    def elapsed_time = curtimesecs - first_time
    def num_processed = items_processed

    // total elapsed secs, items per second, seconds per item
    def tes_ips_spi = {
      val total_elapsed_secs = elapsed_time
      (total_elapsed_secs,
        items_processed.toDouble / total_elapsed_secs,
        total_elapsed_secs / items_processed)
    }

    class MeterTimer(action: String) extends Timer(
      action, verbose = verbose, secs_between_output = secs_between_output
    ) {
      override def print_elapsed_time(curtime: Double) {
        // Already printed timestamp.
        print_elapsed_time_and_rate(curtime, include_timestamp = false)
      }
    }

    val meter_timer = new MeterTimer(construct_predicate)

    def item_unit = {
      if (items_processed == 1)
        item_name
      else
        plural_item_name
    }

    protected def construct_predicate = {
      val plurit = plural_item_name
      if (verb contains "%s")
        verb.replace("%s", plurit)
      else
        verb + " " + plurit
    }

    def start() = {
      meter_timer.start()
      this
    }

    def print_elapsed_time_and_rate(curtime: Double = curtimesecs,
        include_timestamp: Boolean = true) {
      /* Don't do anything if already printed for this item. */
      if (printed_stats)
        return
      printed_stats = true
      val (tes, ips, spi) = tes_ips_spi
      val attime = if (!include_timestamp) "" else
        "At %s: " format humandate_time(curtime)
      val since_start = format_minutes_seconds(get_program_time_usage)
      if (verbose) {
        errprint("%sElapsed time: %s (since start: %s)",
                 attime,
                 format_minutes_seconds(tes, include_hours = false),
                 since_start)
        errprint("%s %s processed, %s items/sec, %s sec/item",
          items_processed, item_unit, format_double(ips), format_double(spi))
      } else
        errprint("%s%s %s processed (since start: %s)",
          attime, items_processed, item_unit, since_start)
    }

    /**
     * Indicate that an item has been processed.  Return value is `true` if
     * processing should stop (maximum time exceeded), `false` otherwise.<commit_msg>metering: When verbose, output memory usage each 15 secs<commit_after>///////////////////////////////////////////////////////////////////////////////
//  metering.scala
//
//  Copyright (C) 2011-2014 <NAME>, The University of Texas at Austin
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
///////////////////////////////////////////////////////////////////////////////

package opennlp.textgrounder
package util

import scala.util.control.Breaks._
import scala.collection.{GenTraversable, GenTraversableOnce, GenTraversableLike}
import scala.collection.generic.CanBuildFrom

import collection.{InterruptibleIterator, SideEffectIterator}
import os._
import numeric.format_double
import print.errprint
import string.pluralize
import time._

/////////////////////////////////////////////////////////////////////////////
//                             Metered Tasks                               //
/////////////////////////////////////////////////////////////////////////////

protected class MeteringPackage {
  protected var started = false
  protected var first_time: Double = _
  protected var last_time: Double = _

  class Timer(action: String, verbose: Boolean = false,
      secs_between_output: Double = 15) {
    def start() {
      if (verbose)
        errprint("--------------------------------------------------------")
      first_time = curtimesecs
      last_time = first_time
      errprint("Beginning %s at %s.", action, humandate_full(first_time))
      started = true
    }

    def print_elapsed_time(curtime: Double) {
      val total_elapsed_secs = curtime - first_time
      errprint("Elapsed time: %s (since start: %s)",
               format_minutes_seconds(total_elapsed_secs,
                 include_hours = false),
               format_minutes_seconds(get_program_time_usage))
    }

    def finish() {
      val curtime = curtimesecs
      if (verbose || curtime - first_time >= secs_between_output) {
        errprint("Finished %s at %s.", action, humandate_full(curtime))
        // Don't include timestamp since we already output it.
        print_elapsed_time(curtime)
      }
      if (verbose)
        output_resource_usage(omit_time = true)
    }
  }

  /**
   * Time an action, displaying the elapsed time.
   *
   * @param action String describing action.
   * @param verbose If true, display more detailed messages, including
   *    amount of memory used at end, etc. Otherwise, be minimal.
   * @param fn Action, expressed as a body of code, similar to the
   *   body of a while statement.
   */
  def time_action[T](action: String, verbose: Boolean = false)(fn: => T) = {
    {
      val timer = new Timer(action, verbose = verbose)
      timer.start()
      val retval = fn
      timer.finish()
      retval
    }
  }

  /**
   * Class for tracking number of items processed in a long task, and
   * reporting periodic status messages concerning how many items
   * processed and how much real time involved.  Use in any of the following
   * ways:
   *
   * (1) Call `start` to begin processing, `item_processed` every time you have
   *     processed an item, and `finish` to end processing.
   * (1) Call `foreach` to process each item in a Traversable (either an
   *     Iterable or Iterator), automatically calling `start, `item_processed`
   *     and `end` as appropriate so that progress messages are output.
   * (3) Use `iterate` to wrap an iterator in such a way that progress messages
   *     will be output as items are fetched from the iterator.  This ensures
   *     that `start` and `finish` are executed when iteration starts and ends,
   *     not immediately -- important since the iterator may not actually be
   *     processed until some indefinite time in the future.
   *
   * @param verb Transitive verb in its -ing form indicating what is being
   *    done to the items being processed; used for status messages.  If there
   *    is a %s in the text, the item name will be substituted, else appended.
   * @param item_name Generic name of the items being processed; used for
   *    status messages.
   * @param verbose If true, display more detailed messages, including
   *    items/sec, amount of memory used at end, etc. Otherwise, be minimal.
   * @param secs_between_output Number of elapsed seconds between successive
   *    periodic status messages.
   * @param maxtime Maximum amount of time allowed for execution, in seconds.
   *    If 0.0 (the default), no maximum. This affects the return value from
   *    `item_processed`, which will be `true` if further processing should
   *    stop. This will be respected by `foreach` and `iterate`, which will
   *    stop further processing.
   * @param maxitems Maximum number of items allowed to be processed.  If 0
   *    (the default), no maximum. This works similarly to `maxtime`.
   */
  class Meter(verb: String, item_name: String,
      verbose: Boolean = false,
      secs_between_output: Double = 15, maxtime: Double = 0.0,
      maxitems: Int = 0) {
    val plural_item_name = pluralize(item_name)
    protected var items_processed = 0
    // Whether we've already printed stats after the most recent item
    // processed
    protected var printed_stats = false

    def elapsed_time = curtimesecs - first_time
    def num_processed = items_processed

    // total elapsed secs, items per second, seconds per item
    def tes_ips_spi = {
      val total_elapsed_secs = elapsed_time
      (total_elapsed_secs,
        items_processed.toDouble / total_elapsed_secs,
        total_elapsed_secs / items_processed)
    }

    class MeterTimer(action: String) extends Timer(
      action, verbose = verbose, secs_between_output = secs_between_output
    ) {
      override def print_elapsed_time(curtime: Double) {
        // Already printed timestamp.
        print_elapsed_time_and_rate(curtime, include_timestamp = false)
      }
    }

    val meter_timer = new MeterTimer(construct_predicate)

    def item_unit = {
      if (items_processed == 1)
        item_name
      else
        plural_item_name
    }

    protected def construct_predicate = {
      val plurit = plural_item_name
      if (verb contains "%s")
        verb.replace("%s", plurit)
      else
        verb + " " + plurit
    }

    def start() = {
      meter_timer.start()
      this
    }

    def print_elapsed_time_and_rate(curtime: Double = curtimesecs,
        include_timestamp: Boolean = true) {
      /* Don't do anything if already printed for this item. */
      if (printed_stats)
        return
      printed_stats = true
      val (tes, ips, spi) = tes_ips_spi
      val attime = if (!include_timestamp) "" else
        "At %s: " format humandate_time(curtime)
      val since_start = format_minutes_seconds(get_program_time_usage)
      if (verbose) {
        errprint("%sElapsed time: %s (since start: %s)",
                 attime,
                 format_minutes_seconds(tes, include_hours = false),
                 since_start)
        errprint("%s %s processed, %s items/sec, %s sec/item",
          items_processed, item_unit, format_double(ips), format_double(spi))
        output_resource_usage(omit_time = true)
      } else
        errprint("%s%s %s processed (since start: %s)",
          attime, items_processed, item_unit, since_start)
    }

    /**
     * Indicate that an item has been processed.  Return value is `true` if
     * processing should stop (maximum time exceeded), `false` otherwise.
{'fraction_non_alphanumeric': 0.12205594493906674, 'fraction_numerical': 0.003536023236724127, 'mean_word_length': 2.3189438390611903, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5758934', 'n_tokens_mistral': 4375, 'n_tokens_neox': 4205, 'n_words': 1541}