<commit_before>/* Copyright (c) 2011 The Regents of the University of California */

package org.cdlib.was.ngIndexer;

import java.io.{BufferedInputStream,File,InputStream,FileInputStream,FileOutputStream,OutputStream};
import java.util.Date;

import org.archive.util.ArchiveUtils;

import scala.util.matching.Regex;

object Utility {
  val ARC_RE = new Regex(""".*?([A-Za-z0-9\.-]+arc.gz).*""");

  /**
   * Convert a possibly null thing into an Option version.
   */
  def null2option[A] (what : A) : Option[A] = {
    if (what == null) None;
    else Some(what);
  }

  def writeStreamToTempFile (prefix : String, in : InputStream) : File = { 
    val tempFile = File.createTempFile(prefix, null);
    readStreamIntoFile(tempFile, in);
    tempFile.deleteOnExit();
    return tempFile;
  }

  /**
   * Read a set of bytes from an input stream and pass them on to a function.
   *
   * @param stream The InputStream to return from.
   * @param f a function taking two arguments, an Int and Array[byte]. Used for side effects.
   * @return Unit
   */
  def readBytes (stream : InputStream, f : (Int, Array[Byte]) => Unit) : Unit = {
    var buffer = new Array[Byte](1024);
    var bytesRead = stream.read(buffer);
    while (bytesRead != -1)  {
      f (bytesRead, buffer);
      bytesRead = stream.read(buffer);
    }
  }

  /**
   *
   * Open an OutputStream on a File, ensuring that the stream is closed.
   *
   * @param file The File to open.
   * @param f a function taking two arguments, an Int and Array[byte]. Used for side effects.
   */
  def withFileOutputStream[A] (file : File, f : OutputStream => A) : A = {
    val out = new FileOutputStream(file);
    try {
      f (out);
    } finally {
      out.close();
    }
  }

  def withFileInputStream[A] (file : File) (f : InputStream => A) : A = {
    val in = new FileInputStream(file)
    try {
      f(in);
    } finally {
      in.close();
    }
  }

  def readStreamIntoFile (file : File, in : InputStream) = {
    withFileOutputStream (file, (out) =>
      readBytes(in, (bytesRead, buffer) =>
        out.write(buffer, 0, bytesRead)));
  }

  def flushStream (in : InputStream, out : OutputStream) : Unit = {
    readBytes(in, (bytesRead, buffer) =>
      out.write(buffer, 0, bytesRead));
  }

  val dateFormatter = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
  dateFormatter.setTimeZone(java.util.TimeZone.getTimeZone("UTC"))
  
  implicit def date2string (d : Date) : String =
    dateFormatter.format(d);
    
  implicit def string2date (s : String) : Date = s match {
    case ds : String if ds.length == 14 =>
      ArchiveUtils.parse14DigitDate(s);
    case ds : String if ds.length == 20 =>
      dateFormatter.parse(s);
    }

  def timeout[T] (msec : Int) (f: => T) : Option[T] = {
    class TimeoutThread extends Thread {
      var retval : Option[T] = None;
      var ex : Option[Throwable] = None;
      override def run {
        try {
          retval = Some(f);
        } catch {
          /* finished, do nothing */ 
          case t : InterruptedException => ()<commit_msg>add dumpStream method, curry functions<commit_after>/* Copyright (c) 2011 The Regents of the University of California */

package org.cdlib.was.ngIndexer;

import java.io.{BufferedInputStream,File,InputStream,FileInputStream,FileOutputStream,OutputStream};
import java.util.Date;

import org.archive.util.ArchiveUtils;

import scala.util.matching.Regex;

object Utility {
  val ARC_RE = new Regex(""".*?([A-Za-z0-9\.-]+arc.gz).*""");

  /**
   * Convert a possibly null thing into an Option version.
   */
  def null2option[A] (what : A) : Option[A] = {
    if (what == null) None;
    else Some(what);
  }

  def writeStreamToTempFile (prefix : String, in : InputStream) : File = { 
    val tempFile = File.createTempFile(prefix, null);
    readStreamIntoFile(tempFile, in);
    tempFile.deleteOnExit();
    return tempFile;
  }

  /**
   * Read a set of bytes from an input stream and pass them on to a function.
   *
   * @param stream The InputStream to return from.
   * @param f a function taking two arguments, an Int and Array[byte]. Used for side effects.
   * @return Unit
   */
  def readBytes (stream : InputStream) (f : (Int, Array[Byte]) => Unit) : Unit = {
    var buffer = new Array[Byte](1024);
    var bytesRead = stream.read(buffer);
    while (bytesRead != -1)  {
      f (bytesRead, buffer);
      bytesRead = stream.read(buffer);
    }
  }

  /**
   *
   * Open an OutputStream on a File, ensuring that the stream is closed.
   *
   * @param file The File to open.
   * @param f a function taking two arguments, an Int and Array[byte]. Used for side effects.
   */
  def withFileOutputStream[A] (file : File) (f : OutputStream => A) : A = {
    val out = new FileOutputStream(file);
    try {
      f (out);
    } finally {
      out.close();
    }
  }

  def withFileInputStream[A] (file : File) (f : InputStream => A) : A = {
    val in = new FileInputStream(file)
    try {
      f(in);
    } finally {
      in.close();
    }
  }

  def readStreamIntoFile (file : File, in : InputStream) = {
    withFileOutputStream (file) { out =>
      readBytes(in) { (bytesRead, buffer) =>
        out.write(buffer, 0, bytesRead);
      }
    }
  }

  def flushStream (in : InputStream, out : OutputStream) : Unit = {
    readBytes(in) { (bytesRead, buffer) =>
      out.write(buffer, 0, bytesRead);
    }
  }

  def dumpStream (in : InputStream) : Unit =
    readBytes(in) { (bytesRead, buffer) => () };

  val dateFormatter = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
  dateFormatter.setTimeZone(java.util.TimeZone.getTimeZone("UTC"))
  
  implicit def date2string (d : Date) : String =
    dateFormatter.format(d);
    
  implicit def string2date (s : String) : Date = s match {
    case ds : String if ds.length == 14 =>
      ArchiveUtils.parse14DigitDate(s);
    case ds : String if ds.length == 20 =>
      dateFormatter.parse(s);
    }

  def timeout[T] (msec : Int) (f: => T) : Option[T] = {
    class TimeoutThread extends Thread {
      var retval : Option[T] = None;
      var ex : Option[Throwable] = None;
      override def run {
        try {
          retval = Some(f);
        } catch {
          /* finished, do nothing */ 
          case t : InterruptedException => ()
{'fraction_non_alphanumeric': 0.13323804986501508, 'fraction_numerical': 0.006987454343338097, 'mean_word_length': 2.764494919306635, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '367712', 'n_tokens_mistral': 2066, 'n_tokens_neox': 2009, 'n_words': 586}