<commit_before>/* Copyright (c) 2011 The Regents of the University of California */

package org.cdlib.was.weari;

import java.io.{BufferedInputStream,EOFException,File,InputStream,FileInputStream,FileOutputStream,OutputStream};
import java.util.Date;
import java.util.zip.GZIPInputStream;

import org.archive.util.ArchiveUtils;

import scala.util.matching.Regex;

object Utility {
  /* some of arcs end in .open or .gz.gz - we should fix */
  val ARC_RE = new Regex("""^.*?([A-Za-z0-9\.-]+arc\.gz)(?:\.open|\.gz)?$""");

  /**
   * Remove .open or extra .gz at end of arc file, and junk at beginning.
   * Returns None if we cannot extract an arcname.
   */
  def extractArcname(arcname : String) : Option[String] = arcname match {
    case ARC_RE(extracted) => Some(extracted);
    case _                 => None;
  }
    
  /**
   * Convert a possibly null thing into an Option version.
   */
  def null2option[A] (what : A) : Option[A] = {
    if (what == null) None;
    else Some(what);
  }

  def writeStreamToTempFile (prefix : String, in : InputStream) : File = { 
    val tempFile = File.createTempFile(prefix, null);
    readStreamIntoFile(tempFile, in);
    tempFile.deleteOnExit();
    return tempFile;
  }

  /**
   * Read a set of bytes from an input stream and pass them on to a function.
   *
   * @param stream The InputStream to return from.
   * @param f a function taking two arguments, an Int and Array[byte]. Used for side effects.
   * @return Unit
   */
  def readBytes (stream : InputStream) (f : (Int, Array[Byte]) => Unit) : Unit = {
    var buffer = new Array[Byte](1024);
    var bytesRead = stream.read(buffer);
    while (bytesRead != -1)  {
      f (bytesRead, buffer);
      bytesRead = stream.read(buffer);<commit_msg>add null2seq<commit_after>/* Copyright (c) 2011 The Regents of the University of California */

package org.cdlib.was.weari;

import java.io.{BufferedInputStream,EOFException,File,InputStream,FileInputStream,FileOutputStream,OutputStream};
import java.util.{Collection=>JCollection,Date};
import java.util.zip.GZIPInputStream;

import org.archive.util.ArchiveUtils;

import scala.util.matching.Regex;
import scala.collection.JavaConversions.collectionAsScalaIterable;

object Utility {
  /* some of arcs end in .open or .gz.gz - we should fix */
  val ARC_RE = new Regex("""^.*?([A-Za-z0-9\.-]+arc\.gz)(?:\.open|\.gz)?$""");

  /**
   * Remove .open or extra .gz at end of arc file, and junk at beginning.
   * Returns None if we cannot extract an arcname.
   */
  def extractArcname(arcname : String) : Option[String] = arcname match {
    case ARC_RE(extracted) => Some(extracted);
    case _                 => None;
  }
    
  /**
   * Convert a possibly null thing into an Option version.
   */
  def null2option[A] (what : A) : Option[A] = {
    if (what == null) None;
    else Some(what);
  }

  /**
   * Convert a possibly null Java Collection into a scala Seq.
   * If the collection is null, returns an empty collection
   */
  def null2seq[A](collection : JCollection[A]) : Seq[A] = 
    null2option(collection).map(collectionAsScalaIterable(_)).getOrElse(List[A]()).toSeq;

  def writeStreamToTempFile (prefix : String, in : InputStream) : File = { 
    val tempFile = File.createTempFile(prefix, null);
    readStreamIntoFile(tempFile, in);
    tempFile.deleteOnExit();
    return tempFile;
  }

  /**
   * Read a set of bytes from an input stream and pass them on to a function.
   *
   * @param stream The InputStream to return from.
   * @param f a function taking two arguments, an Int and Array[byte]. Used for side effects.
   * @return Unit
   */
  def readBytes (stream : InputStream) (f : (Int, Array[Byte]) => Unit) : Unit = {
    var buffer = new Array[Byte](1024);
    var bytesRead = stream.read(buffer);
    while (bytesRead != -1)  {
      f (bytesRead, buffer);
      bytesRead = stream.read(buffer);
{'fraction_non_alphanumeric': 0.12953901622456862, 'fraction_numerical': 0.00695338655678599, 'mean_word_length': 3.2541073384446877, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2134025', 'n_tokens_mistral': 1257, 'n_tokens_neox': 1219, 'n_words': 374}