<commit_before>package scalaz.stream
package examples

import org.scalacheck._
import Prop._
import scalaz.concurrent.Task
import scodec.bits.ByteVector

object StartHere extends Properties("examples.StartHere") {
  
  /*

  Let's start by looking at a reasonably complete example.
  This program opens a file `fahrenheit.txt`, containing
  temperatures in degrees fahrenheit, one per line, and 
  converts each temperature to celsius, incrementally
  writing to the file `celsius.txt`. Both files will be
  closed, regardless of whether any errors occur.

  */

  property("simple file I/O") = secure {

    val converter: Task[Unit] = 
      io.linesR("testdata/fahrenheit.txt")
        .filter(s => !s.trim.isEmpty && !s.startsWith("//"))
        .map(line => fahrenheitToCelsius(line.toDouble).toString)
        .intersperse("\n")
        .pipe(text.utf8Encode)
        .to(io.fileChunkW("testdata/celsius.txt"))
        .run

    converter.run
    true
  }

  /*

  Let's dissect this line by line - 

  */

  property("dissected simple file I/O") = secure {
    
    /* 

    `Process[Task, String]` is a stream of `String` values which may
    periodically evaluate a `scalaz.concurrent.Task` in order to
    produce additional values. `Process` is the core data type of 
    `scalaz.stream`. It is parameterized on a type constructor (here, `Task`)
    which defines what sort of external requests it can make, and
    an output type (here, `String`), which defines what type of 
    values it _emits_.

    Many operations on `Process` are defined for any choice of 
    type constructor, not just `Task`.

    `scalaz.stream.io` has a number of helper functions for 
    constructing or working with streams that talk to the outside world.
    `linesR` creates a stream of lines from a filename. It encapsulates
    the logic for opening and closing the file, so that users of this
    stream do not need to remember to close the file when the are done
    or in the event of exceptions during processing of the stream.<commit_msg>Fix typo.<commit_after>package scalaz.stream
package examples

import org.scalacheck._
import Prop._
import scalaz.concurrent.Task
import scodec.bits.ByteVector

object StartHere extends Properties("examples.StartHere") {
  
  /*

  Let's start by looking at a reasonably complete example.
  This program opens a file `fahrenheit.txt`, containing
  temperatures in degrees fahrenheit, one per line, and 
  converts each temperature to celsius, incrementally
  writing to the file `celsius.txt`. Both files will be
  closed, regardless of whether any errors occur.

  */

  property("simple file I/O") = secure {

    val converter: Task[Unit] = 
      io.linesR("testdata/fahrenheit.txt")
        .filter(s => !s.trim.isEmpty && !s.startsWith("//"))
        .map(line => fahrenheitToCelsius(line.toDouble).toString)
        .intersperse("\n")
        .pipe(text.utf8Encode)
        .to(io.fileChunkW("testdata/celsius.txt"))
        .run

    converter.run
    true
  }

  /*

  Let's dissect this line by line - 

  */

  property("dissected simple file I/O") = secure {
    
    /* 

    `Process[Task, String]` is a stream of `String` values which may
    periodically evaluate a `scalaz.concurrent.Task` in order to
    produce additional values. `Process` is the core data type of 
    `scalaz.stream`. It is parameterized on a type constructor (here, `Task`)
    which defines what sort of external requests it can make, and
    an output type (here, `String`), which defines what type of 
    values it _emits_.

    Many operations on `Process` are defined for any choice of 
    type constructor, not just `Task`.

    `scalaz.stream.io` has a number of helper functions for 
    constructing or working with streams that talk to the outside world.
    `linesR` creates a stream of lines from a filename. It encapsulates
    the logic for opening and closing the file, so that users of this
    stream do not need to remember to close the file when they are done
    or in the event of exceptions during processing of the stream.
{'fraction_non_alphanumeric': 0.08163765628830595, 'fraction_numerical': 0.0004903162539838195, 'mean_word_length': 3.108761329305136, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5007756', 'n_tokens_mistral': 1176, 'n_tokens_neox': 1127, 'n_words': 472}