<commit_before>// Copyright 2012 <NAME>. All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

/*
Package pipe provides concurrent and (relatively) transparent transformations
along Golang channels.

For example, to count the number of items passing through a channel:

  // Define our counter
  type PipeCounter struct {
    Count int
  }

  // tell it what to do with each item
  func (c *PipeCounter) ForEach(item interface{}) {
    c.Count++ // increment the counter
  }

  // Set up our pipe
  input := make(chan interface{}, 5)
  output := make(chan interface{}, 5)
  pipe := NewPipe(input, output)

  // Add our counter
  counter := &PipeCounter{}
  pipe.ForEach(counter)

  // Now we send some items
  input <- true
  input <- true
  input <- true

  // Check how many have gone through
  fmt.Println(counter.Count) // prints "3"

You can, of course, modify the items flowing through the pipe:

  // Set up our pipe
  input := make(chan interface{}, 5)
  output := make(chan interface{}, 5)
  pipe := NewPipe(input, output)

  // Only allow ints divisible by 5
  pipe.Filter(func(item interface{}) bool {
    return (item.(int) % 5) == 0
  })

  // Now we send some items
  input <- 1 // will be dropped
  input <- 5 // will come through
*/
package pipe
<commit_msg>updating docs<commit_after>// Copyright 2012 <NAME>. All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

/*
Package pipe provides concurrent and (relatively) transparent transformations
along Golang channels.

For example, to count the number of items passing through a channel:

  // Define our counter
  type PipeCounter struct {
    Count int
  }

  // tell it what to do with each item
  func (c *PipeCounter) ForEach(item interface{}) {
    c.Count++ // increment the counter
  }

  // Set up our pipe
  input := make(chan interface{}, 5)
  output := make(chan interface{}, 5)
  pipe := NewPipe(input, output)

  // Add our counter
  counter := &PipeCounter{}
  pipe.ForEach(counter)

  // Now we send some items
  input <- true
  input <- true
  input <- true

  // Check how many have gone through
  fmt.Println(counter.Count) // prints "3"

You can, of course, modify the items flowing through the pipe:

  // Set up our pipe
  input := make(chan interface{}, 5)
  output := make(chan interface{}, 5)

  NewPipe(input, output).Filter(func(item interface{}) bool {
    // Only allow ints divisible by 5
    return (item.(int) % 5) == 0
  }).Map(func(item interface{}) interface{} {
    // Add 2 to each
    return item.(int) + 2
  }

  // Now we send some items
  input <- 1 // will be dropped
  input <- 5 // will come through as 7
*/
package pipe

{'fraction_non_alphanumeric': 0.1101511879049676, 'fraction_numerical': 0.011159107271418287, 'mean_word_length': 2.925141242937853, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1926023', 'n_tokens_mistral': 862, 'n_tokens_neox': 797, 'n_words': 353}