<commit_before>autoio.go
=========

A lightweight (and work in progress) Go library to provide a simple interface to use compressed files
like you use normal files.

Installation
------------

    $ go get github.com/abeconnelly/autoio

Example
-------

    package main

    import "fmt"
    import "github.com/abeconnelly/autoio"

    func parrot_file( fn string ) {
      aio,err := autoio.OpenScanner( fn )
      if err!=nil { panic(err) }
      defer aio.Close()

      for aio.Scanner.Scan()  {
        l := aio.Scanner.Text()
        fmt.Println(l)
      }

    }

    func main() {

      parrot_file( "w.txt" )
      parrot_file( "x.txt.gz" )
      parrot_file( "y.txt.bz2" )
      parrot_file( "-" )

    }
<commit_msg>adding golang syntax highlighting<commit_after>autoio.go
=========

A lightweight (and work in progress) Go library to provide a simple interface to use compressed files
like you use normal files.

Installation
------------

    $ go get github.com/abeconnelly/autoio

Example
-------

```go
package main

import "fmt"
import "github.com/abeconnelly/autoio"

func parrot_file( fn string ) {
  aio,err := autoio.OpenScanner( fn )
  if err!=nil { panic(err) }
  defer aio.Close()

  for aio.Scanner.Scan()  {
    l := aio.Scanner.Text()
    fmt.Println(l)
  }

}

func main() {

  parrot_file( "w.txt" )
  parrot_file( "x.txt.gz" )
  parrot_file( "y.txt.bz2" )
  parrot_file( "-" )

}
```

{'fraction_non_alphanumeric': 0.153954802259887, 'fraction_numerical': 0.0014124293785310734, 'mean_word_length': 2.4901477832512313, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6141256', 'n_tokens_mistral': 530, 'n_tokens_neox': 502, 'n_words': 124}