<commit_before>package csvutil
/* 
*  File: reader.go
*  Author: <NAME> [<EMAIL>] 
*  Created: Sat May 28 23:53:36 PDT 2011
*  Description: CSV reader library.
*
*   This file is part of csvutil.
*
*   csvutil is free software: you can redistribute it and/or modify
*   it under the terms of the GNU Lesser Public License as published by
*   the Free Software Foundation, either version 3 of the License, or
*   (at your option) any later version.
*
*   csvutil is distributed in the hope that it will be useful,
*   but WITHOUT ANY WARRANTY; without even the implied warranty of
*   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*   GNU Lesser Public License for more details.
*
*   You should have received a copy of the GNU Lesser Public License
*   along with csvutil.  If not, see <http://www.gnu.org/licenses/>.
*/
import (
	"io"
	"bufio"
	"strings"
	"os"
)

//  A reader object for CSV data utilizing the bufio package.
type Reader struct {
	Sep int "Field separator character."
	Trim bool "Remove excess whitespace from field values."
	Cutset string "Set of characters to trim."
    LastRow Row "The last row read by the Reader."
	r io.Reader "Base reader object."
	br *bufio.Reader "For reading lines."
    bufSize int "Initial size of internal line buffers."
}

//  A simple row structure for rows read by a csvutil.Reader that
//  encapsulates any read error enountered along with any data read
//  prior to encountering an error.
type Row struct {
	Fields []string "CSV row field data"
	Error os.Error "Error encountered reading"
}

func mkrow(fields []string, err os.Error) Row {
	var r Row
	r.Fields = fields
	r.Error = err
	return r
}

//  An object for iterating over the rows of a Reader.
//      rit = reader.RowIterAuto()
//      for r := range reader.RowsChan {
//          if r.Error != nil {
//              panic(r.Error)
//          }
//          var fields []string = r.Fields
//          // Process the desired entries of "fields".
//          rit.Next()
//      }
//  The (*Reader) RowIterAuto() (*RowReaderIterator) method creates a
//  new iterator object and immediately reads a row from the Reader.
//  If this behavior is not desired, use the underlying alternate method
//  (*Reader) RowIter() (*RowReaderIterator).
type ReaderRowIterator struct {
    stopped bool
    RowsChan <-chan Row
    control chan<- bool
}

//  A ReaderRowIterator meant for reading until the end of the
//  data stream of the corresponding Reader's io.Reader. 
//      rit = reader.RowIterAuto()
//      for r := range reader.RowsChan {<commit_msg>Fix some erroneous reader.go documentation.<commit_after>package csvutil
/* 
*  File: reader.go
*  Author: <NAME> [<EMAIL>] 
*  Created: Sat May 28 23:53:36 PDT 2011
*  Description: CSV reader library.
*
*   This file is part of csvutil.
*
*   csvutil is free software: you can redistribute it and/or modify
*   it under the terms of the GNU Lesser Public License as published by
*   the Free Software Foundation, either version 3 of the License, or
*   (at your option) any later version.
*
*   csvutil is distributed in the hope that it will be useful,
*   but WITHOUT ANY WARRANTY; without even the implied warranty of
*   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*   GNU Lesser Public License for more details.
*
*   You should have received a copy of the GNU Lesser Public License
*   along with csvutil.  If not, see <http://www.gnu.org/licenses/>.
*/
import (
	"io"
	"bufio"
	"strings"
	"os"
)

//  A reader object for CSV data utilizing the bufio package.
type Reader struct {
	Sep int "Field separator character."
	Trim bool "Remove excess whitespace from field values."
	Cutset string "Set of characters to trim."
    LastRow Row "The last row read by the Reader."
	r io.Reader "Base reader object."
	br *bufio.Reader "For reading lines."
    bufSize int "Initial size of internal line buffers."
}

//  A simple row structure for rows read by a csvutil.Reader that
//  encapsulates any read error enountered along with any data read
//  prior to encountering an error.
type Row struct {
	Fields []string "CSV row field data"
	Error os.Error "Error encountered reading"
}

func mkrow(fields []string, err os.Error) Row {
	var r Row
	r.Fields = fields
	r.Error = err
	return r
}

//  An object for iterating over the rows of a Reader.
//      rit = reader.RowIterStarted()
//      for r := range reader.RowsChan {
//          if r.Error != nil {
//              rit.Break()
//              panic(r.Error)
//          }
//          var fields []string = r.Fields
//          if !fieldsOk(fields) {
//              rit.Break()
//              break
//          }
//          // Process the desired entries of "fields".
//          rit.Next()
//      }
//  This itererator is safe to break out of. For iterators meant for
//  parsing an entire stream, see the ReaderRowIteratorAuto type.
type ReaderRowIterator struct {
    stopped bool
    RowsChan <-chan Row
    control chan<- bool
}

//  A ReaderRowIteratorAuto is meant for reading until encountering the
//  end of the data stream corresponding to a Reader's underlying
//  io.Reader. 
//      rit = reader.RowIterAuto()
//      for r := range reader.RowsChan {
{'fraction_non_alphanumeric': 0.09068251533742332, 'fraction_numerical': 0.004984662576687116, 'mean_word_length': 2.7237687366167025, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4761436', 'n_tokens_mistral': 1554, 'n_tokens_neox': 1476, 'n_words': 674}