<commit_before>/*
 * A Go package to handles access to a CSV file as Dataset
 *
 * Copyright (C) 2016 <NAME> <<EMAIL>>
 *
 * Licensed under an MIT licence.  Please see LICENCE.md for details.
 */

// Package dcsv handles access to a CSV file as Dataset
package dcsv

import (
	"encoding/csv"
	"github.com/lawrencewoodman/ddataset"
	"github.com/lawrencewoodman/dlit"
	"io"
	"os"
)

// DCSV represents a CSV file Dataset
type DCSV struct {
	filename   string
	fieldNames []string
	hasHeader  bool
	separator  rune
	numFields  int
}

// DCSVConn represents a connection to a DCSV Dataset
type DCSVConn struct {
	dataset       *DCSV
	file          *os.File
	reader        *csv.Reader
	currentRecord ddataset.Record
	err           error
}

// New creates a new DCSV Dataset
func New(
	filename string,
	hasHeader bool,
	separator rune,
	fieldNames []string,
) ddataset.Dataset {
	return &DCSV{
		filename:   filename,
		fieldNames: fieldNames,
		hasHeader:  hasHeader,
		separator:  separator,
		numFields:  len(fieldNames),
	}
}

// Open creates a connection to the Dataset
func (c *DCSV) Open() (ddataset.Conn, error) {
	f, r, err := makeCsvReader(c.filename, c.separator, c.hasHeader)
	if err != nil {
		return nil, err
	}
	r.Comma = c.separator

	return &DCSVConn{
		dataset:       c,
		file:          f,
		reader:        r,
		currentRecord: make(ddataset.Record, c.numFields),
		err:           nil,
	}, nil
}

// GetFieldNames returns the field names used by the Dataset
func (c *DCSV) GetFieldNames() []string {
	return c.fieldNames
}

// Next returns whether there is a Record to be Read
func (cc *DCSVConn) Next() bool {
	if cc.err != nil {
		return false
	}
	if cc.reader == nil {
		cc.err = ddataset.ErrConnClosed
		return false
	}
	row, err := cc.reader.Read()
	if err == io.EOF {
		return false
	} else if err != nil {
		cc.Close()
		cc.err = err
		return false
	}
	if err := cc.makeRowCurrentRecord(row); err != nil {
		cc.Close()
		cc.err = err
		return false
	}
	return true
}

// Err returns any errors from the connection
func (cc *DCSVConn) Err() error {
	return cc.err
}

// Read returns the current Record
func (cc *DCSVConn) Read() ddataset.Record {
	return cc.currentRecord
}

// Close closes the connection
func (cc *DCSVConn) Close() error {
	err := cc.file.Close()
	cc.file = nil
	cc.reader = nil
	return err
}

func (cc *DCSVConn) getNumFields() int {
	return cc.dataset.numFields
}

func (cc *DCSVConn) getFieldNames() []string {
	return cc.dataset.fieldNames
}

func (cc *DCSVConn) makeRowCurrentRecord(row []string) error {<commit_msg>Remove redundant getFieldNames() in dcsv<commit_after>/*
 * A Go package to handles access to a CSV file as Dataset
 *
 * Copyright (C) 2016 <NAME> <<EMAIL>>
 *
 * Licensed under an MIT licence.  Please see LICENCE.md for details.
 */

// Package dcsv handles access to a CSV file as Dataset
package dcsv

import (
	"encoding/csv"
	"github.com/lawrencewoodman/ddataset"
	"github.com/lawrencewoodman/dlit"
	"io"
	"os"
)

// DCSV represents a CSV file Dataset
type DCSV struct {
	filename   string
	fieldNames []string
	hasHeader  bool
	separator  rune
	numFields  int
}

// DCSVConn represents a connection to a DCSV Dataset
type DCSVConn struct {
	dataset       *DCSV
	file          *os.File
	reader        *csv.Reader
	currentRecord ddataset.Record
	err           error
}

// New creates a new DCSV Dataset
func New(
	filename string,
	hasHeader bool,
	separator rune,
	fieldNames []string,
) ddataset.Dataset {
	return &DCSV{
		filename:   filename,
		fieldNames: fieldNames,
		hasHeader:  hasHeader,
		separator:  separator,
		numFields:  len(fieldNames),
	}
}

// Open creates a connection to the Dataset
func (c *DCSV) Open() (ddataset.Conn, error) {
	f, r, err := makeCsvReader(c.filename, c.separator, c.hasHeader)
	if err != nil {
		return nil, err
	}
	r.Comma = c.separator

	return &DCSVConn{
		dataset:       c,
		file:          f,
		reader:        r,
		currentRecord: make(ddataset.Record, c.numFields),
		err:           nil,
	}, nil
}

// GetFieldNames returns the field names used by the Dataset
func (c *DCSV) GetFieldNames() []string {
	return c.fieldNames
}

// Next returns whether there is a Record to be Read
func (cc *DCSVConn) Next() bool {
	if cc.err != nil {
		return false
	}
	if cc.reader == nil {
		cc.err = ddataset.ErrConnClosed
		return false
	}
	row, err := cc.reader.Read()
	if err == io.EOF {
		return false
	} else if err != nil {
		cc.Close()
		cc.err = err
		return false
	}
	if err := cc.makeRowCurrentRecord(row); err != nil {
		cc.Close()
		cc.err = err
		return false
	}
	return true
}

// Err returns any errors from the connection
func (cc *DCSVConn) Err() error {
	return cc.err
}

// Read returns the current Record
func (cc *DCSVConn) Read() ddataset.Record {
	return cc.currentRecord
}

// Close closes the connection
func (cc *DCSVConn) Close() error {
	err := cc.file.Close()
	cc.file = nil
	cc.reader = nil
	return err
}

func (cc *DCSVConn) getNumFields() int {
	return cc.dataset.numFields
}

func (cc *DCSVConn) makeRowCurrentRecord(row []string) error {
{'fraction_non_alphanumeric': 0.10760118460019744, 'fraction_numerical': 0.0015794669299111549, 'mean_word_length': 3.1456628477905073, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6557733', 'n_tokens_mistral': 1862, 'n_tokens_neox': 1785, 'n_words': 579}