<commit_before># Copyright 2006-2012 <NAME>
"""TSV reading classes"""
import sys,csv
from pycbio.sys import fileOps
from pycbio.tsv.tsvRow import TSVRow
from pycbio.tsv.tsvError import TSVError

# FIXME:  pass owndership of row to Row instead of having Row inherit from list
# FIXME: create error with file name/line number
# FIMXE: carefully consider naming of functions in Row, as they could
#  conflict with fields.  Maybe put in a base-class??
# FIXME: put in same module as TSV
# FIXME: TSVError and preserving the traceback is a pain
# FIXME: is colMap needed any more???
# FIXME: need to add write stuff. (see GeneCheck), where str() is called on
#        alfor all column ty
# FIXME: change TSVRow to be based on collections.namedtuple **** (thanks Max)
#
# rename  typeMap -> colTypes
# FIXME: make a column object.
# FIXME: save original column names before mapping for output

# typeMap converter for str types were empty represents None
strOrNoneType = (lambda v: None if (v == "") else v,
                 lambda v: "" if (v == None) else v)

# typeMap converter for int types were empty represents None
intOrNoneType = (lambda v: None if (v == "") else int(v),
                 lambda v: "" if (v == None) else str(v))

class TSVReader(object):
    """Class for reading TSV files.  Reads header and builds column name to
    column index map.  After a next, object contains a row and each column
    becomes a field name.  It is also indexable by column name or int index.<commit_msg>added fixme<commit_after># Copyright 2006-2012 <NAME>
"""TSV reading classes"""
import sys,csv
from pycbio.sys import fileOps
from pycbio.tsv.tsvRow import TSVRow
from pycbio.tsv.tsvError import TSVError

# FIXME:  pass owndership of row to Row instead of having Row inherit from list
# FIXME: create error with file name/line number
# FIMXE: carefully consider naming of functions in Row, as they could
#  conflict with fields.  Maybe put in a base-class??
# FIXME: put in same module as TSV
# FIXME: TSVError and preserving the traceback is a pain
# FIXME: is colMap needed any more???
# FIXME: need to add write stuff. (see GeneCheck), where str() is called on
#        alfor all column ty
# FIXME: change TSVRow to be based on collections.namedtuple **** (thanks Max)
#
# rename  typeMap -> colTypes
# FIXME: make a column object.
# FIXME: save original column names before mapping for output

# FIXME: switch row to deriving from named tuple or using properties?

# typeMap converter for str types were empty represents None
strOrNoneType = (lambda v: None if (v == "") else v,
                 lambda v: "" if (v == None) else v)

# typeMap converter for int types were empty represents None
intOrNoneType = (lambda v: None if (v == "") else int(v),
                 lambda v: "" if (v == None) else str(v))

class TSVReader(object):
    """Class for reading TSV files.  Reads header and builds column name to
    column index map.  After a next, object contains a row and each column
    becomes a field name.  It is also indexable by column name or int index.
{'fraction_non_alphanumeric': 0.0799347471451876, 'fraction_numerical': 0.0052202283849918435, 'mean_word_length': 3.6244343891402715, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5301821', 'n_tokens_mistral': 891, 'n_tokens_neox': 880, 'n_words': 443}