# Path: pump/vivopump.py
# def read_csv_fp(fp, skip=True, delimiter="|"):
#     """
#     Given a filename, read the CSV file with that name.  We use "|" as a
#     separator in CSV files to allow commas to appear in values.
# 
#     CSV files read by this function follow these conventions:
#     --  use delimiter as a separator. Defaults to vertical bar.
#     --  have a first row that contains column headings.
#     --  all elements must have values.  To specify a missing value, use
#         the string "None" or "NULL" between separators, that is |None| or |NULL|
#     --  leading and trailing whitespace in values is ignored.  | The  | will be
#         read as "The"
#     -- if skip=True, rows with too many or too few data elements are skipped.
#        if skip=False, a RowError is thrown
# 
#     CSV files processed by read_csv will be returned as a dictionary of
#     dictionaries, one dictionary per row keyed by an integer row number.  This supports
#     maintaining the order of the data input, which is important for some applications
#     """
# 
#     class RowError(Exception):
#         """
#         Thrown when the number of data elements on a row in a CSV is not equal to the number of header elements
#         """
#         pass
# 
#     heading = []
#     row_number = 0
#     data = {}
#     for row in UnicodeCsvReader(fp, delimiter=delimiter):
#         i = 0
#         for r in row:
#             # remove white space fore and aft
#             row[i] = r.strip(string.whitespace)
#             i += 1
#         if len(heading) == 0:
#             heading = row  # the first row is the heading
#             continue
#         row_number += 1
#         if len(row) == len(heading):
#             data[row_number] = {}
#             i = 0
#             for r in row:
#                 data[row_number][heading[i]] = r
#                 i += 1
#         elif not skip:
#             raise RowError("On row " + str(row_number) + ", expecting " +
#                            str(len(heading)) + " data values. Found " +
#                            str(len(row)) + " data values. Row contents = " +
#                            str(row))
#         else:
#             pass  # row has wrong number of columns and skip is True
#     logger.debug("loader returns {} rows".format(len(data)))
#     return data
# 
# def write_csv_fp(fp, data, delimiter='|'):
#     """
#     Write a CSV to a file pointer.  Used to support stdout.
#     :param fp: File pointer.  Could be stdout.
#     :param data: data to be written
#     :param delimiter: field delimiter for output
#     :return:
#     """
#     assert(len(data.keys()) > 0)
# 
#     # create a list of var_names from the first row
#     var_names = data[data.keys()[0]].keys()
#     fp.write(delimiter.join(var_names) + '\n')
# 
#     for key in sorted(data.keys()):
#         fp.write(delimiter.join([data[key][x] for x in var_names]) + '\n')
# 
# Path: pump/vivopump.py
# class DefNotFoundException(Exception):
# class InvalidDefException(Exception):
# class InvalidSourceException(Exception):
# class PathLengthException(Exception):
# class UnicodeCsvReader(object):
# class UnicodeDictReader(csv.DictReader):
#     class RowError(Exception):
#     def __init__(self, value):
#     def __str__(self):
#     def __init__(self, value):
#     def __str__(self):
#     def __init__(self, value):
#     def __str__(self):
#     def __init__(self, value):
#     def __str__(self):
#     def __init__(self, f, encoding="utf-8", **kwargs):
#     def __iter__(self):
#     def next(self):
#     def line_num(self):
#     def __init__(self, f, encoding="utf-8", fieldnames=None, **kwds):
# def read_csv(filename, skip=True, delimiter='|'):
# def read_csv_fp(fp, skip=True, delimiter="|"):
# def write_csv_fp(fp, data, delimiter='|'):
# def write_csv(filename, data, delimiter='|'):
# def replace_initials(s):
#     def repl_function(m):
# def key_string(s):
# def get_vivo_types(selector, parms, separator=';'):
# def get_vivo_ufid(parms):
# def get_vivo_publishers(parms):
# def get_vivo_journals(parms):
# def get_vivo_ccn(parms):
# def get_vivo_sponsorid(parms):
# def get_vivo_authors(parms):
# def get_vivo_positions(parms):
# def read_update_def(filename, prefix):
#     def make_prefix_dict(prefix_text):
#     def cast_to_rdflib(t):
#     def fixit(current_object, prefix_dictionary):
#     def add_order(a, b):
#     def validate_update_def(a):
#     def add_object_names_and_step_attributes(a):
# def add_qualifiers(input_path):
# def gather_types(input_step, varname):
# def make_update_query(entity_sparql, path):
# def make_rdf_term(row_term):
# def get_graph(update_def, query_parms):
# def new_uri(parms):
# def vivo_query(query, parms):
# def write_update_def(update_def, filename):
# def parse_pages(pages):
# def parse_date_parts(month, year):
# def get_args():
# def get_parms():
# def add_type_restriction(step):
# def make_get_query(update_def):
# def unique_path(path):
# def make_get_data(update_def, result_set):
# def make_rdf_term_from_source(value, step):
# def prepare_column_values(update_string, intra, step_def, enum, row, column_name):
# def load_enum(update_def):
# def create_enum(filename, query, parms, trim=0, skip=0):


import sys
from pump.vivopump import read_csv_fp, write_csv_fp
from pump.vivopump import improve_title, parse_pages, parse_date_parts
from disambiguate.utils import print_err
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
pub_columns_filter.py -- add needed columns, remove unused columns
"""

__author__ = "Michael Conlon"
__copyright__ = "Copyright 2015 (c) Michael Conlon"
__license__ = "New BSD License"
__version__ = "0.01"




# Path: pump/vivopump.py
# def read_csv_fp(fp, skip=True, delimiter="|"):
#     """
#     Given a filename, read the CSV file with that name.  We use "|" as a
#     separator in CSV files to allow commas to appear in values.
# 
#     CSV files read by this function follow these conventions:
#     --  use delimiter as a separator. Defaults to vertical bar.
#     --  have a first row that contains column headings.
#     --  all elements must have values.  To specify a missing value, use
#         the string "None" or "NULL" between separators, that is |None| or |NULL|
#     --  leading and trailing whitespace in values is ignored.  | The  | will be
#         read as "The"
#     -- if skip=True, rows with too many or too few data elements are skipped.
#        if skip=False, a RowError is thrown
# 
#     CSV files processed by read_csv will be returned as a dictionary of
#     dictionaries, one dictionary per row keyed by an integer row number.  This supports
#     maintaining the order of the data input, which is important for some applications
#     """
# 
#     class RowError(Exception):
#         """
#         Thrown when the number of data elements on a row in a CSV is not equal to the number of header elements
#         """
#         pass
# 
#     heading = []
#     row_number = 0
#     data = {}
#     for row in UnicodeCsvReader(fp, delimiter=delimiter):
#         i = 0
#         for r in row:
#             # remove white space fore and aft
#             row[i] = r.strip(string.whitespace)
#             i += 1
#         if len(heading) == 0:
#             heading = row  # the first row is the heading
#             continue
#         row_number += 1
#         if len(row) == len(heading):
#             data[row_number] = {}
#             i = 0
#             for r in row:
#                 data[row_number][heading[i]] = r
#                 i += 1
#         elif not skip:
#             raise RowError("On row " + str(row_number) + ", expecting " +
#                            str(len(heading)) + " data values. Found " +
#                            str(len(row)) + " data values. Row contents = " +
#                            str(row))
#         else:
#             pass  # row has wrong number of columns and skip is True
#     logger.debug("loader returns {} rows".format(len(data)))
#     return data
# 
# def write_csv_fp(fp, data, delimiter='|'):
#     """
#     Write a CSV to a file pointer.  Used to support stdout.
#     :param fp: File pointer.  Could be stdout.
#     :param data: data to be written
#     :param delimiter: field delimiter for output
#     :return:
#     """
#     assert(len(data.keys()) > 0)
# 
#     # create a list of var_names from the first row
#     var_names = data[data.keys()[0]].keys()
#     fp.write(delimiter.join(var_names) + '\n')
# 
#     for key in sorted(data.keys()):
#         fp.write(delimiter.join([data[key][x] for x in var_names]) + '\n')
# 
# Path: pump/vivopump.py
# class DefNotFoundException(Exception):
# class InvalidDefException(Exception):
# class InvalidSourceException(Exception):
# class PathLengthException(Exception):
# class UnicodeCsvReader(object):
# class UnicodeDictReader(csv.DictReader):
#     class RowError(Exception):
#     def __init__(self, value):
#     def __str__(self):
#     def __init__(self, value):
#     def __str__(self):
#     def __init__(self, value):
#     def __str__(self):
#     def __init__(self, value):
#     def __str__(self):
#     def __init__(self, f, encoding="utf-8", **kwargs):
#     def __iter__(self):
#     def next(self):
#     def line_num(self):
#     def __init__(self, f, encoding="utf-8", fieldnames=None, **kwds):
# def read_csv(filename, skip=True, delimiter='|'):
# def read_csv_fp(fp, skip=True, delimiter="|"):
# def write_csv_fp(fp, data, delimiter='|'):
# def write_csv(filename, data, delimiter='|'):
# def replace_initials(s):
#     def repl_function(m):
# def key_string(s):
# def get_vivo_types(selector, parms, separator=';'):
# def get_vivo_ufid(parms):
# def get_vivo_publishers(parms):
# def get_vivo_journals(parms):
# def get_vivo_ccn(parms):
# def get_vivo_sponsorid(parms):
# def get_vivo_authors(parms):
# def get_vivo_positions(parms):
# def read_update_def(filename, prefix):
#     def make_prefix_dict(prefix_text):
#     def cast_to_rdflib(t):
#     def fixit(current_object, prefix_dictionary):
#     def add_order(a, b):
#     def validate_update_def(a):
#     def add_object_names_and_step_attributes(a):
# def add_qualifiers(input_path):
# def gather_types(input_step, varname):
# def make_update_query(entity_sparql, path):
# def make_rdf_term(row_term):
# def get_graph(update_def, query_parms):
# def new_uri(parms):
# def vivo_query(query, parms):
# def write_update_def(update_def, filename):
# def parse_pages(pages):
# def parse_date_parts(month, year):
# def get_args():
# def get_parms():
# def add_type_restriction(step):
# def make_get_query(update_def):
# def unique_path(path):
# def make_get_data(update_def, result_set):
# def make_rdf_term_from_source(value, step):
# def prepare_column_values(update_string, intra, step_def, enum, row, column_name):
# def load_enum(update_def):
# def create_enum(filename, query, parms, trim=0, skip=0):


# Path: uf_examples/publications/filters/pub_columns_filter.py
import sys
from pump.vivopump import read_csv_fp, write_csv_fp
from pump.vivopump import improve_title, parse_pages, parse_date_parts
from disambiguate.utils import print_err

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
pub_columns_filter.py -- add needed columns, remove unused columns
"""

__author__ = "Michael Conlon"
__copyright__ = "Copyright 2015 (c) Michael Conlon"
__license__ = "New BSD License"
__version__ = "0.01"




data_in = read_csv_fp(sys.stdin)