# Path: cis/data_io/aeronet.py
# def load_aeronet(filename, variables=None):
#     """
#     Loads aeronet csv file.
# 
#     :param filename: data file name
#     :param variables: A list of variables to return
#     :return: A dictionary of variables names and numpy arrays containing the data for that variable
#     """
#     from cis.exceptions import InvalidVariableError
#     from cis.time_util import cis_standard_time_unit
#     from numpy.ma import masked_invalid
#     from pandas import read_csv, to_datetime
# 
#     version = get_aeronet_version(filename)
#     ordered_vars = get_aeronet_file_variables(filename, version)
#     if len(ordered_vars) == 0:
#         return {}
# 
#     # Load all available geolocation information and any requested variables
#     cols = [var for var in ("date", "time", "latitude", "longitude", "altitude") if var in ordered_vars]
#     if cols is not None and variables is not None:
#         cols.extend(variables)
# 
#     dtypes = {var:'str' if var in ("date", "time") else "float" for var in cols}
# 
#     try:
#         rawd = read_csv(filename, sep=",", header=AERONET_HEADER_LENGTH[version]-1, names=ordered_vars,
#                         index_col=False, usecols=cols, na_values=AERONET_MISSING_VALUE[version], dtype=dtypes,
#                         parse_dates={"datetime":["date", "time"]}, infer_datetime_format=True, dayfirst=True,
#                         error_bad_lines=False, warn_bad_lines=True, #low_memory="All_Sites_Times_All_Points" in filename
#         )
#     except ValueError:
#         raise InvalidVariableError("{} not available in {}".format(variables, filename))
# 
#     # Empty file
#     if rawd.shape[0] == 0:
#         return {"datetime":[], "latitude":[], "longitude":[], "altitude":[]}
# 
#     # Convert pandas Timestamps into CIS standard numbers
#     rawd["datetime"] = [cis_standard_time_unit.date2num(timestamp.to_pydatetime())
#                         for timestamp in to_datetime(rawd["datetime"], format='%d:%m:%Y %H:%M:%S')]
# 
#     # Add position metadata that isn't listed in every line for some formats
#     if version.startswith("MAN"):
#         rawd["altitude"] = 0.
# 
#     elif version.endswith("2"):
#         metadata = get_file_metadata(filename)
#         rawd["longitude"] = float(metadata.misc[2][1].split("=")[1])
#         rawd["latitude"] = float(metadata.misc[2][2].split("=")[1])
#         rawd["altitude"] = float(metadata.misc[2][3].split("=")[1])
# 
#     return {var : masked_invalid(arr) for var, arr in rawd.items()}
# 
# Path: cis/exceptions.py
# class InvalidVariableError(CISError):
#     pass


import unittest
from cis.test.integration_test_data import valid_aeronet_filename, valid_aeronet_variable
from cis.data_io.aeronet import load_aeronet
from nose.tools import assert_almost_equal, raises, assert_equal
from cis.exceptions import InvalidVariableError
        from cf_units import Unit
        from cis import read_data_list
        from datetime import datetime
        from cis.time_util import cis_standard_time_unit as ct



class TestAeronet(unittest.TestCase):

    def test_aeronet_variable_name_parsing(self):

        aeronet_data = read_data_list(valid_aeronet_filename, ["AOT_440", "Water(cm)"])

        assert_equal(aeronet_data[0].name(), "AOT_440")
        assert_equal(aeronet_data[0].units, Unit('1'))
        assert_equal(aeronet_data[1].name(), "Water(cm)")
        assert_equal(aeronet_data[1].var_name, "Water")
        assert_equal(aeronet_data[1].units, Unit('cm'))

    def test_aeronet_time_parsing(self):
        # 1.8s

        aeronet_data = load_aeronet(valid_aeronet_filename, [valid_aeronet_variable])

        assert_almost_equal(aeronet_data['datetime'][0], ct.date2num(datetime(2003, 9, 25, 6, 47, 9)))
        assert_almost_equal(aeronet_data['datetime'][5], ct.date2num(datetime(2003, 9, 25, 7, 10, 37)))
        assert_almost_equal(aeronet_data['datetime'][76], ct.date2num(datetime(2003, 9, 27, 13, 28, 2)))

    @raises(InvalidVariableError)
    def test_aeronet_missing_variable(self):

# Path: cis/data_io/aeronet.py
# def load_aeronet(filename, variables=None):
#     """
#     Loads aeronet csv file.
# 
#     :param filename: data file name
#     :param variables: A list of variables to return
#     :return: A dictionary of variables names and numpy arrays containing the data for that variable
#     """
#     from cis.exceptions import InvalidVariableError
#     from cis.time_util import cis_standard_time_unit
#     from numpy.ma import masked_invalid
#     from pandas import read_csv, to_datetime
# 
#     version = get_aeronet_version(filename)
#     ordered_vars = get_aeronet_file_variables(filename, version)
#     if len(ordered_vars) == 0:
#         return {}
# 
#     # Load all available geolocation information and any requested variables
#     cols = [var for var in ("date", "time", "latitude", "longitude", "altitude") if var in ordered_vars]
#     if cols is not None and variables is not None:
#         cols.extend(variables)
# 
#     dtypes = {var:'str' if var in ("date", "time") else "float" for var in cols}
# 
#     try:
#         rawd = read_csv(filename, sep=",", header=AERONET_HEADER_LENGTH[version]-1, names=ordered_vars,
#                         index_col=False, usecols=cols, na_values=AERONET_MISSING_VALUE[version], dtype=dtypes,
#                         parse_dates={"datetime":["date", "time"]}, infer_datetime_format=True, dayfirst=True,
#                         error_bad_lines=False, warn_bad_lines=True, #low_memory="All_Sites_Times_All_Points" in filename
#         )
#     except ValueError:
#         raise InvalidVariableError("{} not available in {}".format(variables, filename))
# 
#     # Empty file
#     if rawd.shape[0] == 0:
#         return {"datetime":[], "latitude":[], "longitude":[], "altitude":[]}
# 
#     # Convert pandas Timestamps into CIS standard numbers
#     rawd["datetime"] = [cis_standard_time_unit.date2num(timestamp.to_pydatetime())
#                         for timestamp in to_datetime(rawd["datetime"], format='%d:%m:%Y %H:%M:%S')]
# 
#     # Add position metadata that isn't listed in every line for some formats
#     if version.startswith("MAN"):
#         rawd["altitude"] = 0.
# 
#     elif version.endswith("2"):
#         metadata = get_file_metadata(filename)
#         rawd["longitude"] = float(metadata.misc[2][1].split("=")[1])
#         rawd["latitude"] = float(metadata.misc[2][2].split("=")[1])
#         rawd["altitude"] = float(metadata.misc[2][3].split("=")[1])
# 
#     return {var : masked_invalid(arr) for var, arr in rawd.items()}
# 
# Path: cis/exceptions.py
# class InvalidVariableError(CISError):
#     pass


# Path: cis/test/integration/test_io/test_aeronet.py
import unittest
from cis.test.integration_test_data import valid_aeronet_filename, valid_aeronet_variable
from cis.data_io.aeronet import load_aeronet
from nose.tools import assert_almost_equal, raises, assert_equal
from cis.exceptions import InvalidVariableError
        from cf_units import Unit
        from cis import read_data_list
        from datetime import datetime
        from cis.time_util import cis_standard_time_unit as ct




class TestAeronet(unittest.TestCase):

    def test_aeronet_variable_name_parsing(self):

        aeronet_data = read_data_list(valid_aeronet_filename, ["AOT_440", "Water(cm)"])

        assert_equal(aeronet_data[0].name(), "AOT_440")
        assert_equal(aeronet_data[0].units, Unit('1'))
        assert_equal(aeronet_data[1].name(), "Water(cm)")
        assert_equal(aeronet_data[1].var_name, "Water")
        assert_equal(aeronet_data[1].units, Unit('cm'))

    def test_aeronet_time_parsing(self):
        # 1.8s

        aeronet_data = load_aeronet(valid_aeronet_filename, [valid_aeronet_variable])

        assert_almost_equal(aeronet_data['datetime'][0], ct.date2num(datetime(2003, 9, 25, 6, 47, 9)))
        assert_almost_equal(aeronet_data['datetime'][5], ct.date2num(datetime(2003, 9, 25, 7, 10, 37)))
        assert_almost_equal(aeronet_data['datetime'][76], ct.date2num(datetime(2003, 9, 27, 13, 28, 2)))

    @raises(InvalidVariableError)
    def test_aeronet_missing_variable(self):

        aeronet_data = load_aeronet(valid_aeronet_filename, ['invalid_variable'])