<commit_before>from os import remove, listdir
from os.path import join, isfile, basename
from copy import deepcopy
import warnings
from unittest.mock import patch, MagicMock
import datetime


import numpy as np
import pandas as pd
import re
import pytest
import f90nml
from openscm.scmdataframe.base import ScmDataFrameBase


import pymagicc.definitions
from pymagicc import MAGICC6
from pymagicc.io import (
    MAGICCData,
    _Reader,
    _ConcInReader,
    _ScenWriter,
    read_cfg_file,
    get_special_scen_code,
    NoReaderWriterError,
    InvalidTemporalResError,
    pull_cfg_from_parameters_out_file,
    pull_cfg_from_parameters_out,
    get_generic_rcp_name,
    determine_tool,
)
from .conftest import (
    MAGICC6_DIR,
    TEST_DATA_DIR,
    TEST_OUT_DIR,
    EXPECTED_FILES_DIR,
    run_writing_comparison,
)


# Not all files can be read in
TEST_OUT_FILES = listdir(TEST_OUT_DIR)

INVALID_OUT_FILES = [
    r"CARBONCYCLE.*OUT",
    r".*SUBANN.*OUT",
    r"DAT_VOLCANIC_RF\..*OUT",
    r"PF.*OUT",
    r"DATBASKET_.*",
    r"PRECIPINPUT.*OUT",
    r"TEMP_OCEANLAYERS.*\.BINOUT",
    r"INVERSEEMIS\.BINOUT",
    r".*INVERSE\_.*EMIS.*OUT",
    r"TIMESERIESMIX.*OUT",
    r"SUMMARY_INDICATORS.OUT",
]


def generic_mdata_tests(mdata, include_todo=True):
    """Resusable tests to ensure data format"""
    assert mdata.is_loaded == True

    assert isinstance(mdata, ScmDataFrameBase)
    index = ["model", "scenario", "region", "variable", "unit", "climate_model"]
    if include_todo:
    	index += ["todo"]
    pd.testing.assert_index_equal(
        mdata.meta.columns,
        pd.Index(index),
    )

    assert mdata["variable"].dtype == "object"
    assert mdata["todo"].dtype == "object"
    assert mdata["unit"].dtype == "object"
    assert mdata["region"].dtype == "object"
    assert mdata["scenario"].dtype == "object"
    assert mdata["model"].dtype == "object"
    assert mdata["climate_model"].dtype == "object"

    for key in ["units", "unit", "firstdatarow", "dattype"]:
        with pytest.raises(KeyError):
            mdata.metadata[key]
    assert isinstance(mdata.metadata["header"], str)


def assert_mdata_value(mdata, value, **kwargs):
    res = mdata.filter(**kwargs)
    assert len(res) == 1
    if value < 0.1:
        np.testing.assert_allclose(res.timeseries().iloc[0], value, rtol=1e-4)
    else:
        np.testing.assert_allclose(res.timeseries().iloc[0], value)


def test_cant_find_reader_writer():
    test_name = "HISTRCP_CO2I_EMIS.txt"

    expected_message = (
        r"^"
        + re.escape("Couldn't find appropriate reader for {}.".format(test_name))
        + r"\n"
        + re.escape(
            "The file must be one "
            "of the following types and the filepath must match its "
            "corresponding regular expression:"
        )
        + r"(\n.*)*"  # dicts aren't ordered in Python3.5
        + re.escape("SCEN: ^.*\\.SCEN$")
        + r"(\n.*)*$"
    )

    with pytest.raises(NoReaderWriterError, match=expected_message):
        determine_tool(join(TEST_DATA_DIR, test_name), "reader")

    expected_message = expected_message.replace("reader", "writer")
    with pytest.raises(NoReaderWriterError, match=expected_message):
        determine_tool(join(TEST_DATA_DIR, test_name), "writer")


def test_get_invalid_tool():
    junk_tool = "junk tool"
    expected_error_msg = (
        r"^\"?"
        + re.escape(
            "MAGICCData does not know how to get a {}, "
            "valid options are:".format(junk_tool)
        )
        + r".*\"?$"
    )
    with pytest.raises(KeyError, match=expected_error_msg):
        determine_tool("EXAMPLE.SCEN", junk_tool)


def test_load_magicc6_emis():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_CO2I_EMIS.IN"))
    assert mdata.is_loaded == True
    generic_mdata_tests(mdata)

    assert_mdata_value(
        mdata,
        1.7682027e000,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|R5ASIA",
        year=2000,
        unit="Gt C / yr",
        todo="SET",
    )


def test_load_magicc6_emis_hyphen_in_units():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_N2OI_EMIS.IN"))
    generic_mdata_tests(mdata)

    assert_mdata_value(
        mdata,
        0.288028519,
        variable="Emissions|N2O|MAGICC Fossil and Industrial",
        region="World|R5ASIA",
        year=2000,
        unit="Mt N2ON / yr",
        todo="SET",
    )


def test_load_magicc5_emis():
    mdata = MAGICCData(join(MAGICC6_DIR, "MARLAND_CO2I_EMIS.IN"))
    generic_mdata_tests(mdata)

    assert_mdata_value(
        mdata,
        6.20403698,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|Northern Hemisphere",
        year=2000,
        unit="Gt C / yr",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.495812385,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|Southern Hemisphere",
        year=2002,
        unit="Gt C / yr",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.0,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|Southern Hemisphere",
        year=1751,
        unit="Gt C / yr",
        todo="SET",
    )


def test_load_magicc5_emis_not_renamed_error():
    test_path = TEST_DATA_DIR
    test_name = "MARLAND_CO2_EMIS_FOSSIL&IND.IN"

    expected_error_msg = re.escape(
        "Cannot determine variable from filepath: {}".format(join(test_path, test_name))
    )
    with pytest.raises(ValueError, match=expected_error_msg):
        MAGICCData(join(test_path, test_name))


def test_load_magicc6_conc():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_CO2_CONC.IN"))

    assert (mdata["unit"] == "ppm").all()
    generic_mdata_tests(mdata)
    assert_mdata_value(
        mdata,
        2.80435733e002,
        variable="Atmospheric Concentrations|CO2",
        region="World",
        year=1048,
        unit="ppm",
        todo="SET",
    )


def test_load_magicc6_conc_old_style_name_umlaut_metadata():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_HFC245fa_CONC.IN"))

    assert (mdata["unit"] == "ppt").all()
    assert mdata.metadata["data"] == "Global average mixing ratio"
    generic_mdata_tests(mdata)
    assert_mdata_value(
        mdata,
        0.0,
        variable="Atmospheric Concentrations|HFC245fa",
        region="World",
        year=2000,
        unit="ppt",
        todo="SET",
    )


def test_load_magicc6_conc_old_style_name_with_hyphen():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_HFC43-10_CONC.IN"))

    assert (mdata["unit"] == "ppt").all()
    generic_mdata_tests(mdata)
    assert_mdata_value(
        mdata,
        0.0,
        variable="Atmospheric Concentrations|HFC4310",
        region="World",
        year=2000,
        unit="ppt",
        todo="SET",
    )


def test_load_magicc7_emis_umlaut_metadata():
    mdata = MAGICCData(join(TEST_DATA_DIR, "HISTSSP_CO2I_EMIS.IN"))

    generic_mdata_tests(mdata)
    assert (
        mdata.metadata["contact"]
        == "<NAME>, Australian-German Climate and Energy College, University of Melbourne, <EMAIL>"
    )
    assert mdata.metadata["description"] == "Test line by näme with ümlauts ëh ça"
    assert (mdata["unit"] == "Gt C / yr").all()

    assert_mdata_value(
        mdata,
        0.6638,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|R6REF",
        year=2013,
        unit="Gt C / yr",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        1.6911,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|R6ASIA",
        year=2000,
        unit="Gt C / yr",
        todo="SET",
    )


def test_load_ot():
    mdata = MAGICCData(join(MAGICC6_DIR, "MIXED_NOXI_OT.IN"))

    generic_mdata_tests(mdata)

    assert mdata.metadata["data"] == "Optical Thickness"
    assert (
        mdata.metadata["description"]
        == "the land/ocean ratio of optical depth of NOXI is scaled with the hemispheric EDGAR NOXI emissions. NOXI opt. depth as available on http://www.giss.nasa.gov/data/simodel/trop.aer/"
    )
    assert (
        mdata.metadata["source"]
        == "Mixed: EDGAR: www.mnp.nl; NASA-GISS: http://data.giss.nasa.gov/"
    )
    assert (
        mdata.metadata["compiled by"]
        == "<NAME>, Lauder NZ, NCAR/PIK, <EMAIL>"
    )
    assert mdata.metadata["date"] == "18-Jul-2006 11:02:48"
    assert mdata.metadata["unit normalisation"] == "Normalized to 1 in year 2000"

    assert (mdata["unit"] == "dimensionless").all()
    assert (mdata["todo"] == "SET").all()
    assert (
        mdata["variable"] == "Optical Thickness|NOx|MAGICC Fossil and Industrial"
    ).all()

    assert_mdata_value(
        mdata,
        0.00668115649,
        variable="Optical Thickness|NOx|MAGICC Fossil and Industrial",
        region="World|Northern Hemisphere|Ocean",
        year=1765,
        unit="dimensionless",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.526135104,
        variable="Optical Thickness|NOx|MAGICC Fossil and Industrial",
        region="World|Northern Hemisphere|Land",
        year=1865,
        unit="dimensionless",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.612718845,
        variable="Optical Thickness|NOx|MAGICC Fossil and Industrial",
        region="World|Southern Hemisphere|Ocean",
        year=1965,
        unit="dimensionless",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        3.70378,
        variable="Optical Thickness|NOx|MAGICC Fossil and Industrial",
        region="World|Southern Hemisphere|Land",
        year=2000,
        unit="dimensionless",
        todo="SET",
    )


def test_load_rf():
    mdata = MAGICCData(join(MAGICC6_DIR, "GISS_BCB_RF.IN"))

    generic_mdata_tests(mdata)

    assert mdata.metadata["data"] == "Radiative Forcing"
    assert (
        mdata.metadata["description"]
        == "BCB - Radiative Forcing of year 2000 (as provided on http://data.giss.nasa.gov/efficacy/) for four MAGICC boxes; scaled over time with optical thickness of file giss_bcb_ot"
    )
    assert (
        mdata.metadata["source"]
        == "Original-GISS-Description:  NET RADIA. AT   0 MB (W/M+2) ANN  E2BCBx6a-E2AarM20A (yr 1850->2000 dBCB*6 - adj"
    )
    assert (
        mdata.metadata["compiled by"]
        == "<NAME>, Lauder NZ, NCAR/PIK, <EMAIL>"
    )
    assert mdata.metadata["date"] == "18-Jul-2006 11:05:18"

    assert (mdata["unit"] == "W / m^2").all()
    assert (mdata["todo"] == "SET").all()
    assert (mdata["variable"] == "Radiative Forcing|BC|MAGICC AFOLU").all()

    assert_mdata_value(
        mdata,
        0.0,
        variable="Radiative Forcing|BC|MAGICC AFOLU",
        region="World|Northern Hemisphere|Ocean",
        year=1765,
        # unit="W / m^2",  # bug, can't use ^ in filter now as regexp means it looks for not, propose removing such behaviour in pyam based on usefulness of units and fact that complicated regexp can be re-enabled with regexp=True
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.268436597,
        variable="Radiative Forcing|BC|MAGICC AFOLU",
        region="World|Northern Hemisphere|Land",
        year=1865,
        # unit="W / m^2",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.443357552,
        variable="Radiative Forcing|BC|MAGICC AFOLU",
        region="World|Southern Hemisphere|Ocean",
        year=1965,
        # unit="W / m^2",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        1.53987244,
        variable="Radiative Forcing|BC|MAGICC AFOLU",
        region="World|Southern Hemisphere|Land",
        year=2000,
        # unit="W / m^2",
        todo="SET",
    )


def test_load_solar_rf():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP6SCP6to45_SOLAR_RF.IN"))

    generic_mdata_tests(mdata)

    assert mdata.metadata["data"] == "Radiative Forcing, kept constant after 2250"
    assert (
        mdata.metadata["description"]
        == "Solar irradiance by Lean et al. as recommended for CMIP5, as documented here: http://www.geo.fu-berlin.de/en/met/ag/strat/forschung/SOLARIS/Input_data/CMIP5_solar_irradiance.html - converted into radiative forcing by dividing by 4 (geometrical) and multiplying by 0.7 (albedo) effect. Furthermore, the data is normalized to have an average zero around 1750 (22 years around 1750)."
    )
    # second definition of metadata wins out
    assert (
        mdata.metadata["source"]
        == "RCP data as presented on http://www.iiasa.ac.at/web-apps/tnt/RcpDb, August 2009"
    )
    assert (
        mdata.metadata["compiled by"]
        == "<NAME>, <EMAIL>, www.primap.org"
    )
    assert mdata.metadata["date"] == "24-Jul-2009 17:05:30"

    assert (mdata["unit"] == "W / m^2").all()
    assert (mdata["todo"] == "SET").all()
    assert (mdata["variable"] == "Radiative Forcing|Solar").all()
    assert (mdata["region"] == "World").all()

    assert_mdata_value(
        mdata,
        0.0149792391,
        variable="Radiative Forcing|Solar",
        region="World",
        year=1610,
        # unit="W / m^2",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        -0.00160201087,
        variable="Radiative Forcing|Solar",
        year=1865,
        # unit="W / m^2",
    )

    assert_mdata_value(
        mdata,
        0.0652917391,
        variable="Radiative Forcing|Solar",
        year=1965,
        # unit="W / m^2",
    )

    assert_mdata_value(
        mdata,
        0.0446329891,
        variable="Radiative Forcing|Solar",
        year=2183,
        # unit="W / m^2",
    )

    assert_mdata_value(
        mdata,
        0.121325148,
        variable="Radiative Forcing|Solar",
        year=2600,
        # unit="W / m^2",
    )


@pytest.mark.xfail(
    reason="Not currently supporting subannual files because scmdataframe cannot handle merging annual and subannual timeseries"
)
def test_load_volcanic_rf():
    mdata = MAGICCData(join(MAGICC6_DIR, "HIST_VOLCANIC_RF.MON"))

    generic_mdata_tests(mdata)

    assert mdata.metadata["data"] == "Radiative Forcing"
    assert (
        mdata.metadata["description"]
        == "Monthly Volcanic radiative forcing - relative / unscaled - as in NASA GISS model, using a optical thickness to radiative forcing conversion of -23.5"
    )
    # second definition of metadata wins out
    assert mdata.metadata["source"] == "NASA-GISS: http://data.giss.nasa.gov/"
    assert (
        mdata.metadata["compiled by"]
        == "<NAME>, Lauder NZ, NCAR/PIK, <EMAIL>, manually extended by zero forcing from 2001 to 2006"
    )
    assert mdata.metadata["date"] == "15-Jun-2006 00:20:54"

    assert (mdata["unit"] == "W / m^2").all()
    assert (mdata["todo"] == "SET").all()
    assert (mdata["variable"] == "Radiative Forcing|Volcanic").all()

    assert_mdata_value(
        mdata,
        0.0,
        variable="Radiative Forcing|Volcanic",
        region="World|Northern Hemisphere|Land",
        year=1000,
        month=1,
        # unit="W / m^2",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        -0.0187500000,
        variable="Radiative Forcing|Volcanic",
        region="World|Northern Hemisphere|Land",
        year=1002,
        month=7,
        # unit="W / m^2",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.0,
        variable="Radiative Forcing|Volcanic",
        region="World|Northern Hemisphere|Ocean",
        year=1013,
        month=3,
        # unit="W / m^2",
        todo="SET",
    )<commit_msg>Sort out subannual reading tests<commit_after>from os import remove, listdir
from os.path import join, isfile, basename
from copy import deepcopy
import warnings
from unittest.mock import patch, MagicMock
import datetime


import numpy as np
import pandas as pd
import re
import pytest
import f90nml
from openscm.scmdataframe.base import ScmDataFrameBase


import pymagicc.definitions
from pymagicc import MAGICC6
from pymagicc.io import (
    MAGICCData,
    _Reader,
    _ConcInReader,
    _ScenWriter,
    read_cfg_file,
    get_special_scen_code,
    NoReaderWriterError,
    InvalidTemporalResError,
    pull_cfg_from_parameters_out_file,
    pull_cfg_from_parameters_out,
    get_generic_rcp_name,
    determine_tool,
)
from .conftest import (
    MAGICC6_DIR,
    TEST_DATA_DIR,
    TEST_OUT_DIR,
    EXPECTED_FILES_DIR,
    run_writing_comparison,
)


# Not all files can be read in
TEST_OUT_FILES = listdir(TEST_OUT_DIR)

INVALID_OUT_FILES = [
    r"CARBONCYCLE.*OUT",
    r".*SUBANN.*\.BINOUT",
    r"DAT_VOLCANIC_RF\.BINOUT",
    r"PF.*OUT",
    r"DATBASKET_.*",
    r"PRECIPINPUT.*OUT",
    r"TEMP_OCEANLAYERS.*\.BINOUT",
    r"INVERSEEMIS\.BINOUT",
    r".*INVERSE\_.*EMIS.*OUT",
    r"TIMESERIESMIX.*OUT",
    r"SUMMARY_INDICATORS.OUT",
]


def generic_mdata_tests(mdata, include_todo=True):
    """Resusable tests to ensure data format"""
    assert mdata.is_loaded == True

    assert isinstance(mdata, ScmDataFrameBase)
    index = ["model", "scenario", "region", "variable", "unit", "climate_model"]
    if include_todo:
    	index += ["todo"]
    pd.testing.assert_index_equal(
        mdata.meta.columns,
        pd.Index(index),
    )

    assert mdata["variable"].dtype == "object"
    assert mdata["todo"].dtype == "object"
    assert mdata["unit"].dtype == "object"
    assert mdata["region"].dtype == "object"
    assert mdata["scenario"].dtype == "object"
    assert mdata["model"].dtype == "object"
    assert mdata["climate_model"].dtype == "object"

    for key in ["units", "unit", "firstdatarow", "dattype"]:
        with pytest.raises(KeyError):
            mdata.metadata[key]
    assert isinstance(mdata.metadata["header"], str)


def assert_mdata_value(mdata, value, **kwargs):
    res = mdata.filter(**kwargs)
    assert len(res) == 1
    if value < 0.1:
        np.testing.assert_allclose(res.timeseries().iloc[0], value, rtol=1e-4)
    else:
        np.testing.assert_allclose(res.timeseries().iloc[0], value)


def test_cant_find_reader_writer():
    test_name = "HISTRCP_CO2I_EMIS.txt"

    expected_message = (
        r"^"
        + re.escape("Couldn't find appropriate reader for {}.".format(test_name))
        + r"\n"
        + re.escape(
            "The file must be one "
            "of the following types and the filepath must match its "
            "corresponding regular expression:"
        )
        + r"(\n.*)*"  # dicts aren't ordered in Python3.5
        + re.escape("SCEN: ^.*\\.SCEN$")
        + r"(\n.*)*$"
    )

    with pytest.raises(NoReaderWriterError, match=expected_message):
        determine_tool(join(TEST_DATA_DIR, test_name), "reader")

    expected_message = expected_message.replace("reader", "writer")
    with pytest.raises(NoReaderWriterError, match=expected_message):
        determine_tool(join(TEST_DATA_DIR, test_name), "writer")


def test_get_invalid_tool():
    junk_tool = "junk tool"
    expected_error_msg = (
        r"^\"?"
        + re.escape(
            "MAGICCData does not know how to get a {}, "
            "valid options are:".format(junk_tool)
        )
        + r".*\"?$"
    )
    with pytest.raises(KeyError, match=expected_error_msg):
        determine_tool("EXAMPLE.SCEN", junk_tool)


def test_load_magicc6_emis():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_CO2I_EMIS.IN"))
    assert mdata.is_loaded == True
    generic_mdata_tests(mdata)

    assert_mdata_value(
        mdata,
        1.7682027e000,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|R5ASIA",
        year=2000,
        unit="Gt C / yr",
        todo="SET",
    )


def test_load_magicc6_emis_hyphen_in_units():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_N2OI_EMIS.IN"))
    generic_mdata_tests(mdata)

    assert_mdata_value(
        mdata,
        0.288028519,
        variable="Emissions|N2O|MAGICC Fossil and Industrial",
        region="World|R5ASIA",
        year=2000,
        unit="Mt N2ON / yr",
        todo="SET",
    )


def test_load_magicc5_emis():
    mdata = MAGICCData(join(MAGICC6_DIR, "MARLAND_CO2I_EMIS.IN"))
    generic_mdata_tests(mdata)

    assert_mdata_value(
        mdata,
        6.20403698,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|Northern Hemisphere",
        year=2000,
        unit="Gt C / yr",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.495812385,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|Southern Hemisphere",
        year=2002,
        unit="Gt C / yr",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.0,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|Southern Hemisphere",
        year=1751,
        unit="Gt C / yr",
        todo="SET",
    )


def test_load_magicc5_emis_not_renamed_error():
    test_path = TEST_DATA_DIR
    test_name = "MARLAND_CO2_EMIS_FOSSIL&IND.IN"

    expected_error_msg = re.escape(
        "Cannot determine variable from filepath: {}".format(join(test_path, test_name))
    )
    with pytest.raises(ValueError, match=expected_error_msg):
        MAGICCData(join(test_path, test_name))


def test_load_magicc6_conc():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_CO2_CONC.IN"))

    assert (mdata["unit"] == "ppm").all()
    generic_mdata_tests(mdata)
    assert_mdata_value(
        mdata,
        2.80435733e002,
        variable="Atmospheric Concentrations|CO2",
        region="World",
        year=1048,
        unit="ppm",
        todo="SET",
    )


def test_load_magicc6_conc_old_style_name_umlaut_metadata():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_HFC245fa_CONC.IN"))

    assert (mdata["unit"] == "ppt").all()
    assert mdata.metadata["data"] == "Global average mixing ratio"
    generic_mdata_tests(mdata)
    assert_mdata_value(
        mdata,
        0.0,
        variable="Atmospheric Concentrations|HFC245fa",
        region="World",
        year=2000,
        unit="ppt",
        todo="SET",
    )


def test_load_magicc6_conc_old_style_name_with_hyphen():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_HFC43-10_CONC.IN"))

    assert (mdata["unit"] == "ppt").all()
    generic_mdata_tests(mdata)
    assert_mdata_value(
        mdata,
        0.0,
        variable="Atmospheric Concentrations|HFC4310",
        region="World",
        year=2000,
        unit="ppt",
        todo="SET",
    )


def test_load_magicc7_emis_umlaut_metadata():
    mdata = MAGICCData(join(TEST_DATA_DIR, "HISTSSP_CO2I_EMIS.IN"))

    generic_mdata_tests(mdata)
    assert (
        mdata.metadata["contact"]
        == "<NAME>, Australian-German Climate and Energy College, University of Melbourne, <EMAIL>"
    )
    assert mdata.metadata["description"] == "Test line by näme with ümlauts ëh ça"
    assert (mdata["unit"] == "Gt C / yr").all()

    assert_mdata_value(
        mdata,
        0.6638,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|R6REF",
        year=2013,
        unit="Gt C / yr",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        1.6911,
        variable="Emissions|CO2|MAGICC Fossil and Industrial",
        region="World|R6ASIA",
        year=2000,
        unit="Gt C / yr",
        todo="SET",
    )


def test_load_ot():
    mdata = MAGICCData(join(MAGICC6_DIR, "MIXED_NOXI_OT.IN"))

    generic_mdata_tests(mdata)

    assert mdata.metadata["data"] == "Optical Thickness"
    assert (
        mdata.metadata["description"]
        == "the land/ocean ratio of optical depth of NOXI is scaled with the hemispheric EDGAR NOXI emissions. NOXI opt. depth as available on http://www.giss.nasa.gov/data/simodel/trop.aer/"
    )
    assert (
        mdata.metadata["source"]
        == "Mixed: EDGAR: www.mnp.nl; NASA-GISS: http://data.giss.nasa.gov/"
    )
    assert (
        mdata.metadata["compiled by"]
        == "<NAME>, Lauder NZ, NCAR/PIK, <EMAIL>"
    )
    assert mdata.metadata["date"] == "18-Jul-2006 11:02:48"
    assert mdata.metadata["unit normalisation"] == "Normalized to 1 in year 2000"

    assert (mdata["unit"] == "dimensionless").all()
    assert (mdata["todo"] == "SET").all()
    assert (
        mdata["variable"] == "Optical Thickness|NOx|MAGICC Fossil and Industrial"
    ).all()

    assert_mdata_value(
        mdata,
        0.00668115649,
        variable="Optical Thickness|NOx|MAGICC Fossil and Industrial",
        region="World|Northern Hemisphere|Ocean",
        year=1765,
        unit="dimensionless",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.526135104,
        variable="Optical Thickness|NOx|MAGICC Fossil and Industrial",
        region="World|Northern Hemisphere|Land",
        year=1865,
        unit="dimensionless",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.612718845,
        variable="Optical Thickness|NOx|MAGICC Fossil and Industrial",
        region="World|Southern Hemisphere|Ocean",
        year=1965,
        unit="dimensionless",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        3.70378,
        variable="Optical Thickness|NOx|MAGICC Fossil and Industrial",
        region="World|Southern Hemisphere|Land",
        year=2000,
        unit="dimensionless",
        todo="SET",
    )


def test_load_rf():
    mdata = MAGICCData(join(MAGICC6_DIR, "GISS_BCB_RF.IN"))

    generic_mdata_tests(mdata)

    assert mdata.metadata["data"] == "Radiative Forcing"
    assert (
        mdata.metadata["description"]
        == "BCB - Radiative Forcing of year 2000 (as provided on http://data.giss.nasa.gov/efficacy/) for four MAGICC boxes; scaled over time with optical thickness of file giss_bcb_ot"
    )
    assert (
        mdata.metadata["source"]
        == "Original-GISS-Description:  NET RADIA. AT   0 MB (W/M+2) ANN  E2BCBx6a-E2AarM20A (yr 1850->2000 dBCB*6 - adj"
    )
    assert (
        mdata.metadata["compiled by"]
        == "<NAME>, Lauder NZ, NCAR/PIK, malte.mein<EMAIL>"
    )
    assert mdata.metadata["date"] == "18-Jul-2006 11:05:18"

    assert (mdata["unit"] == "W / m^2").all()
    assert (mdata["todo"] == "SET").all()
    assert (mdata["variable"] == "Radiative Forcing|BC|MAGICC AFOLU").all()

    assert_mdata_value(
        mdata,
        0.0,
        variable="Radiative Forcing|BC|MAGICC AFOLU",
        region="World|Northern Hemisphere|Ocean",
        year=1765,
        # unit="W / m^2",  # bug, can't use ^ in filter now as regexp means it looks for not, propose removing such behaviour in pyam based on usefulness of units and fact that complicated regexp can be re-enabled with regexp=True
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.268436597,
        variable="Radiative Forcing|BC|MAGICC AFOLU",
        region="World|Northern Hemisphere|Land",
        year=1865,
        # unit="W / m^2",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.443357552,
        variable="Radiative Forcing|BC|MAGICC AFOLU",
        region="World|Southern Hemisphere|Ocean",
        year=1965,
        # unit="W / m^2",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        1.53987244,
        variable="Radiative Forcing|BC|MAGICC AFOLU",
        region="World|Southern Hemisphere|Land",
        year=2000,
        # unit="W / m^2",
        todo="SET",
    )


def test_load_solar_rf():
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP6SCP6to45_SOLAR_RF.IN"))

    generic_mdata_tests(mdata)

    assert mdata.metadata["data"] == "Radiative Forcing, kept constant after 2250"
    assert (
        mdata.metadata["description"]
        == "Solar irradiance by Lean et al. as recommended for CMIP5, as documented here: http://www.geo.fu-berlin.de/en/met/ag/strat/forschung/SOLARIS/Input_data/CMIP5_solar_irradiance.html - converted into radiative forcing by dividing by 4 (geometrical) and multiplying by 0.7 (albedo) effect. Furthermore, the data is normalized to have an average zero around 1750 (22 years around 1750)."
    )
    # second definition of metadata wins out
    assert (
        mdata.metadata["source"]
        == "RCP data as presented on http://www.iiasa.ac.at/web-apps/tnt/RcpDb, August 2009"
    )
    assert (
        mdata.metadata["compiled by"]
        == "<NAME>, <EMAIL>, www.primap.org"
    )
    assert mdata.metadata["date"] == "24-Jul-2009 17:05:30"

    assert (mdata["unit"] == "W / m^2").all()
    assert (mdata["todo"] == "SET").all()
    assert (mdata["variable"] == "Radiative Forcing|Solar").all()
    assert (mdata["region"] == "World").all()

    assert_mdata_value(
        mdata,
        0.0149792391,
        variable="Radiative Forcing|Solar",
        region="World",
        year=1610,
        # unit="W / m^2",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        -0.00160201087,
        variable="Radiative Forcing|Solar",
        year=1865,
        # unit="W / m^2",
    )

    assert_mdata_value(
        mdata,
        0.0652917391,
        variable="Radiative Forcing|Solar",
        year=1965,
        # unit="W / m^2",
    )

    assert_mdata_value(
        mdata,
        0.0446329891,
        variable="Radiative Forcing|Solar",
        year=2183,
        # unit="W / m^2",
    )

    assert_mdata_value(
        mdata,
        0.121325148,
        variable="Radiative Forcing|Solar",
        year=2600,
        # unit="W / m^2",
    )


def test_load_volcanic_rf():
    # test that starting with an annual file doesn't make things blow up
    mdata = MAGICCData(join(MAGICC6_DIR, "HISTRCP_CO2_CONC.IN"))
    mdata = mdata.append(join(MAGICC6_DIR, "HIST_VOLCANIC_RF.MON"))

    generic_mdata_tests(mdata)

    assert mdata.metadata["data"] == "Radiative Forcing"
    assert (
        mdata.metadata["description"]
        == "Monthly Volcanic radiative forcing - relative / unscaled - as in NASA GISS model, using a optical thickness to radiative forcing conversion of -23.5"
    )
    # second definition of metadata wins out
    assert mdata.metadata["source"] == "NASA-GISS: http://data.giss.nasa.gov/"
    assert (
        mdata.metadata["compiled by"]
        == "<NAME>, Lauder NZ, NCAR/PIK, <EMAIL>, manually extended by zero forcing from 2001 to 2006"
    )
    assert mdata.metadata["date"] == "15-Jun-2006 00:20:54"

    assert (mdata.filter(variable="*Forcing*")["unit"] == "W / m^2").all()
    assert (mdata["todo"] == "SET").all()
    assert (mdata.filter(variable="*Forcing*")["variable"] == "Radiative Forcing|Volcanic").all()

    assert_mdata_value(
        mdata,
        0.0,
        variable="Radiative Forcing|Volcanic",
        region="World|Northern Hemisphere|Land",
        year=1000,
        month=1,
        # unit="W / m^2",  # TODO: fix pyam filtering with / and ^
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        -0.0187500000,
        variable="Radiative Forcing|Volcanic",
        region="World|Northern Hemisphere|Land",
        year=1002,
        month=7,
        # unit="W / m^2",
        todo="SET",
    )

    assert_mdata_value(
        mdata,
        0.0,
        variable="Radiative Forcing|Volcanic",
        region="World|Northern Hemisphere|Ocean",
        year=1013,
        month=3,
        # unit="W / m^2",
        todo="SET",
    )
{'fraction_non_alphanumeric': 0.12815786964949633, 'fraction_numerical': 0.03215863230480791, 'mean_word_length': 2.6828554710356936, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4286527', 'n_tokens_mistral': 11695, 'n_tokens_neox': 10781, 'n_words': 2085}