<commit_before>'''
Collection of tests for orbital correction.

Created on 31/3/13

.. codeauthor:: <NAME>, <NAME>
'''

import unittest
from os.path import join
from itertools import product
import os
import numpy as np
import tempfile
from numpy.linalg import pinv, inv
from numpy import nan, isnan, array
from numpy import empty, dot, concatenate, float32
from numpy.testing import assert_array_equal, assert_array_almost_equal

from pyrate import algorithm
from pyrate.shared import Ifg
from pyrate.orbital import OrbitalError, orbital_correction
from pyrate.orbital import get_design_matrix, get_network_design_matrix, get_num_params
from pyrate.orbital import INDEPENDENT_METHOD, NETWORK_METHOD, PLANAR, QUADRATIC, PART_CUBIC
from common import sydney5_mock_ifgs, MockIfg, SYD_TEST_TIF, sydney5_ifgs
from scipy.linalg import lstsq
from pyrate.tests.common import SYD_TEST_MATLAB_ORBITAL_DIR, TEMPDIR
from pyrate.tests.common import SYD_TEST_DIR
from pyrate import config as cf
from pyrate.tests.common import SYD_TEST_TIF, sydney_data_setup
from pyrate.tests.common import sydney_data_setup_ifg_file_list
import shutil
from pyrate.tests import common
from pyrate.shared import nanmedian
from pyrate.scripts import run_pyrate

DEG_LOOKUP = {
    2: PLANAR,
    5: QUADRATIC,
    6: PART_CUBIC}

NUM_COEF_LOOKUP = {
    PLANAR: 2,
    QUADRATIC: 5,
    PART_CUBIC: 6}


class SingleDesignMatrixTests(unittest.TestCase):
    """
    Tests to verify correctness of basic planar & quadratic design matrices or
    DMs. This class serves two purposes, ensuring the independent method DMs are
    produced correctly. Secondly, these indivdual DMs are subsets of the larger
    DM 'grid' required for the networked orbital correction method.
    """

    def setUp(self):
        # faked cell sizes
        self.xs = 0.75
        self.ys = 0.8
        self.ifg = Ifg(join(SYD_TEST_TIF, 'geo_060619-061002.tif'))
        self.ifg.open()
        self.ifg.nodata_value = 0

        self.m = MockIfg(self.ifg, 3, 4)
        self.m.x_size = self.xs
        self.m.y_size = self.ys

    # tests for planar model

    def test_create_planar_dm(self):
        offset = False
        act = get_design_matrix(self.m, PLANAR, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 2))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, PLANAR, offset)
        assert_array_equal(act, exp)

    def test_create_planar_dm_offsets(self):
        offset = True
        act = get_design_matrix(self.m, PLANAR, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 3))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, PLANAR, offset)
        assert_array_almost_equal(act, exp)


    # tests for quadratic model

    def test_create_quadratic_dm(self):
        offset = False
        act = get_design_matrix(self.m, QUADRATIC, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 5))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, QUADRATIC, offset)
        assert_array_equal(act, exp)

    def test_create_quadratic_dm_offsets(self):
        offset = True
        act = get_design_matrix(self.m, QUADRATIC, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 6))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, QUADRATIC, offset)
        assert_array_equal(act, exp)

    # tests for partial cubic model

    def test_create_partcubic_dm(self):
        offset = False
        act = get_design_matrix(self.m, PART_CUBIC, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 6))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, PART_CUBIC, offset)
        assert_array_equal(act, exp)


    def test_create_partcubic_dm_offsets(self):
        offset = True
        act = get_design_matrix(self.m, PART_CUBIC, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 7))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, PART_CUBIC, offset)
        assert_array_equal(act, exp)


    # tests for unittest_dm() assuming network method

    def test_create_planar_dm_network(self):
        # networked method planar version should not have offsets col
        ncol_exp = 2
        exp = unittest_dm(self.m, NETWORK_METHOD, PLANAR, False)
        self.assertEqual(exp.shape, (self.m.num_cells, ncol_exp))
        exp2 = unittest_dm(self.m, NETWORK_METHOD, PLANAR, True)
        self.assertEqual(exp2.shape, (self.m.num_cells, ncol_exp))
        assert_array_equal(exp, exp2)

    def test_create_quadratic_dm_network(self):
        # quadratic version with networked method does not have offsets col
        ncol_exp = 5
        exp = unittest_dm(self.m, NETWORK_METHOD, QUADRATIC, False)
        self.assertEqual(exp.shape, (self.m.num_cells, ncol_exp))
        exp2 = unittest_dm(self.m, NETWORK_METHOD, QUADRATIC, True)
        self.assertEqual(exp2.shape, (self.m.num_cells, ncol_exp))
        assert_array_equal(exp, exp2)

    def test_create_partcubic_dm_network(self):
        # partial cubic version with networked method does not have offsets col
        ncol_exp = 6
        exp = unittest_dm(self.m, NETWORK_METHOD, PART_CUBIC, False)
        self.assertEqual(exp.shape, (self.m.num_cells, ncol_exp))
        exp2 = unittest_dm(self.m, NETWORK_METHOD, PART_CUBIC, True)
        self.assertEqual(exp2.shape, (self.m.num_cells, ncol_exp))
        assert_array_equal(exp, exp2)


class IndependentCorrectionTests(unittest.TestCase):
    """Test cases for the orbital correction component of PyRate."""

    def setUp(self):
        self.ifgs = sydney5_mock_ifgs()
        _add_nodata(self.ifgs)

        for ifg in self.ifgs:
            ifg.x_size = 90.0
            ifg.y_size = 89.5
            ifg.open()

    def alt_orbital_correction(self, ifg, deg, offset):
        # almost an exact translation of MATLAB version
        data = ifg.phase_data.reshape(ifg.num_cells)
        dm = get_design_matrix(ifg, deg, offset)[~isnan(data)]
        fd = data[~isnan(data)].reshape((dm.shape[0], 1))

        dmt = dm.T
        invNbb = inv(dmt.dot(dm))
        orbparams = invNbb.dot(dmt.dot(fd))
        alt_params = lstsq(dm, fd)[0]
        # FIXME: precision
        assert_array_almost_equal(orbparams, alt_params, decimal=2)

        dm2 = get_design_matrix(ifg, deg, offset)

        if offset:
            fullorb = np.reshape(np.dot(dm2[:, :-1], orbparams[:-1]),
                             ifg.phase_data.shape)
        else:
            fullorb = np.reshape(np.dot(dm2, orbparams), ifg.phase_data.shape)

        offset_removal = nanmedian(
            np.reshape(ifg.phase_data - fullorb, (1, -1)))
        fwd_correction = fullorb - offset_removal
        # ifg.phase_data -= (fullorb - offset_removal)
        return ifg.phase_data - fwd_correction

    def check_correction(self, degree, method, offset, decimal=2):
        orig = array([c.phase_data.copy() for c in self.ifgs])
        exp = [self.alt_orbital_correction(i, degree, offset) for i in self.ifgs]
        params = dict()
        params[cf.ORBITAL_FIT_METHOD] = method
        params[cf.ORBITAL_FIT_DEGREE] = degree
        params[cf.PARALLEL] = False
        orbital_correction(self.ifgs, params, None, offset)
        corrected = array([c.phase_data for c in self.ifgs])

        self.assertFalse((orig == corrected).all())
        self.check_results(self.ifgs, orig)  # test shape, data is non zero

        # FIXME: is decimal=2 close enough?
        for i, (e, a) in enumerate(zip(exp, corrected)):
            assert_array_almost_equal(e, a, decimal=decimal)<commit_msg>[orbital_test] updated to work with new changes in shared and orbital.py<commit_after>'''
Collection of tests for orbital correction.

Created on 31/3/13

.. codeauthor:: <NAME>, <NAME>
'''

import unittest
from os.path import join
from itertools import product
import os
import numpy as np
import tempfile
from numpy.linalg import pinv, inv
from numpy import nan, isnan, array
from numpy import empty, dot, concatenate, float32
from numpy.testing import assert_array_equal, assert_array_almost_equal

from pyrate import algorithm
from pyrate.shared import Ifg
from pyrate.orbital import OrbitalError, orbital_correction
from pyrate.orbital import get_design_matrix, get_network_design_matrix, get_num_params
from pyrate.orbital import INDEPENDENT_METHOD, NETWORK_METHOD, PLANAR, QUADRATIC, PART_CUBIC
from common import sydney5_mock_ifgs, MockIfg, SYD_TEST_TIF, sydney5_ifgs
from scipy.linalg import lstsq
from pyrate.tests.common import SYD_TEST_MATLAB_ORBITAL_DIR, TEMPDIR
from pyrate.tests.common import SYD_TEST_DIR
from pyrate import config as cf
from pyrate.tests.common import SYD_TEST_TIF, sydney_data_setup
from pyrate.tests.common import sydney_data_setup_ifg_file_list
import shutil
from pyrate.tests import common
from pyrate.shared import nanmedian
from pyrate.scripts import run_pyrate

DEG_LOOKUP = {
    2: PLANAR,
    5: QUADRATIC,
    6: PART_CUBIC}

NUM_COEF_LOOKUP = {
    PLANAR: 2,
    QUADRATIC: 5,
    PART_CUBIC: 6}


class SingleDesignMatrixTests(unittest.TestCase):
    """
    Tests to verify correctness of basic planar & quadratic design matrices or
    DMs. This class serves two purposes, ensuring the independent method DMs are
    produced correctly. Secondly, these indivdual DMs are subsets of the larger
    DM 'grid' required for the networked orbital correction method.
    """

    def setUp(self):
        # faked cell sizes
        self.xs = 0.75
        self.ys = 0.8
        self.ifg = Ifg(join(SYD_TEST_TIF, 'geo_060619-061002.tif'))
        self.ifg.open()
        self.ifg.nodata_value = 0

        self.m = MockIfg(self.ifg, 3, 4)
        self.m.x_size = self.xs
        self.m.y_size = self.ys

    # tests for planar model

    def test_create_planar_dm(self):
        offset = False
        act = get_design_matrix(self.m, PLANAR, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 2))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, PLANAR, offset)
        assert_array_equal(act, exp)

    def test_create_planar_dm_offsets(self):
        offset = True
        act = get_design_matrix(self.m, PLANAR, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 3))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, PLANAR, offset)
        assert_array_almost_equal(act, exp)


    # tests for quadratic model

    def test_create_quadratic_dm(self):
        offset = False
        act = get_design_matrix(self.m, QUADRATIC, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 5))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, QUADRATIC, offset)
        assert_array_equal(act, exp)

    def test_create_quadratic_dm_offsets(self):
        offset = True
        act = get_design_matrix(self.m, QUADRATIC, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 6))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, QUADRATIC, offset)
        assert_array_equal(act, exp)

    # tests for partial cubic model

    def test_create_partcubic_dm(self):
        offset = False
        act = get_design_matrix(self.m, PART_CUBIC, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 6))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, PART_CUBIC, offset)
        assert_array_equal(act, exp)


    def test_create_partcubic_dm_offsets(self):
        offset = True
        act = get_design_matrix(self.m, PART_CUBIC, offset)
        self.assertEqual(act.shape, (self.m.num_cells, 7))
        exp = unittest_dm(self.m, INDEPENDENT_METHOD, PART_CUBIC, offset)
        assert_array_equal(act, exp)


    # tests for unittest_dm() assuming network method

    def test_create_planar_dm_network(self):
        # networked method planar version should not have offsets col
        ncol_exp = 2
        exp = unittest_dm(self.m, NETWORK_METHOD, PLANAR, False)
        self.assertEqual(exp.shape, (self.m.num_cells, ncol_exp))
        exp2 = unittest_dm(self.m, NETWORK_METHOD, PLANAR, True)
        self.assertEqual(exp2.shape, (self.m.num_cells, ncol_exp))
        assert_array_equal(exp, exp2)

    def test_create_quadratic_dm_network(self):
        # quadratic version with networked method does not have offsets col
        ncol_exp = 5
        exp = unittest_dm(self.m, NETWORK_METHOD, QUADRATIC, False)
        self.assertEqual(exp.shape, (self.m.num_cells, ncol_exp))
        exp2 = unittest_dm(self.m, NETWORK_METHOD, QUADRATIC, True)
        self.assertEqual(exp2.shape, (self.m.num_cells, ncol_exp))
        assert_array_equal(exp, exp2)

    def test_create_partcubic_dm_network(self):
        # partial cubic version with networked method does not have offsets col
        ncol_exp = 6
        exp = unittest_dm(self.m, NETWORK_METHOD, PART_CUBIC, False)
        self.assertEqual(exp.shape, (self.m.num_cells, ncol_exp))
        exp2 = unittest_dm(self.m, NETWORK_METHOD, PART_CUBIC, True)
        self.assertEqual(exp2.shape, (self.m.num_cells, ncol_exp))
        assert_array_equal(exp, exp2)


class IndependentCorrectionTests(unittest.TestCase):
    """Test cases for the orbital correction component of PyRate."""

    def setUp(self):
        self.ifgs = sydney5_mock_ifgs()
        _add_nodata(self.ifgs)

        for ifg in self.ifgs:
            ifg.x_size = 90.0
            ifg.y_size = 89.5
            ifg.open()

    def alt_orbital_correction(self, ifg, deg, offset):
        # almost an exact translation of MATLAB version
        data = ifg.phase_data.reshape(ifg.num_cells)
        dm = get_design_matrix(ifg, deg, offset)[~isnan(data)]
        fd = data[~isnan(data)].reshape((dm.shape[0], 1))

        dmt = dm.T
        invNbb = inv(dmt.dot(dm))
        orbparams = invNbb.dot(dmt.dot(fd))
        alt_params = lstsq(dm, fd)[0]
        # FIXME: precision
        assert_array_almost_equal(orbparams, alt_params, decimal=2)

        dm2 = get_design_matrix(ifg, deg, offset)

        if offset:
            fullorb = np.reshape(np.dot(dm2[:, :-1], orbparams[:-1]),
                             ifg.phase_data.shape)
        else:
            fullorb = np.reshape(np.dot(dm2, orbparams), ifg.phase_data.shape)

        offset_removal = nanmedian(
            np.reshape(ifg.phase_data - fullorb, (1, -1)))
        fwd_correction = fullorb - offset_removal
        # ifg.phase_data -= (fullorb - offset_removal)
        return ifg.phase_data - fwd_correction

    def check_correction(self, degree, method, offset, decimal=2):
        orig = array([c.phase_data.copy() for c in self.ifgs])
        exp = [self.alt_orbital_correction(i, degree, offset) for i in self.ifgs]
        params = dict()
        params[cf.ORBITAL_FIT_METHOD] = method
        params[cf.ORBITAL_FIT_DEGREE] = degree
        params[cf.PARALLEL] = False
        params[cf.NO_DATA_VALUE] = 0
        params[cf.NAN_CONVERSION] = False
        for i in self.ifgs:
            i.mm_converted = True
        orbital_correction(self.ifgs, params, None, offset)
        corrected = array([c.phase_data for c in self.ifgs])

        self.assertFalse((orig == corrected).all())
        self.check_results(self.ifgs, orig)  # test shape, data is non zero

        # FIXME: is decimal=2 close enough?
        for i, (e, a) in enumerate(zip(exp, corrected)):
            assert_array_almost_equal(e, a, decimal=decimal)
{'fraction_non_alphanumeric': 0.09284776902887139, 'fraction_numerical': 0.009645669291338583, 'mean_word_length': 3.054535780792764, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5259128', 'n_tokens_mistral': 5368, 'n_tokens_neox': 5097, 'n_words': 1197}