<commit_before>#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# **       PTS -- Python Toolkit for working with SKIRT          **
# **       © Astronomical Observatory, Ghent University          **
# *****************************************************************

## \package pts.dustpedia.core.sdss Contains the SDSSMosaicMaker class.

# -----------------------------------------------------------------

# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function

# Import standard modules
import numpy as np
from scipy.interpolate import interp2d

# Import astronomical modules
from astropy.io.fits import open as open_fits
#from astroquery.sdss import SDSS

# Import the relevant PTS classes and modules
from ...core.basics.configurable import Configurable
from ...core.tools.logging import log
from .dataprocessing import DustPediaDataProcessing
from .sample import DustPediaSample
from ...core.tools import filesystem as fs
from ...magic.basics.coordinatesystem import CoordinateSystem
from ...magic.core.image import Image
from ...core.tools import network
from ...magic.basics.coordinate import SkyCoordinate
from ...magic.tools import mosaicing
from ...core.filter.broad import BroadBandFilter
from ...core.tools import formatting as fmt
from ...magic.core.frame import Frame, sum_frames, sum_frames_quadratically
from ...core.tools.parallelization import ParallelTarget
from ...core.basics.configuration import print_mapping
from ...core.tools.formatting import print_files_in_path, print_directories_in_path
from ...core.tools import archive

# -----------------------------------------------------------------

sdss_bands = ["u", "g", "r", "i", "z"]
sdss_filter_number = {"u": 0, "g": 1, "r": 2, "i": 3, "z": 4}

# -----------------------------------------------------------------

boss_photoobj_url =	"https://data.sdss.org/sas/dr13/eboss/photoObj"

# -----------------------------------------------------------------

class SDSSMosaicMaker(Configurable):

    """
    This class ...
    """

    def __init__(self, *args, **kwargs):

        """
        This function ...
        :param interactive:
        """

        # Call the constructor of the base class
        super(SDSSMosaicMaker, self).__init__(*args, **kwargs)

        # The DustPedia sample object
        self.sample = None

        # The DustPedia data processing object
        self.dpdp = None

        # The NGC name
        self.ngc_name = None

        # The cutout properties
        self.cutout_center = None
        self.cutout_width = None

        # The WCS
        self.rebin_wcs = None

        # Temporary root path
        self.root_path = None

        # Paths for different bands
        self.band_paths = dict()
        self.fields_paths = dict()
        self.photofield_paths = dict()
        self.counts_paths = dict()
        self.poisson_paths = dict()
        self.rebinned_paths = dict()
        self.mosaics_paths = dict()
        self.result_paths = dict()

        # The URLs from where the images were downloaded
        self.urls = dict()
        self.photofield_urls = dict()

        # The mosaic frames and error maps
        self.mosaics = dict()
        self.error_maps = dict()
        self.relative_error_maps = dict()

    # -----------------------------------------------------------------

    def run(self, **kwargs):

        """
        This function ...
        :param kwargs:
        :return:
        """

        # 1. Call the setup function
        self.setup(**kwargs)

        # 2. Create directories
        self.create_directories()

        # 3. Get the cutout range
        self.get_range()

        # 4. Get target header
        self.get_target_header()

        # 5. Get the fields
        self.get_fields()

        # 6. Convert to counts (decalibrate)
        self.convert_to_counts()

        # 7. Convert to counts
        self.convert_to_nanomaggies()

        # 8. Rebin
        self.rebin()

        # 9. Make the mosaics
        self.mosaic()

        # 10. Convert to desired unit
        self.convert_units()

        # 11. Finish
        self.finish()

        # 12. Show
        self.show()

        # 13. Write
        self.write()

    # -----------------------------------------------------------------

    def nurls_for_band(self, band):

        """
        This function ...
        :param band:
        :return:
        """

        return len(self.urls[band])

    # -----------------------------------------------------------------

    def setup(self, **kwargs):

        """
        This function ...
        :param kwargs:
        :return:
        """

        # Call the setup function of the base class
        super(SDSSMosaicMaker, self).setup(**kwargs)

        # Show the configuration
        if log.is_debug():
            log.debug("Configuration:")
            print_mapping(self.config, empty_lines=False, indent="  ")
            print("")

        # Create the DustPedia sample object
        self.sample = DustPediaSample()

        # Create the DustPedia data processing instance
        self.dpdp = DustPediaDataProcessing()

        # Get the NGC name
        self.ngc_name = self.sample.get_name(self.config.galaxy_name)

    # -----------------------------------------------------------------

    def create_directories(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Creating directories ...")

        # Root path
        #self.root_path = fs.join(fs.home(), time.unique_name("SDSS_" + self.config.galaxy_name))
        #fs.create_directory(self.root_path)

        self.root_path = self.config.path

        # Loop over the bands
        for band in self.config.bands:

            # Create root directory
            path = fs.create_directory_in(self.root_path, band)

            # Create directories
            fields_path = fs.create_directory_in(path, "fields") # frames with HDU0, HDU1, HDU2 .. (primary, calib, sky, ...) IN NANOMAGGIES PER PIXEL
            photofield_path = fs.create_directory_in(path, "photofield") # photoField files
            counts_path = fs.create_directory_in(path, "counts") # frames IN COUNTS (DN)
            poisson_path = fs.create_directory_in(path, "poisson") # error maps in NANOMAGGIES PER PIXEL
            rebinned_path = fs.create_directory_in(path, "rebinned") # images with frame and corresponding error map in NANOMAGGIES, REBINNED
            mosaics_path = fs.create_directory_in(path, "mosaics") # the mosaic images

            # Set paths
            self.band_paths[band] = path
            self.fields_paths[band] = fields_path
            self.photofield_paths[band] = photofield_path
            self.counts_paths[band] = counts_path
            self.poisson_paths[band] = poisson_path
            self.rebinned_paths[band] = rebinned_path
            self.mosaics_paths[band] = mosaics_path

            # Debugging
            print_directories_in_path(path)

    # -----------------------------------------------------------------

    def get_range(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting coordinate range for the mosaic ...")

        # Get the coordinate range first for this galaxy
        ra, dec, width = self.dpdp.get_cutout_range_for_galaxy(self.ngc_name)

        # Set the cutout center and cutout width
        self.cutout_center = SkyCoordinate(ra, dec, unit="deg", frame="fk5")
        self.cutout_width = width

        # Debugging
        log.debug("Cutout center: " + str(self.cutout_center))
        log.debug("Cutout width: " + str(self.cutout_width))

    # -----------------------------------------------------------------

    def get_target_header(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Creating the header and WCS for the mosaic ...")

        # Get the target header
        header = self.dpdp.get_header_for_galaxy(self.ngc_name, "SDSS")

        # To coordinate system
        self.rebin_wcs = CoordinateSystem(header)

        # Debugging
        log.debug("Rebin WCS:")
        if log.is_debug():
            print("")
            print(self.rebin_wcs)
            print("")

    # -----------------------------------------------------------------

    def get_fields(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting the fields ...")

        # 1. Get the urls
        self.get_urls()

        # 2. Download SDSS fields
        self.download_fields()

        # 4. Download photofield data
        self.download_photofield()

        # 3. Filter those not overlapping the target position
        self.filter_non_overlapping()

    # -----------------------------------------------------------------

    def get_urls(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting URLs ...")

        # 1. Fields
        self.get_field_urls()

        # 2. Photofield tables
        self.get_photofield_urls()

    # -----------------------------------------------------------------

    def get_field_urls(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting field URLs ...")

        # Loop over the bands
        for band in self.config.bands:

            # Inform the user
            log.info("Getting URLs of the SDSS primary fields for " + self.ngc_name + " in the " + band + " band ...")

            # Get the urls
            urls = self.get_sdss_primary_field_urls_for_galaxy(band)

            ## NEW: USE DR13 INSTEAD OF DR10  (BETTER CALIBRATION?)
            # new_urls = [url.replace("dr10/boss", "dr13/eboss") for url in urls] # DOESN'T WORK: RESTRICTED ACCESS: ASKS FOR USERNAME AND PASSWORD!!
            ###

            # USE DR12
            new_urls = [url.replace("dr10/boss", "dr12/boss") for url in urls]

            # Limit the number of fields if requested
            limit_band = self.config["max_nobservations_" + band]
            if limit_band is not None:
                if self.config.max_nobservations_fuv is not None: log.debug("The maximum number of observations for the " + band + " band is " + str(limit_band))
                new_urls = new_urls[:min(len(new_urls), limit_band)] # Remove the last x fields

            # Save the URLS for this band
            # The original urls from which the images were downloaded
            self.urls[band] = new_urls

            # Debugging
            log.debug("The number of primary fields for the " + band + " band is " + str(self.nurls_for_band(band)))

    # -----------------------------------------------------------------

    def get_photofield_urls(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting the URLs of the photofield tables ...")

        # Start of URLs
        field_url_start = "http://data.sdss3.org/sas/dr12/env/BOSS_PHOTOOBJ"  # = $BOSS_PHOTOOBJ/RERUN/RUN
        # NOT WORKING WITH DR13 : BECAUSE OF PERMISSION ISSUES (PASSWORD IS ASKED WHEN ENTERED IN BROWSER !!)

        # FIELD URL = field_url_start / RERUN / RUN    + / photoField-6digits-CAMCOL.fits
        # example: http://data.sdss3.org/sas/dr12/env/BOSS_PHOTOOBJ/301/4294/photoField-004294-5.fits

        # FRAME URL = $BOSS_PHOTOOBJ / frames / RERUN / RUN / CAMCOL    +   /frame-[ugriz]-6digits-CAMCOL-FRAMESEQ.fits.bz2
        # example: http://data.sdss3.org/sas/dr10/boss/photoObj/frames/301/4294/5/frame-i-004294-5-0229.fits.bz2

        # Loop over the bands
        for band in self.config.bands:

            # Inform the user
            log.info("Getting the URLs of the photofield tables for the " + band + " band ...")

            # Initialize a list to contain the URLs of the field tables
            field_urls = []
            for url in self.urls[band]:

                # Construct photofield table URL
                rerun_run_camcol = url.split("frames/")[1].split("/frame-")[0]
                rerun, run, camcol = rerun_run_camcol.split("/")
                rerun_run = rerun + "/" + run
                band, digits, camcol, frameseq = url.split("frame-")[1].split(".fits")[0].split("-")
                field_url = field_url_start + "/" + rerun_run + "/photoField-" + digits + "-" + camcol + ".fits"

                # Add the URL
                field_urls.append(field_url)

            # Remove duplicates (one field table contains the info for all 5 SDSS bands!)
            field_urls = list(set(field_urls))

            # Set the URLs
            self.photofield_urls[band] = field_urls

            # Debugging
            log.debug("The number of photofield urls for the " + band + " band is " + str(len(self.photofield_urls[band])))

    # -----------------------------------------------------------------

    def has_existing_fields(self, band):

        """
        This fucntion ...
        :param band: 
        :return: 
        """

        if self.config.fields_directories is not None and band in self.config.fields_directories: return True
        else: return False

    # -----------------------------------------------------------------

    def existing_fields_path(self, band):
        
        """
        This function ...
        :param band: 
        :return: 
        """

        if self.config.fields_directories is not None and band in self.config.fields_directories: return fs.absolute_path(self.config.fields_directories[band])
        else: return None

    # -----------------------------------------------------------------

    def download_fields(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Downloading fields (in parallel) ...")

        # Parallel execution
        with ParallelTarget(network.download_and_decompress_files, self.config.nprocesses) as target:

            # Download for different bands in parallel, or copy from existing directory
            for band in self.config.bands:

                if self.has_existing_fields(band):

                    log.debug("Existing fields path for band: " + self.existing_fields_path(band))
                    fmt.print_files_in_path(self.existing_fields_path(band))
                    extensions = ["fits"]
                    extensions.extend(archive.extensions)
                    paths = fs.files_in_path(self.existing_fields_path(band), not_contains="meta", extension=extensions)
                    fmt.print_files_in_list(paths, "existing fields")
                    fs.copy_and_decompress_files(paths, self.fields_paths[band])
                    #fs.copy_from_directory(self.existing_fields_path(band), self.fields_paths[band], not_contains="meta", extension="fits")

                else: target(self.urls[band], self.fields_paths[band])

        # Debugging
        for band in self.config.bands: print_files_in_path(self.fields_paths[band])

    # -----------------------------------------------------------------

    def get_sdss_primary_field_urls_for_galaxy(self, band):

        """
        This function ...
        :param band:
        :return:
        """

        # Inform the user
        log.info("Getting the URLs of the primary field for galaxy " + self.ngc_name + " for the " + band + " band ...")

        # Get the SDSS fields that cover this coordinate range (from Montage) (URLS)
        table = mosaicing.get_field_table(self.cutout_center, self.cutout_width, band)

        # Get the set of unique URLs
        sdss_urls = list(set(table["url"]))

        # ...
        dr12_pri = []
        for i in range(len(self.dpdp.sdss_field_table)):
            entry = str(int(self.dpdp.sdss_field_table['RUN'][i])) + ' ' + str(int(self.dpdp.sdss_field_table['CAMCOL'][i])) + ' ' + str(int(self.dpdp.sdss_field_table['FIELD'][i]))
            dr12_pri.append(entry)

        # Get the URLS of the primary fields
        sdss_urls_pri = filter_sdss_urls_for_primary_frames(sdss_urls, dr12_pri)

        # Return the list of URLS
        return sdss_urls_pri

    # -----------------------------------------------------------------

    def filter_non_overlapping(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Filtering non-overlapping SDSS primary fields (in parallel) ...")

        # Parallel execution
        with ParallelTarget(mosaicing.filter_non_overlapping, self.config.nprocesses) as target:

            # Loop over the bands
            for band in self.config.bands: target(self.ngc_name, band, self.fields_paths[band], self.cutout_center, self.cutout_width, path=self.band_paths[band])

        # Debugging
        for band in self.config.bands: print_files_in_path(self.fields_paths[band])

    # -----------------------------------------------------------------

    def download_photofield(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Downloading the photofield tables (in parallel) ...")

        # Parallel execution
        with ParallelTarget(network.download_files, self.config.nprocesses) as target:

            # Loop over the bands
            for band in self.config.bands: target(self.photofield_urls[band], self.photofield_paths[band])

        # Debugging
        for band in self.config.bands: print_files_in_path(self.photofield_paths[band])

    # -----------------------------------------------------------------

    def convert_to_counts(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Converting the fields to counts (decalibrating) (in parallel) ...")

        # Loop over the bands
        for band in self.config.bands:

            fields_path = self.fields_paths[band]
            counts_path = self.counts_paths[band]

            # Parallel execution
            with ParallelTarget(convert_frame_to_counts, self.config.nprocesses) as convert_frame:

                # Loop over the downloaded "frame" files
                # extension must be specified because there is also the meta.dat and overlap.dat files!!
                for path in fs.files_in_path(fields_path, extension="fits"): convert_frame(path, counts_path)

            # Debugging
            print_files_in_path(counts_path)

    # -----------------------------------------------------------------

    def convert_to_nanomaggies(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Converting fields to nanomaggies (in parallel) ...")

        # Loop over the bands
        for band in self.config.bands:

            # Inform the user
            log.info("Computing error maps for the " + band + " band, for each SDSS frame in nanomaggies per pixel ...")

            # Get paths
            fields_path = self.fields_paths[band]
            photofields_path = self.photofield_paths[band]
            counts_path = self.counts_paths[band]
            poisson_path = self.poisson_paths[band]

            # Parallel execution
            with ParallelTarget(convert_frame_to_nanomaggies, self.config.nprocesses) as convert_frame:

                # Loop over the files in the counts path
                for path, name in fs.files_in_path(counts_path, extension="fits", returns=["path", "name"]): convert_frame(path, name, fields_path, photofields_path, poisson_path)

            # Debugging
            print_files_in_path(poisson_path)

    # -----------------------------------------------------------------

    def rebin(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Rebinning frames ...")

        # Loop over the bands
        for band in self.config.bands:

            # Get paths
            fields_path = self.fields_paths[band]
            poisson_path = self.poisson_paths[band]
            rebinned_path = self.rebinned_paths[band]

            # Inform the user
            log.info("Rebinning primary SDSS frames and corresponding error maps to the target coordinate system ...")

            # Parallel execution
            with ParallelTarget(rebin_frame, self.config.nprocesses) as rebin:

                # Loop over the files in the raw directory
                for path, name in fs.files_in_path(fields_path, extension="fits", returns=["path", "name"]): rebin(path, name, poisson_path, self.rebin_wcs, rebinned_path)

            # Debugging
            print_files_in_path(rebinned_path)

    # -----------------------------------------------------------------

    def mosaic(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Performing the mosaicing (in parallel) ...")

        # Parallel execution
        with ParallelTarget(create_mosaic, self.config.nprocesses) as target:

            # Loop over the bands
            for band in self.config.bands:

                # Get paths
                rebinned_path = self.rebinned_paths[band]
                mosaics_path = self.mosaics_paths[band]

                # Call the target function
                target(band, rebinned_path, self.rebin_wcs, mosaics_path)

        # Debugging
        for band in self.config.bands: print_files_in_path(self.mosaics_paths[band])

    # -----------------------------------------------------------------

    def convert_units(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Converting to Jansky ...")

        results = dict()

        # Parallel execution
        with ParallelTarget(convert_mosaic_to_jansky, self.config.nprocesses) as target:

            # Loop over the bands
            for band in self.config.bands:

                # Get path
                mosaics_path = self.mosaics_paths[band]

                # Call the target function
                result = target(mosaics_path)

                # Set result
                results[band] = result

        # Load the mosaics
        for band in self.config.bands:

            # Debugging
            print_files_in_path(self.mosaics_paths[band])

            results[band].request()
            output = results[band].output

            result_mosaic_path = output[0]

            self.result_paths[band] = result_mosaic_path

    # -----------------------------------------------------------------

    def finish(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Finishing ...")

        # Loop over the bands
        for band in self.config.bands:

            # Load the image
            image = Image.from_file(self.result_paths[band])
<commit_msg>Fixed small bug<commit_after>#!/usr/bin/env python
# -*- coding: utf8 -*-
# *****************************************************************
# **       PTS -- Python Toolkit for working with SKIRT          **
# **       © Astronomical Observatory, Ghent University          **
# *****************************************************************

## \package pts.dustpedia.core.sdss Contains the SDSSMosaicMaker class.

# -----------------------------------------------------------------

# Ensure Python 3 compatibility
from __future__ import absolute_import, division, print_function

# Import standard modules
import numpy as np
from scipy.interpolate import interp2d

# Import astronomical modules
from astropy.io.fits import open as open_fits
#from astroquery.sdss import SDSS

# Import the relevant PTS classes and modules
from ...core.basics.configurable import Configurable
from ...core.tools.logging import log
from .dataprocessing import DustPediaDataProcessing
from .sample import DustPediaSample
from ...core.tools import filesystem as fs
from ...magic.basics.coordinatesystem import CoordinateSystem
from ...magic.core.image import Image
from ...core.tools import network
from ...magic.basics.coordinate import SkyCoordinate
from ...magic.tools import mosaicing
from ...core.filter.broad import BroadBandFilter
from ...core.tools import formatting as fmt
from ...magic.core.frame import Frame, sum_frames, sum_frames_quadratically
from ...core.tools.parallelization import ParallelTarget
from ...core.basics.configuration import print_mapping
from ...core.tools.formatting import print_files_in_path, print_directories_in_path
from ...core.tools import archive
from ...core.tools import types

# -----------------------------------------------------------------

sdss_bands = ["u", "g", "r", "i", "z"]
sdss_filter_number = {"u": 0, "g": 1, "r": 2, "i": 3, "z": 4}

# -----------------------------------------------------------------

boss_photoobj_url =	"https://data.sdss.org/sas/dr13/eboss/photoObj"

# -----------------------------------------------------------------

class SDSSMosaicMaker(Configurable):

    """
    This class ...
    """

    def __init__(self, *args, **kwargs):

        """
        This function ...
        :param interactive:
        """

        # Call the constructor of the base class
        super(SDSSMosaicMaker, self).__init__(*args, **kwargs)

        # The DustPedia sample object
        self.sample = None

        # The DustPedia data processing object
        self.dpdp = None

        # The NGC name
        self.ngc_name = None

        # The cutout properties
        self.cutout_center = None
        self.cutout_width = None

        # The WCS
        self.rebin_wcs = None

        # Temporary root path
        self.root_path = None

        # Paths for different bands
        self.band_paths = dict()
        self.fields_paths = dict()
        self.photofield_paths = dict()
        self.counts_paths = dict()
        self.poisson_paths = dict()
        self.rebinned_paths = dict()
        self.mosaics_paths = dict()
        self.result_paths = dict()

        # The URLs from where the images were downloaded
        self.urls = dict()
        self.photofield_urls = dict()

        # The mosaic frames and error maps
        self.mosaics = dict()
        self.error_maps = dict()
        self.relative_error_maps = dict()

    # -----------------------------------------------------------------

    def run(self, **kwargs):

        """
        This function ...
        :param kwargs:
        :return:
        """

        # 1. Call the setup function
        self.setup(**kwargs)

        # 2. Create directories
        self.create_directories()

        # 3. Get the cutout range
        self.get_range()

        # 4. Get target header
        self.get_target_header()

        # 5. Get the fields
        self.get_fields()

        # 6. Convert to counts (decalibrate)
        self.convert_to_counts()

        # 7. Convert to counts
        self.convert_to_nanomaggies()

        # 8. Rebin
        self.rebin()

        # 9. Make the mosaics
        self.mosaic()

        # 10. Convert to desired unit
        self.convert_units()

        # 11. Finish
        self.finish()

        # 12. Show
        self.show()

        # 13. Write
        self.write()

    # -----------------------------------------------------------------

    def nurls_for_band(self, band):

        """
        This function ...
        :param band:
        :return:
        """

        return len(self.urls[band])

    # -----------------------------------------------------------------

    def setup(self, **kwargs):

        """
        This function ...
        :param kwargs:
        :return:
        """

        # Call the setup function of the base class
        super(SDSSMosaicMaker, self).setup(**kwargs)

        # Show the configuration
        if log.is_debug():
            log.debug("Configuration:")
            print_mapping(self.config, empty_lines=False, indent="  ")
            print("")

        # Create the DustPedia sample object
        self.sample = DustPediaSample()

        # Create the DustPedia data processing instance
        self.dpdp = DustPediaDataProcessing()

        # Get the NGC name
        self.ngc_name = self.sample.get_name(self.config.galaxy_name)

    # -----------------------------------------------------------------

    def create_directories(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Creating directories ...")

        # Root path
        #self.root_path = fs.join(fs.home(), time.unique_name("SDSS_" + self.config.galaxy_name))
        #fs.create_directory(self.root_path)

        self.root_path = self.config.path

        # Loop over the bands
        for band in self.config.bands:

            # Create root directory
            path = fs.create_directory_in(self.root_path, band)

            # Create directories
            fields_path = fs.create_directory_in(path, "fields") # frames with HDU0, HDU1, HDU2 .. (primary, calib, sky, ...) IN NANOMAGGIES PER PIXEL
            photofield_path = fs.create_directory_in(path, "photofield") # photoField files
            counts_path = fs.create_directory_in(path, "counts") # frames IN COUNTS (DN)
            poisson_path = fs.create_directory_in(path, "poisson") # error maps in NANOMAGGIES PER PIXEL
            rebinned_path = fs.create_directory_in(path, "rebinned") # images with frame and corresponding error map in NANOMAGGIES, REBINNED
            mosaics_path = fs.create_directory_in(path, "mosaics") # the mosaic images

            # Set paths
            self.band_paths[band] = path
            self.fields_paths[band] = fields_path
            self.photofield_paths[band] = photofield_path
            self.counts_paths[band] = counts_path
            self.poisson_paths[band] = poisson_path
            self.rebinned_paths[band] = rebinned_path
            self.mosaics_paths[band] = mosaics_path

            # Debugging
            print_directories_in_path(path)

    # -----------------------------------------------------------------

    def get_range(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting coordinate range for the mosaic ...")

        # Get the coordinate range first for this galaxy
        ra, dec, width = self.dpdp.get_cutout_range_for_galaxy(self.ngc_name)

        # Set the cutout center and cutout width
        self.cutout_center = SkyCoordinate(ra, dec, unit="deg", frame="fk5")
        self.cutout_width = width

        # Debugging
        log.debug("Cutout center: " + str(self.cutout_center))
        log.debug("Cutout width: " + str(self.cutout_width))

    # -----------------------------------------------------------------

    def get_target_header(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Creating the header and WCS for the mosaic ...")

        # Get the target header
        header = self.dpdp.get_header_for_galaxy(self.ngc_name, "SDSS")

        # To coordinate system
        self.rebin_wcs = CoordinateSystem(header)

        # Debugging
        log.debug("Rebin WCS:")
        if log.is_debug():
            print("")
            print(self.rebin_wcs)
            print("")

    # -----------------------------------------------------------------

    def get_fields(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting the fields ...")

        # 1. Get the urls
        self.get_urls()

        # 2. Download SDSS fields
        self.download_fields()

        # 4. Download photofield data
        self.download_photofield()

        # 3. Filter those not overlapping the target position
        self.filter_non_overlapping()

    # -----------------------------------------------------------------

    def get_urls(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting URLs ...")

        # 1. Fields
        self.get_field_urls()

        # 2. Photofield tables
        self.get_photofield_urls()

    # -----------------------------------------------------------------

    def get_field_urls(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting field URLs ...")

        # Loop over the bands
        for band in self.config.bands:

            # Inform the user
            log.info("Getting URLs of the SDSS primary fields for " + self.ngc_name + " in the " + band + " band ...")

            # Get the urls
            urls = self.get_sdss_primary_field_urls_for_galaxy(band)

            ## NEW: USE DR13 INSTEAD OF DR10  (BETTER CALIBRATION?)
            # new_urls = [url.replace("dr10/boss", "dr13/eboss") for url in urls] # DOESN'T WORK: RESTRICTED ACCESS: ASKS FOR USERNAME AND PASSWORD!!
            ###

            # USE DR12
            new_urls = [url.replace("dr10/boss", "dr12/boss") for url in urls]

            # Limit the number of fields if requested
            limit_band = self.config["max_nobservations_" + band]
            if limit_band is not None:
                if self.config.max_nobservations_fuv is not None: log.debug("The maximum number of observations for the " + band + " band is " + str(limit_band))
                new_urls = new_urls[:min(len(new_urls), limit_band)] # Remove the last x fields

            # Save the URLS for this band
            # The original urls from which the images were downloaded
            self.urls[band] = new_urls

            # Debugging
            log.debug("The number of primary fields for the " + band + " band is " + str(self.nurls_for_band(band)))

    # -----------------------------------------------------------------

    def get_photofield_urls(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Getting the URLs of the photofield tables ...")

        # Start of URLs
        field_url_start = "http://data.sdss3.org/sas/dr12/env/BOSS_PHOTOOBJ"  # = $BOSS_PHOTOOBJ/RERUN/RUN
        # NOT WORKING WITH DR13 : BECAUSE OF PERMISSION ISSUES (PASSWORD IS ASKED WHEN ENTERED IN BROWSER !!)

        # FIELD URL = field_url_start / RERUN / RUN    + / photoField-6digits-CAMCOL.fits
        # example: http://data.sdss3.org/sas/dr12/env/BOSS_PHOTOOBJ/301/4294/photoField-004294-5.fits

        # FRAME URL = $BOSS_PHOTOOBJ / frames / RERUN / RUN / CAMCOL    +   /frame-[ugriz]-6digits-CAMCOL-FRAMESEQ.fits.bz2
        # example: http://data.sdss3.org/sas/dr10/boss/photoObj/frames/301/4294/5/frame-i-004294-5-0229.fits.bz2

        # Loop over the bands
        for band in self.config.bands:

            # Inform the user
            log.info("Getting the URLs of the photofield tables for the " + band + " band ...")

            # Initialize a list to contain the URLs of the field tables
            field_urls = []
            for url in self.urls[band]:

                # Construct photofield table URL
                rerun_run_camcol = url.split("frames/")[1].split("/frame-")[0]
                rerun, run, camcol = rerun_run_camcol.split("/")
                rerun_run = rerun + "/" + run
                band, digits, camcol, frameseq = url.split("frame-")[1].split(".fits")[0].split("-")
                field_url = field_url_start + "/" + rerun_run + "/photoField-" + digits + "-" + camcol + ".fits"

                # Add the URL
                field_urls.append(field_url)

            # Remove duplicates (one field table contains the info for all 5 SDSS bands!)
            field_urls = list(set(field_urls))

            # Set the URLs
            self.photofield_urls[band] = field_urls

            # Debugging
            log.debug("The number of photofield urls for the " + band + " band is " + str(len(self.photofield_urls[band])))

    # -----------------------------------------------------------------

    def has_existing_fields(self, band):

        """
        This fucntion ...
        :param band: 
        :return: 
        """

        if self.config.fields_directories is not None and band in self.config.fields_directories: return True
        else: return False

    # -----------------------------------------------------------------

    def existing_fields_path(self, band):
        
        """
        This function ...
        :param band: 
        :return: 
        """

        if self.config.fields_directories is not None and band in self.config.fields_directories: return fs.absolute_path(self.config.fields_directories[band])
        else: return None

    # -----------------------------------------------------------------

    def download_fields(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Downloading fields (in parallel) ...")

        # Parallel execution
        with ParallelTarget(network.download_and_decompress_files, self.config.nprocesses) as target:

            # Download for different bands in parallel, or copy from existing directory
            for band in self.config.bands:

                if self.has_existing_fields(band):

                    log.debug("Existing fields path for band: " + self.existing_fields_path(band))
                    fmt.print_files_in_path(self.existing_fields_path(band))
                    extensions = ["fits"]
                    extensions.extend(archive.extensions)
                    paths = fs.files_in_path(self.existing_fields_path(band), not_contains="meta", extension=extensions)
                    fmt.print_files_in_list(paths, "existing fields")
                    fs.copy_and_decompress_files(paths, self.fields_paths[band])
                    #fs.copy_from_directory(self.existing_fields_path(band), self.fields_paths[band], not_contains="meta", extension="fits")

                else: target(self.urls[band], self.fields_paths[band])

        # Debugging
        for band in self.config.bands: print_files_in_path(self.fields_paths[band])

    # -----------------------------------------------------------------

    def get_sdss_primary_field_urls_for_galaxy(self, band):

        """
        This function ...
        :param band:
        :return:
        """

        # Inform the user
        log.info("Getting the URLs of the primary field for galaxy " + self.ngc_name + " for the " + band + " band ...")

        # Get the SDSS fields that cover this coordinate range (from Montage) (URLS)
        table = mosaicing.get_field_table(self.cutout_center, self.cutout_width, band)

        # Get the set of unique URLs
        sdss_urls = list(set(table["url"]))

        # ...
        dr12_pri = []
        for i in range(len(self.dpdp.sdss_field_table)):
            entry = str(int(self.dpdp.sdss_field_table['RUN'][i])) + ' ' + str(int(self.dpdp.sdss_field_table['CAMCOL'][i])) + ' ' + str(int(self.dpdp.sdss_field_table['FIELD'][i]))
            dr12_pri.append(entry)

        # Get the URLS of the primary fields
        sdss_urls_pri = filter_sdss_urls_for_primary_frames(sdss_urls, dr12_pri)

        # Return the list of URLS
        return sdss_urls_pri

    # -----------------------------------------------------------------

    def filter_non_overlapping(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Filtering non-overlapping SDSS primary fields (in parallel) ...")

        # Parallel execution
        with ParallelTarget(mosaicing.filter_non_overlapping, self.config.nprocesses) as target:

            # Loop over the bands
            for band in self.config.bands: target(self.ngc_name, band, self.fields_paths[band], self.cutout_center, self.cutout_width, path=self.band_paths[band])

        # Debugging
        for band in self.config.bands: print_files_in_path(self.fields_paths[band])

    # -----------------------------------------------------------------

    def download_photofield(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Downloading the photofield tables (in parallel) ...")

        # Parallel execution
        with ParallelTarget(network.download_files, self.config.nprocesses) as target:

            # Loop over the bands
            for band in self.config.bands: target(self.photofield_urls[band], self.photofield_paths[band])

        # Debugging
        for band in self.config.bands: print_files_in_path(self.photofield_paths[band])

    # -----------------------------------------------------------------

    def convert_to_counts(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Converting the fields to counts (decalibrating) (in parallel) ...")

        # Loop over the bands
        for band in self.config.bands:

            fields_path = self.fields_paths[band]
            counts_path = self.counts_paths[band]

            # Parallel execution
            with ParallelTarget(convert_frame_to_counts, self.config.nprocesses) as convert_frame:

                # Loop over the downloaded "frame" files
                # extension must be specified because there is also the meta.dat and overlap.dat files!!
                for path in fs.files_in_path(fields_path, extension="fits"): convert_frame(path, counts_path)

            # Debugging
            print_files_in_path(counts_path)

    # -----------------------------------------------------------------

    def convert_to_nanomaggies(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Converting fields to nanomaggies (in parallel) ...")

        # Loop over the bands
        for band in self.config.bands:

            # Inform the user
            log.info("Computing error maps for the " + band + " band, for each SDSS frame in nanomaggies per pixel ...")

            # Get paths
            fields_path = self.fields_paths[band]
            photofields_path = self.photofield_paths[band]
            counts_path = self.counts_paths[band]
            poisson_path = self.poisson_paths[band]

            # Parallel execution
            with ParallelTarget(convert_frame_to_nanomaggies, self.config.nprocesses) as convert_frame:

                # Loop over the files in the counts path
                for path, name in fs.files_in_path(counts_path, extension="fits", returns=["path", "name"]): convert_frame(path, name, fields_path, photofields_path, poisson_path)

            # Debugging
            print_files_in_path(poisson_path)

    # -----------------------------------------------------------------

    def rebin(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Rebinning frames ...")

        # Loop over the bands
        for band in self.config.bands:

            # Get paths
            fields_path = self.fields_paths[band]
            poisson_path = self.poisson_paths[band]
            rebinned_path = self.rebinned_paths[band]

            # Inform the user
            log.info("Rebinning primary SDSS frames and corresponding error maps to the target coordinate system ...")

            # Parallel execution
            with ParallelTarget(rebin_frame, self.config.nprocesses) as rebin:

                # Loop over the files in the raw directory
                for path, name in fs.files_in_path(fields_path, extension="fits", returns=["path", "name"]): rebin(path, name, poisson_path, self.rebin_wcs, rebinned_path)

            # Debugging
            print_files_in_path(rebinned_path)

    # -----------------------------------------------------------------

    def mosaic(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Performing the mosaicing (in parallel) ...")

        # Parallel execution
        with ParallelTarget(create_mosaic, self.config.nprocesses) as target:

            # Loop over the bands
            for band in self.config.bands:

                # Get paths
                rebinned_path = self.rebinned_paths[band]
                mosaics_path = self.mosaics_paths[band]

                # Call the target function
                target(band, rebinned_path, self.rebin_wcs, mosaics_path)

        # Debugging
        for band in self.config.bands: print_files_in_path(self.mosaics_paths[band])

    # -----------------------------------------------------------------

    def convert_units(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Converting to Jansky ...")

        results = dict()

        # Parallel execution
        with ParallelTarget(convert_mosaic_to_jansky, self.config.nprocesses) as target:

            # Loop over the bands
            for band in self.config.bands:

                # Get path
                mosaics_path = self.mosaics_paths[band]

                # Call the target function
                result = target(mosaics_path)

                # Set result
                results[band] = result

        # Load the mosaics
        for band in self.config.bands:

            # Debugging
            print_files_in_path(self.mosaics_paths[band])

            results[band].request()
            output = results[band].output

            if types.is_string_type(output): result_mosaic_path = output
            else: result_mosaic_path = output[0]

            self.result_paths[band] = result_mosaic_path

    # -----------------------------------------------------------------

    def finish(self):

        """
        This function ...
        :return:
        """

        # Inform the user
        log.info("Finishing ...")

        # Loop over the bands
        for band in self.config.bands:

            # Load the image
            image = Image.from_file(self.result_paths[band])

{'fraction_non_alphanumeric': 0.16975075935460243, 'fraction_numerical': 0.004782318347299713, 'mean_word_length': 1.598343221762006, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4189455', 'n_tokens_mistral': 13257, 'n_tokens_neox': 12343, 'n_words': 3073}