<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""

@author: <NAME>
@contact: <EMAIL>

European Space Astronomy Centre (ESAC)
European Space Agency (ESA)

Created on 3 Sept 2019


"""
import re
from getpass import getpass
from ...utils.tap.core import TapPlus
from ...query import BaseQuery
import shutil
import cgi
from pathlib import Path
import tarfile
import os
from astroquery import log
import configparser

from astropy.io import fits
from . import conf
from astroquery import log
from astropy.coordinates import SkyCoord
from ...exceptions import LoginError

__all__ = ['XMMNewton', 'XMMNewtonClass']


class XMMNewtonClass(BaseQuery):
    data_url = conf.DATA_ACTION
    data_aio_url = conf.DATA_ACTION_AIO
    metadata_url = conf.METADATA_ACTION
    TIMEOUT = conf.TIMEOUT

    def __init__(self, tap_handler=None):
        super(XMMNewtonClass, self).__init__()
        self.configuration = configparser.ConfigParser()

        if tap_handler is None:
            self._tap = TapPlus(url="https://nxsa.esac.esa.int/tap-server/tap")
        else:
            self._tap = tap_handler
        self._rmf_ftp = str("http://sasdev-xmm.esac.esa.int/pub/ccf/constituents/extras/responses/")

    def download_data(self, observation_id, *, filename=None, verbose=False,
                      cache=True, prop=False, credentials_file=None, **kwargs):
        """
        Download data from XMM-Newton

        Parameters
        ----------
        observation_id : string
            id of the observation to be downloaded, mandatory
            The identifier of the observation we want to retrieve, 10 digits
            example: 0144090201
        filename : string
            file name to be used to store the file
        verbose : bool
            optional, default 'False'
            flag to display information about the process
        prop: boolean
            optional, default 'False'
            flag to download proprietary data, the method will then ask the user to
            input their username and password either manually or using the credentials_file
        credentials_file: string
            optional, default None
            path to where the users config.ini file is stored with their username and password
        level : string
            level to download, optional, by default everything is downloaded
            values: ODF, PPS
        instname : string
            instrument name, optional, two characters, by default everything
            values: OM, R1, R2, M1, M2, PN
        instmode : string
            instrument mode, optional
            examples: Fast, FlatFieldLow, Image, PrimeFullWindow
        filter : string
            filter, optional
            examples: Closed, Open, Thick, UVM2, UVW1, UVW2, V
        expflag : string
            exposure flag, optional, by default everything
            values: S, U, X(not applicable)
        expno : integer
            exposure number with 3 digits, by default all exposures
            examples: 001, 003
        name : string
            product type, optional, 6 characters, by default all product types
            examples: 3COLIM, ATTTSR, EVENLI, SBSPEC, EXPMAP, SRCARF
        datasubsetno : character
            data subset number, optional, by default all
            examples: 0, 1
        sourceno : hex value
            source number, optional, by default all sources
            example: 00A, 021, 001
        extension : string
            file format, optional, by default all formats
            values: ASC, ASZ, FTZ, HTM, IND, PDF, PNG

        Returns
        -------
        None if not verbose. It downloads the observation indicated
        If verbose returns the filename
        """

        # create url to access the aio
        link = self._create_link(observation_id, **kwargs)

        # If the user wants to access proprietary data, ask them for there credentials
        if prop:
            username, password = self._get_username_and_password(credentials_file)
            link = f"{link}&AIOUSER={username}&AIOPWD={password}"

        if verbose:
            log.info(link)

        # get response of created url
        params = self._request_link(link, cache)
        r_filename = params["filename"]
        suffixes = Path(r_filename).suffixes

        # get desired filename
        filename = self._create_filename(filename, observation_id, suffixes)

        if prop:
            """
            Here we change the log level so that it is above 20, this is to stop a log.debug (line 431) in query.py. 
            This debug reveals the url being sent which in turn reveals the users username and password
            """
            previouslevel = log.getEffectiveLevel()
            log.setLevel(21)
            self._download_file(link, filename, head_safe=True, cache=cache)
            log.setLevel(previouslevel)
        else:
            self._download_file(link, filename, head_safe=True, cache=cache)

        if verbose:
            log.info(f"Wrote {link} to {filename}")

    def get_postcard(self, observation_id, *, image_type="OBS_EPIC",
                     filename=None, verbose=False):
        """
        Download postcards from XSA

        Parameters
        ----------
        observation_id : string
            id of the observation for which download the postcard, mandatory
            The identifier of the observation we want to retrieve, regardless<commit_msg>Fixing code style<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""

@author: <NAME>
@contact: <EMAIL>

European Space Astronomy Centre (ESAC)
European Space Agency (ESA)

Created on 3 Sept 2019


"""
import re
from getpass import getpass
from ...utils.tap.core import TapPlus
from ...query import BaseQuery
import shutil
import cgi
from pathlib import Path
import tarfile
import os
from astroquery import log
import configparser

from astropy.io import fits
from . import conf
from astroquery import log
from astropy.coordinates import SkyCoord
from ...exceptions import LoginError

__all__ = ['XMMNewton', 'XMMNewtonClass']


class XMMNewtonClass(BaseQuery):
    data_url = conf.DATA_ACTION
    data_aio_url = conf.DATA_ACTION_AIO
    metadata_url = conf.METADATA_ACTION
    TIMEOUT = conf.TIMEOUT

    def __init__(self, tap_handler=None):
        super(XMMNewtonClass, self).__init__()
        self.configuration = configparser.ConfigParser()

        if tap_handler is None:
            self._tap = TapPlus(url="https://nxsa.esac.esa.int/tap-server/tap")
        else:
            self._tap = tap_handler
        self._rmf_ftp = str("http://sasdev-xmm.esac.esa.int/pub/ccf/constituents/extras/responses/")

    def download_data(self, observation_id, *, filename=None, verbose=False,
                      cache=True, prop=False, credentials_file=None, **kwargs):
        """
        Download data from XMM-Newton

        Parameters
        ----------
        observation_id : string
            id of the observation to be downloaded, mandatory
            The identifier of the observation we want to retrieve, 10 digits
            example: 0144090201
        filename : string
            file name to be used to store the file
        verbose : bool
            optional, default 'False'
            flag to display information about the process
        prop: boolean
            optional, default 'False'
            flag to download proprietary data, the method will then ask the user to
            input their username and password either manually or using the credentials_file
        credentials_file: string
            optional, default None
            path to where the users config.ini file is stored with their username and password
        level : string
            level to download, optional, by default everything is downloaded
            values: ODF, PPS
        instname : string
            instrument name, optional, two characters, by default everything
            values: OM, R1, R2, M1, M2, PN
        instmode : string
            instrument mode, optional
            examples: Fast, FlatFieldLow, Image, PrimeFullWindow
        filter : string
            filter, optional
            examples: Closed, Open, Thick, UVM2, UVW1, UVW2, V
        expflag : string
            exposure flag, optional, by default everything
            values: S, U, X(not applicable)
        expno : integer
            exposure number with 3 digits, by default all exposures
            examples: 001, 003
        name : string
            product type, optional, 6 characters, by default all product types
            examples: 3COLIM, ATTTSR, EVENLI, SBSPEC, EXPMAP, SRCARF
        datasubsetno : character
            data subset number, optional, by default all
            examples: 0, 1
        sourceno : hex value
            source number, optional, by default all sources
            example: 00A, 021, 001
        extension : string
            file format, optional, by default all formats
            values: ASC, ASZ, FTZ, HTM, IND, PDF, PNG

        Returns
        -------
        None if not verbose. It downloads the observation indicated
        If verbose returns the filename
        """

        # create url to access the aio
        link = self._create_link(observation_id, **kwargs)

        # If the user wants to access proprietary data, ask them for there credentials
        if prop:
            username, password = self._get_username_and_password(credentials_file)
            link = f"{link}&AIOUSER={username}&AIOPWD={password}"

        if verbose:
            log.info(link)

        # get response of created url
        params = self._request_link(link, cache)
        r_filename = params["filename"]
        suffixes = Path(r_filename).suffixes

        # get desired filename
        filename = self._create_filename(filename, observation_id, suffixes)
        """
        If prop we change the log level so that it is above 20, this is to stop a log.debug (line 431) in query.py.
        This debug reveals the url being sent which in turn reveals the users username and password
        """
        if prop:
            previouslevel = log.getEffectiveLevel()
            log.setLevel(21)
            self._download_file(link, filename, head_safe=True, cache=cache)
            log.setLevel(previouslevel)
        else:
            self._download_file(link, filename, head_safe=True, cache=cache)

        if verbose:
            log.info(f"Wrote {link} to {filename}")

    def get_postcard(self, observation_id, *, image_type="OBS_EPIC",
                     filename=None, verbose=False):
        """
        Download postcards from XSA

        Parameters
        ----------
        observation_id : string
            id of the observation for which download the postcard, mandatory
            The identifier of the observation we want to retrieve, regardless
{'fraction_non_alphanumeric': 0.06991855037979318, 'fraction_numerical': 0.00933467557426558, 'mean_word_length': 2.2028135990621336, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4802498', 'n_tokens_mistral': 2935, 'n_tokens_neox': 2752, 'n_words': 1038}