<commit_before>from __future__ import (print_function, division, absolute_import,
                        unicode_literals)

import fnmatch
from os import listdir, path
import logging

import numpy as np
import numpy.ma as ma

from astropy.table import Table
import astropy.io.fits as fits
from astropy.extern import six

import warnings
from astropy.utils.exceptions import AstropyUserWarning

from .ccddata import fits_ccddata_reader

logger = logging.getLogger(__name__)

__all__ = ['ImageFileCollection']
__doctest_skip__ = ['*']


class ImageFileCollection(object):
    """
    Representation of a collection of image files.

    The class offers a table summarizing values of
    keywords in the FITS headers of the files in the collection and offers
    convenient methods for iterating over the files in the collection. The
    generator methods use simple filtering syntax and can automate storage
    of any FITS files modified in the loop using the generator.

    Parameters
    ----------
    location : str or None, optional
        Path to directory containing FITS files.
        Default is ``None``.

    keywords : list of str, '*' or None, optional
        Keywords that should be used as column headings in the summary table.
        If the value is or includes '*' then all keywords that appear in any
        of the FITS headers of the files in the collection become table
        columns. Default value is '*' unless ``info_file`` is specified.
        Default is ``None``.

    info_file : str or None, optional
        Path to file that contains a table of information about FITS files.
        In this case the keywords are set to the names of the columns of the
        ``info_file`` unless ``keywords`` is explicitly set to a different
        list.
        Default is ``None``.

    filenames: str, list of str, or None, optional
        List of the names of FITS files which will be added to the collection.
        The filenames are assumed to be in ``location``.
        Default is ``None``.

    Raises
    ------
    ValueError
        Raised if keywords are set to a combination of '*' and any other
        value.
    """
    def __init__(self, location=None, keywords=None, info_file=None,
                 filenames=None):
        self._location = location
        self._filenames = filenames
        self._files = []
        if location:
            self._files = self._get_files()

        if self._files == []:
            warnings.warn("no FITS files in the collection.",
                          AstropyUserWarning)
        self._summary_info = {}
        if keywords is None:
            if info_file is not None:
                # Default to empty list so that keywords will be populated
                # from table columns names.
                keywords = []
            else:
                # Otherwise use all keywords.
                keywords = '*'
        if info_file is not None:
            try:
                info_path = path.join(self.location, info_file)
            except (AttributeError, TypeError):
                info_path = info_file
            try:
                self._summary_info = Table.read(info_path,
                                                format='ascii',
                                                delimiter=',')
                self._summary_info = Table(self._summary_info,
                                           masked=True)
            except IOError:
                if location:
                    logger.warning('unable to open table file %s, will try '
                                   'initializing from location instead.',
                                   info_path)
                else:
                    raise

        # Used internally to keep track of whether the user asked for all
        # keywords or a specific list. The keywords setter takes care of
        # actually setting the correct value, this just ensure that there
        # is always *some* value.
        self._all_keywords = False

        if keywords:
            self.keywords = keywords

    @property
    def summary(self):
        """
        `~astropy.table.Table` of values of FITS keywords for files in the
        collection.

        Each keyword is a column heading. In addition, there is a column
        called ``file`` that contains the name of the FITS file. The directory
        is not included as part of that name.

        The first column is always named ``file``.

        The order of the remaining columns depends on how the summary was
        constructed.

        If a wildcard, ``*`` was used then the order is the order in which
        the keywords appear in the FITS files from which the summary is
        constructed.

        If an explicit list of keywords was supplied in setting up the
        collection then the order of the columns is the order of the
        keywords.
        """
        return self._summary_info

    @property
    def summary_info(self):
        """
        Deprecated -- use `summary` instead -- `~astropy.table.Table` of values
        of FITS keywords for files in the collection.

        Each keyword is a column heading. In addition, there is a column
        called 'file' that contains the name of the FITS file. The directory
        is not included as part of that name.
        """
        return self._summary_info

    @property
    def location(self):
        """
        str, Path name to directory containing FITS files.
        """
        return self._location

    @property
    def keywords(self):
        """
        list of str, Keywords currently in the summary table.

        Setting the keywords causes the summary table to be regenerated unless
        the new keywords are a subset of the old.
        """
        if self.summary_info:
            return self.summary_info.keys()
        else:
            return []

    @keywords.setter
    def keywords(self, keywords=None):
        # since keywords are drawn from self.summary_info, setting
        # summary_info sets the keywords.
        if keywords is None:
            self._summary_info = []
            return

        if keywords == '*':
            self._all_keywords = True
        else:
            self._all_keywords = False

        logging.debug('keywords in setter before pruning: %s.', keywords)

        # remove duplicates and force a copy
        new_keys = list(set(keywords))

        logging.debug('keywords after pruning %s.', new_keys)

        full_new_keys = list(set(new_keys))
        full_new_keys.append('file')
        full_new_set = set(full_new_keys)
        current_set = set(self.keywords)
        if full_new_set.issubset(current_set):
            logging.debug('table columns before trimming: %s.',
                          ' '.join(current_set))
            cut_keys = current_set.difference(full_new_set)
            logging.debug('will try removing columns: %s.',
                          ' '.join(cut_keys))
            for key in cut_keys:
                self._summary_info.remove_column(key)
            logging.debug('after removal column names are: %s.',
                          ' '.join(self.keywords))
        else:
            logging.debug('should be building new table...')
            # Reorder the keywords to match the initial ordering.
            new_keys.sort(key=keywords.index)
            self._summary_info = self._fits_summary(header_keywords=new_keys)

    @property
    def files(self):
        """
        list of str, Unfiltered list of FITS files in location.
        """
        return self._files

    def values(self, keyword, unique=False):
        """
        List of values for a keyword.

        Parameters
        ----------
        keyword : str
            Keyword (i.e. table column) for which values are desired.<commit_msg>removed several list->set or set->list operations in imagefilecollection.keywords setter<commit_after>from __future__ import (print_function, division, absolute_import,
                        unicode_literals)

import fnmatch
from os import listdir, path
import logging

import numpy as np
import numpy.ma as ma

from astropy.table import Table
import astropy.io.fits as fits
from astropy.extern import six

import warnings
from astropy.utils.exceptions import AstropyUserWarning

from .ccddata import fits_ccddata_reader

logger = logging.getLogger(__name__)

__all__ = ['ImageFileCollection']
__doctest_skip__ = ['*']


class ImageFileCollection(object):
    """
    Representation of a collection of image files.

    The class offers a table summarizing values of
    keywords in the FITS headers of the files in the collection and offers
    convenient methods for iterating over the files in the collection. The
    generator methods use simple filtering syntax and can automate storage
    of any FITS files modified in the loop using the generator.

    Parameters
    ----------
    location : str or None, optional
        Path to directory containing FITS files.
        Default is ``None``.

    keywords : list of str, '*' or None, optional
        Keywords that should be used as column headings in the summary table.
        If the value is or includes '*' then all keywords that appear in any
        of the FITS headers of the files in the collection become table
        columns. Default value is '*' unless ``info_file`` is specified.
        Default is ``None``.

    info_file : str or None, optional
        Path to file that contains a table of information about FITS files.
        In this case the keywords are set to the names of the columns of the
        ``info_file`` unless ``keywords`` is explicitly set to a different
        list.
        Default is ``None``.

    filenames: str, list of str, or None, optional
        List of the names of FITS files which will be added to the collection.
        The filenames are assumed to be in ``location``.
        Default is ``None``.

    Raises
    ------
    ValueError
        Raised if keywords are set to a combination of '*' and any other
        value.
    """
    def __init__(self, location=None, keywords=None, info_file=None,
                 filenames=None):
        self._location = location
        self._filenames = filenames
        self._files = []
        if location:
            self._files = self._get_files()

        if self._files == []:
            warnings.warn("no FITS files in the collection.",
                          AstropyUserWarning)
        self._summary_info = {}
        if keywords is None:
            if info_file is not None:
                # Default to empty list so that keywords will be populated
                # from table columns names.
                keywords = []
            else:
                # Otherwise use all keywords.
                keywords = '*'
        if info_file is not None:
            try:
                info_path = path.join(self.location, info_file)
            except (AttributeError, TypeError):
                info_path = info_file
            try:
                self._summary_info = Table.read(info_path,
                                                format='ascii',
                                                delimiter=',')
                self._summary_info = Table(self._summary_info,
                                           masked=True)
            except IOError:
                if location:
                    logger.warning('unable to open table file %s, will try '
                                   'initializing from location instead.',
                                   info_path)
                else:
                    raise

        # Used internally to keep track of whether the user asked for all
        # keywords or a specific list. The keywords setter takes care of
        # actually setting the correct value, this just ensure that there
        # is always *some* value.
        self._all_keywords = False

        if keywords:
            self.keywords = keywords

    @property
    def summary(self):
        """
        `~astropy.table.Table` of values of FITS keywords for files in the
        collection.

        Each keyword is a column heading. In addition, there is a column
        called ``file`` that contains the name of the FITS file. The directory
        is not included as part of that name.

        The first column is always named ``file``.

        The order of the remaining columns depends on how the summary was
        constructed.

        If a wildcard, ``*`` was used then the order is the order in which
        the keywords appear in the FITS files from which the summary is
        constructed.

        If an explicit list of keywords was supplied in setting up the
        collection then the order of the columns is the order of the
        keywords.
        """
        return self._summary_info

    @property
    def summary_info(self):
        """
        Deprecated -- use `summary` instead -- `~astropy.table.Table` of values
        of FITS keywords for files in the collection.

        Each keyword is a column heading. In addition, there is a column
        called 'file' that contains the name of the FITS file. The directory
        is not included as part of that name.
        """
        return self._summary_info

    @property
    def location(self):
        """
        str, Path name to directory containing FITS files.
        """
        return self._location

    @property
    def keywords(self):
        """
        list of str, Keywords currently in the summary table.

        Setting the keywords causes the summary table to be regenerated unless
        the new keywords are a subset of the old.
        """
        if self.summary_info:
            return self.summary_info.keys()
        else:
            return []

    @keywords.setter
    def keywords(self, keywords=None):
        # since keywords are drawn from self.summary_info, setting
        # summary_info sets the keywords.
        if keywords is None:
            self._summary_info = []
            return

        if keywords == '*':
            self._all_keywords = True
        else:
            self._all_keywords = False

        logging.debug('keywords in setter before pruning: %s.', keywords)

        # remove duplicates and force a copy so we can sort the items later
        # by their given position.
        new_keys_set = set(keywords)
        new_keys_lst = list(new_keys_set)
        new_keys_set.add('file')

        logging.debug('keywords after pruning %s.', new_keys_lst)

        current_set = set(self.keywords)
        if new_keys_set.issubset(current_set):
            logging.debug('table columns before trimming: %s.',
                          ' '.join(current_set))
            cut_keys = current_set.difference(new_keys_set)
            logging.debug('will try removing columns: %s.',
                          ' '.join(cut_keys))
            for key in cut_keys:
                self._summary_info.remove_column(key)
            logging.debug('after removal column names are: %s.',
                          ' '.join(self.keywords))
        else:
            logging.debug('should be building new table...')
            # Reorder the keywords to match the initial ordering.
            new_keys_lst.sort(key=keywords.index)
            self._summary_info = self._fits_summary(new_keys_lst)

    @property
    def files(self):
        """
        list of str, Unfiltered list of FITS files in location.
        """
        return self._files

    def values(self, keyword, unique=False):
        """
        List of values for a keyword.

        Parameters
        ----------
        keyword : str
            Keyword (i.e. table column) for which values are desired.
{'fraction_non_alphanumeric': 0.06794391930470088, 'fraction_numerical': 0.0, 'mean_word_length': 1.8332135154565061, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3597478', 'n_tokens_mistral': 3896, 'n_tokens_neox': 3746, 'n_words': 1564}