<commit_before>""" Factory methods to build Data objects from files

Each factory method conforms to the folowing structure, which
helps the GUI Frontend easily load data:

1) The first argument is a file name to open

2) The return value is a Data object

3) The function has a .label attribute that describes (in human
language) what kinds of files it understands

4) The function has a callable .identifier attribute that returns
whether it can handle a requested filename and keyword set

5) The function is added to the __factories__ list

6) Optionally, the function is registered to open a given extension by
default by calling set_default_factory

Putting this together, the simplest data factory code looks like this:

    def dummy_factory(file_name):
        return glue.core.Data()
    dummy_factory.label = "Foo file"
    dummy_factory.identifier = has_extension('foo FOO')
    __factories__.append(dummy_factory)
    set_default_factory("foo", dummy_factory)
"""
import os

import numpy as np

from .data import Component, Data, CategoricalComponent
from .tree import DendroMerge
from .io import extract_data_fits, extract_data_hdf5
from .util import file_format
from .coordinates import coordinates_from_header, coordinates_from_wcs
from ..external.astro import fits


__all__ = ['load_data', 'gridded_data', 'casalike_cube',
           'tabular_data', 'img_data']
__factories__ = []
_default_factory = {}


def as_list(x):
    if isinstance(x, list):
        return x
    return [x]


def _extension(path):
    # split a path into an extension with possibly multipler periods
    #  test.fits -> fits
    #  test.fits.gz -> fits.gz
    _, path = os.path.split(path)
    if '.' not in path:
        return ''
    ext = '.'.join(path.split('.')[1:])
    return ext


def has_extension(exts):
    """
    A simple default filetype identifier function

    It returns a function that tests whether its input
    filename contains a particular extension

    Inputs
    ------
    exts : str
      A space-delimited string listing the extensions
      (e.g., 'txt', or 'txt csv fits')

    Returns
    -------
    A function suitable as a factory identifier function
    """

    def tester(x, **kwargs):
        return _extension(x) in set(exts.split())
    return tester


def is_hdf5(filename):
    # All hdf5 files begin with the same sequence
    with open(filename) as infile:
        return infile.read(8) == '\x89HDF\r\n\x1a\n'


def is_fits(filename):
    try:
        with fits.open(filename):
            return True
    except IOError:
        return False


class LoadLog(object):
    """
    This class attaches some metadata to data created
    from load_data, so that the data can be re-constructed
    when loading saved state. It's only meant to be used
    within load_data
    """
    def __init__(self, path, factory, kwargs):
        self.path = path
        self.factory = factory
        self.kwargs = kwargs
        self.components = []
        self.data = []

    def _log_component(self, component):
        self.components.append(component)

    def _log_data(self, data):
        self.data.append(data)

    def log(self, obj):
        if isinstance(obj, Component):
            self._log_component(obj)<commit_msg>LoadLog uses absolute paths<commit_after>""" Factory methods to build Data objects from files

Each factory method conforms to the folowing structure, which
helps the GUI Frontend easily load data:

1) The first argument is a file name to open

2) The return value is a Data object

3) The function has a .label attribute that describes (in human
language) what kinds of files it understands

4) The function has a callable .identifier attribute that returns
whether it can handle a requested filename and keyword set

5) The function is added to the __factories__ list

6) Optionally, the function is registered to open a given extension by
default by calling set_default_factory

Putting this together, the simplest data factory code looks like this:

    def dummy_factory(file_name):
        return glue.core.Data()
    dummy_factory.label = "Foo file"
    dummy_factory.identifier = has_extension('foo FOO')
    __factories__.append(dummy_factory)
    set_default_factory("foo", dummy_factory)
"""
import os

import numpy as np

from .data import Component, Data, CategoricalComponent
from .tree import DendroMerge
from .io import extract_data_fits, extract_data_hdf5
from .util import file_format
from .coordinates import coordinates_from_header, coordinates_from_wcs
from ..external.astro import fits


__all__ = ['load_data', 'gridded_data', 'casalike_cube',
           'tabular_data', 'img_data']
__factories__ = []
_default_factory = {}


def as_list(x):
    if isinstance(x, list):
        return x
    return [x]


def _extension(path):
    # split a path into an extension with possibly multipler periods
    #  test.fits -> fits
    #  test.fits.gz -> fits.gz
    _, path = os.path.split(path)
    if '.' not in path:
        return ''
    ext = '.'.join(path.split('.')[1:])
    return ext


def has_extension(exts):
    """
    A simple default filetype identifier function

    It returns a function that tests whether its input
    filename contains a particular extension

    Inputs
    ------
    exts : str
      A space-delimited string listing the extensions
      (e.g., 'txt', or 'txt csv fits')

    Returns
    -------
    A function suitable as a factory identifier function
    """

    def tester(x, **kwargs):
        return _extension(x) in set(exts.split())
    return tester


def is_hdf5(filename):
    # All hdf5 files begin with the same sequence
    with open(filename) as infile:
        return infile.read(8) == '\x89HDF\r\n\x1a\n'


def is_fits(filename):
    try:
        with fits.open(filename):
            return True
    except IOError:
        return False


class LoadLog(object):
    """
    This class attaches some metadata to data created
    from load_data, so that the data can be re-constructed
    when loading saved state. It's only meant to be used
    within load_data
    """
    def __init__(self, path, factory, kwargs):
        self.path = os.path.abspath(path)
        self.factory = factory
        self.kwargs = kwargs
        self.components = []
        self.data = []

    def _log_component(self, component):
        self.components.append(component)

    def _log_data(self, data):
        self.data.append(data)

    def log(self, obj):
        if isinstance(obj, Component):
            self._log_component(obj)
{'fraction_non_alphanumeric': 0.0831677113591194, 'fraction_numerical': 0.004280691025837028, 'mean_word_length': 3.0134969325153373, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5972920', 'n_tokens_mistral': 1906, 'n_tokens_neox': 1850, 'n_words': 702}