<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Routines to read galaxy catalogs from disk.
"""

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from os.path import dirname, abspath, splitext, exists as file_exists,\
    join as pjoin
import numpy as np

try:
    from future.utils import bytes_to_native_str
except ImportError:
    print("\n\tPlease run python setup.py install before using "
          "the 'Corrfunc' package\n")
    raise


def read_fastfood_catalog(filename, return_dtype=None, need_header=None):
    """
    Read a galaxy catalog from a fast-food binary file.

    Parameters
    -----------
    filename: string
        Filename containing the galaxy positions

    return_dtype: numpy dtype for returned arrays. Default ``numpy.float``
        Specifies the datatype for the returned arrays. Must be in
        {np.float, np.float32}

    need_header: boolean, default None.
        Returns the header found in the fast-food file in addition to the
        X/Y/Z arrays.

    Returns
    --------

    X, Y, Z: numpy arrays
        Returns the triplet of X/Y/Z positions as separate numpy arrays.

        If need_header is set, then the header is also returned

    Example
    --------
    >>> import numpy as np
    >>> from os.path import dirname, abspath, join as pjoin
    >>> import Corrfunc
    >>> from Corrfunc.io import read_fastfood_catalog
    >>> filename = pjoin(dirname(abspath(Corrfunc.__file__)),
    ...                  "../theory/tests/data/",
    ...                  "gals_Mr19.ff")
    >>> X, Y, Z = read_fastfood_catalog(filename)
    >>> N = 20
    >>> for x,y,z in zip(X[0:N], Y[0:N], Z[0:]):
    ...     print("{0:10.5f} {1:10.5f} {2:10.5f}".format(x, y, z))
    ...     # doctest: +NORMALIZE_WHITESPACE
    419.94550    1.96340    0.01610
    419.88272    1.79736    0.11960
    0.32880   10.63620    4.16550
    0.15314   10.68723    4.06529
    0.46400    8.91150    6.97090
    6.30690    9.77090    8.61080
    5.87160    9.65870    9.29810
    8.06210    0.42350    4.89410
    11.92830    4.38660    4.54410
    11.95543    4.32622    4.51485
    11.65676    4.34665    4.53181
    11.75739    4.26262    4.31666
    11.81329    4.27530    4.49183
    11.80406    4.54737    4.26824
    12.61570    4.14470    3.70140
    13.23640    4.34750    5.26450
    13.19833    4.33196    5.29435
    13.21249    4.35695    5.37418
    13.06805    4.24275    5.35126
    13.19693    4.37618    5.28772

    """
    if return_dtype is None:
        return_dtype = np.float

    if return_dtype not in [np.float32, np.float]:
        msg = "Return data-type must be set and a valid numpy float"
        raise ValueError(msg)

    if not file_exists(filename):
        msg = "Could not find file = {0}".format(filename)
        raise IOError(msg)

    import struct
    with open(filename, "rb") as f:
        skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
        idat = struct.unpack(bytes_to_native_str(b'@iiiii'),
                             f.read(20))[0:5]
        skip2 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
        assert skip1 == 20 and skip2 == 20,\
            "fast-food file seems to be incorrect (reading idat)"
        ngal = idat[1]

        if need_header is not None:
            # now read fdat
            skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
            fdat = struct.unpack(bytes_to_native_str(b'@fffffffff'),
                                 f.read(36))[0:9]
            skip2 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
            assert skip1 == 36 and skip2 == 36,\
                "fast-food file seems to be incorrect (reading fdat )"

            skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
            znow = struct.unpack(bytes_to_native_str(b'@f'), f.read(4))[0]
            skip2 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
            assert skip1 == 4 and skip2 == 4,\
                "fast-food file seems to be incorrect (reading redshift)"<commit_msg>Moved import from future package into the relevant function<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Routines to read galaxy catalogs from disk.
"""

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from os.path import dirname, abspath, splitext, exists as file_exists,\
    join as pjoin
import numpy as np


def read_fastfood_catalog(filename, return_dtype=None, need_header=None):
    """
    Read a galaxy catalog from a fast-food binary file.

    Parameters
    -----------
    filename: string
        Filename containing the galaxy positions

    return_dtype: numpy dtype for returned arrays. Default ``numpy.float``
        Specifies the datatype for the returned arrays. Must be in
        {np.float, np.float32}

    need_header: boolean, default None.
        Returns the header found in the fast-food file in addition to the
        X/Y/Z arrays.

    Returns
    --------

    X, Y, Z: numpy arrays
        Returns the triplet of X/Y/Z positions as separate numpy arrays.

        If need_header is set, then the header is also returned

    Example
    --------
    >>> import numpy as np
    >>> from os.path import dirname, abspath, join as pjoin
    >>> import Corrfunc
    >>> from Corrfunc.io import read_fastfood_catalog
    >>> filename = pjoin(dirname(abspath(Corrfunc.__file__)),
    ...                  "../theory/tests/data/",
    ...                  "gals_Mr19.ff")
    >>> X, Y, Z = read_fastfood_catalog(filename)
    >>> N = 20
    >>> for x,y,z in zip(X[0:N], Y[0:N], Z[0:]):
    ...     print("{0:10.5f} {1:10.5f} {2:10.5f}".format(x, y, z))
    ...     # doctest: +NORMALIZE_WHITESPACE
    419.94550    1.96340    0.01610
    419.88272    1.79736    0.11960
    0.32880   10.63620    4.16550
    0.15314   10.68723    4.06529
    0.46400    8.91150    6.97090
    6.30690    9.77090    8.61080
    5.87160    9.65870    9.29810
    8.06210    0.42350    4.89410
    11.92830    4.38660    4.54410
    11.95543    4.32622    4.51485
    11.65676    4.34665    4.53181
    11.75739    4.26262    4.31666
    11.81329    4.27530    4.49183
    11.80406    4.54737    4.26824
    12.61570    4.14470    3.70140
    13.23640    4.34750    5.26450
    13.19833    4.33196    5.29435
    13.21249    4.35695    5.37418
    13.06805    4.24275    5.35126
    13.19693    4.37618    5.28772

    """
    if return_dtype is None:
        return_dtype = np.float

    if return_dtype not in [np.float32, np.float]:
        msg = "Return data-type must be set and a valid numpy float"
        raise ValueError(msg)

    if not file_exists(filename):
        msg = "Could not find file = {0}".format(filename)
        raise IOError(msg)

    import struct
    try:
        from future.utils import bytes_to_native_str
    except ImportError:
        print("\n\tPlease run python setup.py install before using "
              "the 'Corrfunc' package\n")
        raise
    
    with open(filename, "rb") as f:
        skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
        idat = struct.unpack(bytes_to_native_str(b'@iiiii'),
                             f.read(20))[0:5]
        skip2 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
        assert skip1 == 20 and skip2 == 20,\
            "fast-food file seems to be incorrect (reading idat)"
        ngal = idat[1]

        if need_header is not None:
            # now read fdat
            skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
            fdat = struct.unpack(bytes_to_native_str(b'@fffffffff'),
                                 f.read(36))[0:9]
            skip2 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
            assert skip1 == 36 and skip2 == 36,\
                "fast-food file seems to be incorrect (reading fdat )"

            skip1 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
            znow = struct.unpack(bytes_to_native_str(b'@f'), f.read(4))[0]
            skip2 = struct.unpack(bytes_to_native_str(b'@i'), f.read(4))[0]
            assert skip1 == 4 and skip2 == 4,\
                "fast-food file seems to be incorrect (reading redshift)"
{'fraction_non_alphanumeric': 0.11929908785405664, 'fraction_numerical': 0.10753720595295248, 'mean_word_length': 2.005048683736026, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1013276', 'n_tokens_mistral': 3327, 'n_tokens_neox': 2840, 'n_words': 729}