<commit_before># -*- mode: python; coding: utf-8 -*-
# Copyright (c) 2020 Radio Astronomy Software Group
# Licensed under the 2-clause BSD License

"""Module for low-level interface to MIR files.

This module extracts data types associated with MIR files.

"""
import numpy as np
import os
import copy

__all__ = ["MirParser"]

# MIR structure definitions.
in_dtype = np.dtype(
    [
        ("traid", np.int32),
        ("inhid", np.int32),
        ("ints", np.int32),
        ("az", np.single),
        ("el", np.single),
        ("ha", np.single),
        ("iut", np.int16),
        ("iref_time", np.int16),
        ("dhrs", np.double),
        ("vc", np.single),
        ("sx", np.double),
        ("sy", np.double),
        ("sz", np.double),
        ("rinteg", np.single),
        ("proid", np.int32),
        ("souid", np.int32),
        ("isource", np.int16),
        ("ivrad", np.int16),
        ("offx", np.single),
        ("offy", np.single),
        ("ira", np.int16),
        ("idec", np.int16),
        ("rar", np.double),
        ("decr", np.double),
        ("epoch", np.single),
        ("size", np.single),
        ("vrra", np.single),
        ("vrdec", np.single),
        ("lst", np.single),
        ("iproject", np.int16),
        ("tile", np.int16),
        ("obsmode", np.uint8),
        ("obsflag", np.uint8),
        ("spareshort", np.int16),
        ("spareint6", np.int32),
        ("yIGFreq1", np.double),
        ("yIGFreq2", np.double),
        ("sflux", np.double),
        ("ara", np.double),
        ("adec", np.double),
        ("mjd", np.double),
    ]
)

eng_dtype = np.dtype(
    [
        ("antennaNumber", np.int32),
        ("padNumber", np.int32),
        ("antennaStatus", np.int32),
        ("trackStatus", np.int32),
        ("commStatus", np.int32),
        ("inhid", np.int32),
        ("ints", np.int32),
        ("dhrs", np.double),
        ("ha", np.double),
        ("lst", np.double),
        ("pmdaz", np.double),
        ("pmdel", np.double),
        ("tiltx", np.double),
        ("tilty", np.double),
        ("actual_az", np.double),
        ("actual_el", np.double),
        ("azoff", np.double),
        ("eloff", np.double),
        ("az_tracking_error", np.double),
        ("el_tracking_error", np.double),
        ("refraction", np.double),
        ("chopper_x", np.double),
        ("chopper_y", np.double),
        ("chopper_z", np.double),
        ("chopper_angle", np.double),
        ("tsys", np.double),
        ("tsys_rx2", np.double),
        ("ambient_load_temperature", np.double),
    ]
)

bl_dtype = np.dtype(
    [
        ("blhid", np.int32),
        ("inhid", np.int32),
        ("isb", np.int16),
        ("ipol", np.int16),
        ("ant1rx", np.int16),
        ("ant2rx", np.int16),
        ("pointing", np.int16),
        ("irec", np.int16),
        ("u", np.single),
        ("v", np.single),
        ("w", np.single),
        ("prbl", np.single),
        ("coh", np.single),
        ("avedhrs", np.double),
        ("ampave", np.single),
        ("phaave", np.single),
        ("blsid", np.int32),
        ("iant1", np.int16),
        ("iant2", np.int16),
        ("ant1TsysOff", np.int32),
        ("ant2TsysOff", np.int32),
        ("iblcd", np.int16),
        ("ble", np.single),
        ("bln", np.single),
        ("blu", np.single),
        ("spareint1", np.int32),
        ("spareint2", np.int32),
        ("spareint3", np.int32),
        ("spareint4", np.int32),
        ("spareint5", np.int32),
        ("spareint6", np.int32),
        ("fave", np.double),
        ("bwave", np.double),
        ("sparedbl3", np.double),
        ("sparedbl4", np.double),
        ("sparedbl5", np.double),
        ("sparedbl6", np.double),
    ]
)

sp_dtype = np.dtype(
    [
        ("sphid", np.int32),
        ("blhid", np.int32),
        ("inhid", np.int32),
        ("igq", np.int16),
        ("ipq", np.int16),
        ("iband", np.int16),
        ("ipstate", np.int16),
        ("tau0", np.single),
        ("vel", np.double),
        ("vres", np.single),
        ("fsky", np.double),
        ("fres", np.single),
        ("gunnLO", np.double),
        ("cabinLO", np.double),
        ("corrLO1", np.double),
        ("corrLO2", np.double),
        ("integ", np.single),
        ("wt", np.single),
        ("flags", np.int32),
        ("vradcat", np.single),
        ("nch", np.int16),
        ("nrec", np.int16),
        ("dataoff", np.int32),
        ("rfreq", np.double),
        ("corrblock", np.int16),
        ("corrchunk", np.int16),
        ("correlator", np.int32),
        ("spareint2", np.int32),
        ("spareint3", np.int32),
        ("spareint4", np.int32),
        ("spareint5", np.int32),
        ("spareint6", np.int32),
        ("sparedbl1", np.double),
        ("sparedbl2", np.double),
        ("sparedbl3", np.double),
        ("sparedbl4", np.double),
        ("sparedbl5", np.double),
        ("sparedbl6", np.double),
    ]
)

codes_dtype = np.dtype(
    [("v_name", "S12"), ("icode", np.int16), ("code", "S26"), ("ncode", np.int16)]
)

we_dtype = np.dtype(
    [
        ("scanNumber", np.int32),
        ("flags", np.int32, 11),
        ("N", np.single, 11),
        ("Tamb", np.single, 11),
        ("pressure", np.single, 11),
        ("humid", np.single, 11),
        ("windSpeed", np.single, 11),
        ("windDir", np.single, 11),
        ("h2o", np.single, 11),
    ]
)

ac_read_dtype = np.dtype(
    [
        ("inhid", np.int32),
        ("achid", np.int32),
        ("antenna", np.int32),
        ("nchunks", np.int32),
        ("datasize", np.int32),
        ("dataoff", np.int64),
        ("dhrs", np.double),
    ]
)

antpos_dtype = np.dtype([("antenna", np.int16), ("xyz_pos", np.double, 3)])


class MirParser(object):
    """General class for Mir datasets.

    Does lots of cool things.
    """

    def __init__(self, filepath, load_vis=False, load_raw=False, load_auto=False):
        """
        Read in all files from a mir data set into predefined numpy datatypes.

        Parameters
        ----------
        filepath : str
            filepath is the path to the folder containing the mir data set.
        """
        self.filepath = filepath
        self.in_read = self.read_in_data(filepath)
        self.eng_read = self.read_eng_data(filepath)<commit_msg>Added keyword param comments for the constructor for MirParser.<commit_after># -*- mode: python; coding: utf-8 -*-
# Copyright (c) 2020 Radio Astronomy Software Group
# Licensed under the 2-clause BSD License

"""Module for low-level interface to MIR files.

This module extracts data types associated with MIR files.

"""
import numpy as np
import os
import copy

__all__ = ["MirParser"]

# MIR structure definitions.
in_dtype = np.dtype(
    [
        ("traid", np.int32),
        ("inhid", np.int32),
        ("ints", np.int32),
        ("az", np.single),
        ("el", np.single),
        ("ha", np.single),
        ("iut", np.int16),
        ("iref_time", np.int16),
        ("dhrs", np.double),
        ("vc", np.single),
        ("sx", np.double),
        ("sy", np.double),
        ("sz", np.double),
        ("rinteg", np.single),
        ("proid", np.int32),
        ("souid", np.int32),
        ("isource", np.int16),
        ("ivrad", np.int16),
        ("offx", np.single),
        ("offy", np.single),
        ("ira", np.int16),
        ("idec", np.int16),
        ("rar", np.double),
        ("decr", np.double),
        ("epoch", np.single),
        ("size", np.single),
        ("vrra", np.single),
        ("vrdec", np.single),
        ("lst", np.single),
        ("iproject", np.int16),
        ("tile", np.int16),
        ("obsmode", np.uint8),
        ("obsflag", np.uint8),
        ("spareshort", np.int16),
        ("spareint6", np.int32),
        ("yIGFreq1", np.double),
        ("yIGFreq2", np.double),
        ("sflux", np.double),
        ("ara", np.double),
        ("adec", np.double),
        ("mjd", np.double),
    ]
)

eng_dtype = np.dtype(
    [
        ("antennaNumber", np.int32),
        ("padNumber", np.int32),
        ("antennaStatus", np.int32),
        ("trackStatus", np.int32),
        ("commStatus", np.int32),
        ("inhid", np.int32),
        ("ints", np.int32),
        ("dhrs", np.double),
        ("ha", np.double),
        ("lst", np.double),
        ("pmdaz", np.double),
        ("pmdel", np.double),
        ("tiltx", np.double),
        ("tilty", np.double),
        ("actual_az", np.double),
        ("actual_el", np.double),
        ("azoff", np.double),
        ("eloff", np.double),
        ("az_tracking_error", np.double),
        ("el_tracking_error", np.double),
        ("refraction", np.double),
        ("chopper_x", np.double),
        ("chopper_y", np.double),
        ("chopper_z", np.double),
        ("chopper_angle", np.double),
        ("tsys", np.double),
        ("tsys_rx2", np.double),
        ("ambient_load_temperature", np.double),
    ]
)

bl_dtype = np.dtype(
    [
        ("blhid", np.int32),
        ("inhid", np.int32),
        ("isb", np.int16),
        ("ipol", np.int16),
        ("ant1rx", np.int16),
        ("ant2rx", np.int16),
        ("pointing", np.int16),
        ("irec", np.int16),
        ("u", np.single),
        ("v", np.single),
        ("w", np.single),
        ("prbl", np.single),
        ("coh", np.single),
        ("avedhrs", np.double),
        ("ampave", np.single),
        ("phaave", np.single),
        ("blsid", np.int32),
        ("iant1", np.int16),
        ("iant2", np.int16),
        ("ant1TsysOff", np.int32),
        ("ant2TsysOff", np.int32),
        ("iblcd", np.int16),
        ("ble", np.single),
        ("bln", np.single),
        ("blu", np.single),
        ("spareint1", np.int32),
        ("spareint2", np.int32),
        ("spareint3", np.int32),
        ("spareint4", np.int32),
        ("spareint5", np.int32),
        ("spareint6", np.int32),
        ("fave", np.double),
        ("bwave", np.double),
        ("sparedbl3", np.double),
        ("sparedbl4", np.double),
        ("sparedbl5", np.double),
        ("sparedbl6", np.double),
    ]
)

sp_dtype = np.dtype(
    [
        ("sphid", np.int32),
        ("blhid", np.int32),
        ("inhid", np.int32),
        ("igq", np.int16),
        ("ipq", np.int16),
        ("iband", np.int16),
        ("ipstate", np.int16),
        ("tau0", np.single),
        ("vel", np.double),
        ("vres", np.single),
        ("fsky", np.double),
        ("fres", np.single),
        ("gunnLO", np.double),
        ("cabinLO", np.double),
        ("corrLO1", np.double),
        ("corrLO2", np.double),
        ("integ", np.single),
        ("wt", np.single),
        ("flags", np.int32),
        ("vradcat", np.single),
        ("nch", np.int16),
        ("nrec", np.int16),
        ("dataoff", np.int32),
        ("rfreq", np.double),
        ("corrblock", np.int16),
        ("corrchunk", np.int16),
        ("correlator", np.int32),
        ("spareint2", np.int32),
        ("spareint3", np.int32),
        ("spareint4", np.int32),
        ("spareint5", np.int32),
        ("spareint6", np.int32),
        ("sparedbl1", np.double),
        ("sparedbl2", np.double),
        ("sparedbl3", np.double),
        ("sparedbl4", np.double),
        ("sparedbl5", np.double),
        ("sparedbl6", np.double),
    ]
)

codes_dtype = np.dtype(
    [("v_name", "S12"), ("icode", np.int16), ("code", "S26"), ("ncode", np.int16)]
)

we_dtype = np.dtype(
    [
        ("scanNumber", np.int32),
        ("flags", np.int32, 11),
        ("N", np.single, 11),
        ("Tamb", np.single, 11),
        ("pressure", np.single, 11),
        ("humid", np.single, 11),
        ("windSpeed", np.single, 11),
        ("windDir", np.single, 11),
        ("h2o", np.single, 11),
    ]
)

ac_read_dtype = np.dtype(
    [
        ("inhid", np.int32),
        ("achid", np.int32),
        ("antenna", np.int32),
        ("nchunks", np.int32),
        ("datasize", np.int32),
        ("dataoff", np.int64),
        ("dhrs", np.double),
    ]
)

antpos_dtype = np.dtype([("antenna", np.int16), ("xyz_pos", np.double, 3)])


class MirParser(object):
    """General class for Mir datasets.

    Does lots of cool things.
    """

    def __init__(self, filepath, load_vis=False, load_raw=False, load_auto=False):
        """
        Read in all files from a mir data set into predefined numpy datatypes.

        The full dataset can be quite large, as such the default behavior of
        this function is only to load the metadata. Use the keyword params to
        load other data into memory.

        Parameters
        ----------
        filepath : str
            filepath is the path to the folder containing the mir data set.
        load_vis : bool
            flag to load visibilities into memory, default is False.
        load_raw : bool
            flag to load raw data into memory, default is False.
        load_auto : bool
            flag to load auto-correlations into memory, default is False.
        """
        self.filepath = filepath
        self.in_read = self.read_in_data(filepath)
        self.eng_read = self.read_eng_data(filepath)
{'fraction_non_alphanumeric': 0.20021347971942666, 'fraction_numerical': 0.03171698688624581, 'mean_word_length': 1.9403721138758125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4527857', 'n_tokens_mistral': 4785, 'n_tokens_neox': 4571, 'n_words': 982}