<commit_before>
'''
Match and combine the re-weighted HI MSs

The 14B frequency range includes 2000 channels. That same range in the 17B
data is 2006 channels. So we first regrid and split the data over the same
velocity range before concatenating. The original channels are -206 and
change m/s. Regrid to something common like -210 m/s
'''

import numpy as np

from tasks import mstransform, virtualconcat, rmtables

use_contsub = True

if use_contsub:
    fourteenB_ms = "14B-088_HI_LSRK.ms.contsub"
    seventeenB_ms = "17B-162_HI_spw_0_LSRK.mms.contsub"

    concat_vis = "14B_17B_HI_LSRK.mms.contsub"

else:

    fourteenB_ms = "14B-088_HI_LSRK.ms"
    seventeenB_ms = "17B-162_HI_spw_0_LSRK.mms"

    concat_vis = "14B_17B_HI_LSRK.mms"

# All in km/s
chan_width = 0.21
start_vel = -330
end_vel = -50

# ~1334
n_chan = int(np.ceil((end_vel - start_vel) / chan_width))

seventeenB_ms_regrid = "{}.regrid".format(seventeenB_ms)

mstransform(vis=seventeenB_ms,
            output_vis=seventeenB_ms_regrid,
            spw='0', regridms=True, mode='velocity', veltype='radio',
            start="{}km/s".format(start_vel),
            width="{}km/s".format(chan_width), nchan=n_chan,
            interpolation='fftshift')

# Now the 14B data. Only keep the fields used in the 17B data
fourteenB_ms_regrid = "{}.regrid".format(fourteenB_ms)

# Also convert th 14B data to an MMS w/ the same number of sub-MS as 17B
mstransform(vis=fourteenB_ms,
            output_vis=fourteenB_ms_regrid,
            field='M33_2,M33_3,M33_6,M33_7_center,M33_8,M33_11,M33_12,M33_14',
            spw='0', regridms=True, mode='velocity', veltype='radio',
            start="{}km/s".format(start_vel),
            width="{}km/s".format(chan_width), nchan=n_chan,
            interpolation='fftshift',
            createmms=True, separationaxis='auto', numsubms=31)

# concatenate together, and will also remove the individual MS
virtualconcat(vis=[fourteenB_ms_regrid, seventeenB_ms_regrid],
              concatvis=concat_vis)<commit_msg>Fix kwarg and only keep fields included in the 17B data<commit_after>
'''
Match and combine the re-weighted HI MSs

The 14B frequency range includes 2000 channels. That same range in the 17B
data is 2006 channels. So we first regrid and split the data over the same
velocity range before concatenating. The original channels are -206 and
change m/s. Regrid to something common like -210 m/s
'''

import numpy as np

from tasks import mstransform, virtualconcat, rmtables

use_contsub = True

if use_contsub:
    fourteenB_ms = "14B-088_HI_LSRK.ms.contsub"
    seventeenB_ms = "17B-162_HI_spw_0_LSRK.mms.contsub"

    concat_vis = "14B_17B_HI_LSRK.mms.contsub"

else:

    fourteenB_ms = "14B-088_HI_LSRK.ms"
    seventeenB_ms = "17B-162_HI_spw_0_LSRK.mms"

    concat_vis = "14B_17B_HI_LSRK.mms"

# All in km/s
chan_width = 0.21
start_vel = -330
end_vel = -50

# ~1334
n_chan = int(np.ceil((end_vel - start_vel) / chan_width))

seventeenB_ms_regrid = "{}.regrid".format(seventeenB_ms)

mstransform(vis=seventeenB_ms,
            outputvis=seventeenB_ms_regrid,
            spw='0', regridms=True, mode='velocity', veltype='radio',
            start="{}km/s".format(start_vel),
            width="{}km/s".format(chan_width), nchan=n_chan,
            interpolation='fftshift')

# Now the 14B data. Only keep the fields used in the 17B data
fourteenB_ms_regrid = "{}.regrid".format(fourteenB_ms)

# Also convert th 14B data to an MMS w/ the same number of sub-MS as 17B
mstransform(vis=fourteenB_ms,
            outputvis=fourteenB_ms_regrid,
            field='M33_2,M33_6,M33_7_center,M33_8,M33_11,M33_12,M33_14',
            spw='0', regridms=True, mode='velocity', veltype='radio',
            start="{}km/s".format(start_vel),
            width="{}km/s".format(chan_width), nchan=n_chan,
            interpolation='fftshift',
            createmms=True, separationaxis='auto', numsubms=31)

# concatenate together, and will also remove the individual MS
virtualconcat(vis=[fourteenB_ms_regrid, seventeenB_ms_regrid],
              concatvis=concat_vis)
{'fraction_non_alphanumeric': 0.10574092247301276, 'fraction_numerical': 0.048331697742885184, 'mean_word_length': 3.7024221453287196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4877672', 'n_tokens_mistral': 1653, 'n_tokens_neox': 1514, 'n_words': 339}