<commit_before>#!/usr/bin/env python
import optparse
import os
import sys
import logging
import simplejson as json
import itertools
import subprocess
from StringIO import StringIO
import math

import drivecasa
from drivecasa.keys import clean_results as clean_keys
from driveami import keys as ami_keys
import amisurvey
from amisurvey.keys import obs_info as obs_keys

from tkp.accessors import FitsImage
from tkp.accessors import sourcefinder_image_from_accessor
from tkp.accessors import writefits as tkp_writefits
from tkp.sourcefinder.utils import generate_result_maps
import tkp.bin.pyse

ami_clean_args = {   "spw": '0:3~7',
          "imsize": [512, 512],
          "cell": ['5.0arcsec'],
          "pbcor": False,
#           "weighting": 'natural',
             "weighting": 'briggs',
             "robust": 0.5,
#          "weighting":'uniform',
          "psfmode": 'clark',
          "imagermode": 'csclean',
          }




def handle_args():
    """
    Default values are defined here.
    """
    default_output_dir = os.path.expanduser("/data2/ami_results")
    default_casa_dir = None
    usage = """usage: %prog [options] datasets_to_process.json\n"""
    parser = optparse.OptionParser(usage)

    parser.add_option("-o", "--output-dir", default=default_output_dir,
                      help="Path to output directory (default is : " +
                            default_output_dir + ")")

    parser.add_option("--casa-dir", default=default_casa_dir,
                   help="Path to CASA directory, default: " +
                                str(default_casa_dir))

    options, args = parser.parse_args()
    options.output_dir = os.path.expanduser(options.output_dir)
    if len(args) != 1:
        parser.print_help()
        sys.exit(1)
    print "Reducing files listed in:", args[0]
    return options, args[0]

def main(options, listings_file):
    print "Processing all_obs in:", listings_file
    all_obs = load_listings(listings_file)
    obs_groups = get_grouped_file_listings(all_obs)
    output_preamble_to_log(obs_groups)

    for grp_name in sorted(obs_groups.keys()):
        casa_output_dir = os.path.join(options.output_dir, grp_name, 'casa')
        fits_output_dir = os.path.join(options.output_dir, grp_name, 'images')
        casa_logfile = os.path.join(casa_output_dir, 'casalog.txt')

        casa = drivecasa.Casapy(working_dir=casa_output_dir,
                                casa_logfile=casa_logfile)

        logger.info(' '.join(("Processing", grp_name,
                             ", logfile at: ", casa_logfile)))

        grp_obs = obs_groups[grp_name]
        grp_dir = os.path.join(os.path.expanduser(options.output_dir),
                               str(grp_name))

        good_obs, rejected = reject_bad_obs(grp_obs)

        script, concat_obs = import_and_concatenate(good_obs, casa_output_dir)

        script.extend(make_dirty_map(concat_obs, casa_output_dir, fits_output_dir))
        for obs in good_obs:
            script.extend(make_dirty_map(obs, casa_output_dir, fits_output_dir))

#         Ok, run what we have so far:
        casa_out, errors = casa.run_script(script, raise_on_severe=False)
        logger.debug("Got the following errors (probably all ok)")
        for e in errors:
            logger.debug(e)
        # Now we can grab an estimate of the RMS for each map:

        concat_obs[obs_keys.dirty_rms_est] = get_image_rms_estimate(
                            concat_obs[obs_keys.dirty_maps][clean_keys.image])
        for obs in good_obs:
            dmap = obs[obs_keys.dirty_maps][clean_keys.image]
            obs[obs_keys.dirty_rms_est] = get_image_rms_estimate(dmap)

        # Ok, let's do an open clean on the concat map, to try and create a
        # deep source catalogue:
        script = make_open_clean_map(concat_obs, casa_output_dir, fits_output_dir)
        casa_out, errors = casa.run_script(script, raise_on_severe=False)


        # Perform sourcefinding, determine mask:
        sources = run_sourcefinder(concat_obs[obs_keys.open_clean_fits])
        write_ds9_regionfile(sources,
                             os.path.join(fits_output_dir, 'all_sources.reg'))
        mask_thresh = 5.5
        masked_sources = [s for s in sources if s.sig > mask_thresh]
        write_ds9_regionfile(masked_sources,
                             os.path.join(fits_output_dir, 'mask_sources.reg'))

        mask_coords = [(str(s.ra.value) + 'deg', str(s.dec.value) + 'deg')
                       for s in masked_sources]
        mask = drivecasa.utils.get_circular_mask_string(mask_coords,
                                             aperture_radius="5pix")
        logger.info("Mask:" + mask)

        # Now go and do masked and open cleans for everything:
        script = []<commit_msg>Print mask on separate line.<commit_after>#!/usr/bin/env python
import optparse
import os
import sys
import logging
import simplejson as json
import itertools
import subprocess
from StringIO import StringIO
import math

import drivecasa
from drivecasa.keys import clean_results as clean_keys
from driveami import keys as ami_keys
import amisurvey
from amisurvey.keys import obs_info as obs_keys

from tkp.accessors import FitsImage
from tkp.accessors import sourcefinder_image_from_accessor
from tkp.accessors import writefits as tkp_writefits
from tkp.sourcefinder.utils import generate_result_maps
import tkp.bin.pyse

ami_clean_args = {   "spw": '0:3~7',
          "imsize": [512, 512],
          "cell": ['5.0arcsec'],
          "pbcor": False,
#           "weighting": 'natural',
             "weighting": 'briggs',
             "robust": 0.5,
#          "weighting":'uniform',
          "psfmode": 'clark',
          "imagermode": 'csclean',
          }




def handle_args():
    """
    Default values are defined here.
    """
    default_output_dir = os.path.expanduser("/data2/ami_results")
    default_casa_dir = None
    usage = """usage: %prog [options] datasets_to_process.json\n"""
    parser = optparse.OptionParser(usage)

    parser.add_option("-o", "--output-dir", default=default_output_dir,
                      help="Path to output directory (default is : " +
                            default_output_dir + ")")

    parser.add_option("--casa-dir", default=default_casa_dir,
                   help="Path to CASA directory, default: " +
                                str(default_casa_dir))

    options, args = parser.parse_args()
    options.output_dir = os.path.expanduser(options.output_dir)
    if len(args) != 1:
        parser.print_help()
        sys.exit(1)
    print "Reducing files listed in:", args[0]
    return options, args[0]

def main(options, listings_file):
    print "Processing all_obs in:", listings_file
    all_obs = load_listings(listings_file)
    obs_groups = get_grouped_file_listings(all_obs)
    output_preamble_to_log(obs_groups)

    for grp_name in sorted(obs_groups.keys()):
        casa_output_dir = os.path.join(options.output_dir, grp_name, 'casa')
        fits_output_dir = os.path.join(options.output_dir, grp_name, 'images')
        casa_logfile = os.path.join(casa_output_dir, 'casalog.txt')

        casa = drivecasa.Casapy(working_dir=casa_output_dir,
                                casa_logfile=casa_logfile)

        logger.info(' '.join(("Processing", grp_name,
                             ", logfile at: ", casa_logfile)))

        grp_obs = obs_groups[grp_name]
        grp_dir = os.path.join(os.path.expanduser(options.output_dir),
                               str(grp_name))

        good_obs, rejected = reject_bad_obs(grp_obs)

        script, concat_obs = import_and_concatenate(good_obs, casa_output_dir)

        script.extend(make_dirty_map(concat_obs, casa_output_dir, fits_output_dir))
        for obs in good_obs:
            script.extend(make_dirty_map(obs, casa_output_dir, fits_output_dir))

#         Ok, run what we have so far:
        casa_out, errors = casa.run_script(script, raise_on_severe=False)
        logger.debug("Got the following errors (probably all ok)")
        for e in errors:
            logger.debug(e)
        # Now we can grab an estimate of the RMS for each map:

        concat_obs[obs_keys.dirty_rms_est] = get_image_rms_estimate(
                            concat_obs[obs_keys.dirty_maps][clean_keys.image])
        for obs in good_obs:
            dmap = obs[obs_keys.dirty_maps][clean_keys.image]
            obs[obs_keys.dirty_rms_est] = get_image_rms_estimate(dmap)

        # Ok, let's do an open clean on the concat map, to try and create a
        # deep source catalogue:
        script = make_open_clean_map(concat_obs, casa_output_dir, fits_output_dir)
        casa_out, errors = casa.run_script(script, raise_on_severe=False)


        # Perform sourcefinding, determine mask:
        sources = run_sourcefinder(concat_obs[obs_keys.open_clean_fits])
        write_ds9_regionfile(sources,
                             os.path.join(fits_output_dir, 'all_sources.reg'))
        mask_thresh = 5.5
        masked_sources = [s for s in sources if s.sig > mask_thresh]
        write_ds9_regionfile(masked_sources,
                             os.path.join(fits_output_dir, 'mask_sources.reg'))

        mask_coords = [(str(s.ra.value) + 'deg', str(s.dec.value) + 'deg')
                       for s in masked_sources]
        mask = drivecasa.utils.get_circular_mask_string(mask_coords,
                                             aperture_radius="5pix")
        logger.info("Mask:\n" + mask)

        # Now go and do masked and open cleans for everything:
        script = []
{'fraction_non_alphanumeric': 0.09634031904910854, 'fraction_numerical': 0.004796163069544364, 'mean_word_length': 2.5618269587820275, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5244500', 'n_tokens_mistral': 3093, 'n_tokens_neox': 2986, 'n_words': 646}