<commit_before># system level modules
import logging
import os
import re
# system module functions
from uuid import uuid4
from subprocess import Popen, PIPE
from pprint import pformat
from time import sleep
import time
# job modules
from JobStatus import JobStatus
# output_viewer modules
from output_viewer.index import OutputPage
from output_viewer.index import OutputIndex
from output_viewer.index import OutputRow
from output_viewer.index import OutputGroup
from output_viewer.index import OutputFile
# lib.util modules
from lib.util import print_debug
from lib.util import print_message
from lib.util import check_slurm_job_submission
from lib.util import create_symlink_dir
from lib.util import push_event
from lib.util import render
from lib.util import get_climo_output_files


class AMWGDiagnostic(object):
    """
    A job class to perform the NCAR AMWG Diagnostic
    """
    def __init__(self, config, event_list):
        """
            Setup class attributes

            inputs:
                test_casename: the name of the test case e.g. b40.20th.track1.2deg.001
                test_filetype: the filetype of the history files, either monthly_history or time_series
                test_path_history: path to the directory holding your history files
                test_path_climo: path to directory holding climo files
                test_path_diag: the output path for the diagnostics to go
                control: what type to use for the control set, either OBS for observations, or USER for another model
                    the following are only set with control==USER
                    cntl_casename: the case_name of the control case
                    cntl_filetype: either monthly_history or time_series
                    cntl_path_history: path to the control history file
                    cntl_path_climo: path to the control climo files
        """
        self.event_list = event_list
        self.inputs = {
            'diag_home': '',
            'test_casename': '',
            'test_path': '',
            'test_filetype': 'monthly_history',
            'test_path_history': '',
            'test_path_climo': '',
            'test_path_diag': '',
            'regrided_climo_path': '',
            'start_year': '',
            'end_year': '',
            'set_number': '',
            'run_directory': '',
            'template_path': '',
            'dataset_name': '',
        }
        self.type = 'amwg_diagnostic'
        self.outputs = {}
        self.config = {}
        self.uuid = uuid4().hex
        self.job_id = 0
        self.depends_on = []
        self.slurm_args = {
            'num_cores': '-n 16', # 16 cores
            'run_time': '-t 0-02:00', # 2 hours run time
            'num_machines': '-N 1', # run on one machine
            'oversubscribe': '--oversubscribe'
        }
        self.prevalidate(config)

    def get_type(self):
        return self.type

    def set_status(self, status):
        self.status = status

    def prevalidate(self, config):
        """
            Iterate over given config dictionary making sure all the inputs are set
            and rejecting any inputs that arent in the input dict
        """
        for key, value in config.iteritems():
            if key in self.inputs:
                self.config[key] = value
        for key, value in self.inputs.iteritems():
            if key not in self.config:
                self.config[key] = value

        for d in config.get('depends_on'):
            self.depends_on.append(d)
        self.status = JobStatus.VALID

        if os.path.exists(self.config.get('run_directory')):
            contents = os.listdir(self.config.get('run_directory'))
            if len(contents) > 2300:
                self.status = JobStatus.COMPLETED

    def get_set(self, filename):
        for i in range(len(filename)):
            if filename[i].isdigit():
                s_index = i
                break
        set_number = filename[i:]
        return set_number

    def get_attrs(self, filename):
        filesplit = filename.split('_')
        set_id = self.get_set(filesplit[0])
        seasons = ['DJF', 'MAM', 'JJA', 'SON', 'ANN']
        if filesplit[1] in seasons:
            col = filesplit[1]
            row = filesplit[2]
            group = '_'.join(filesplit[3:])[:-4]
        else:
            s_index = 2
            for i in range(s_index, len(filesplit)):
                if filesplit[i] in seasons:
                    s_index = i
                    break
            col = filesplit[s_index]
            row = '_'.join(filesplit[1: s_index])
            group = '_'.join(filesplit[s_index:])[:-4]
        return set_id, group, row, col

    def generateIndex(self):
        self.event_list = push_event(self.event_list, 'Starting index generataion for AMWG diagnostic')
        contents = [s for s in os.listdir(self.config.get('run_directory')) if s.endswith('png')]
        dataset_name = self.config.get('dataset_name')
        index = OutputIndex('AMWG Diagnostic', version=dataset_name)

        pages = {}
        for item in contents:
            page, group, row, col = self.get_attrs(item)
            if not pages.get(page):
                pages[page] = {}
            if not pages.get(page).get(group):
                pages[page][group] = {}
            if not pages.get(page).get(group).get(row):
                pages[page][group][row] = {}
            if not pages.get(page).get(group).get(row).get(col):
                pages[page][group][row][col] = OutputFile(item)

        for pi, page in pages.items():
            outpage = OutputPage(pi)
            for gi, group in page.items():
                outgroup = OutputGroup(gi)
                for ri, row in group.items():
                    outrow = OutputRow(ri, row)
                    outgroup.addRow(outrow)
                outpage.addGroup(outgroup)
            index.addPage(outpage)
        index.toJSON(os.path.join(self.config.get('run_directory'), 'index.json'))
        self.event_list = push_event(self.event_list, 'Index generataion complete')

    def postvalidate(self):
        """
            Check that what the job was supposed to do actually happened
        """
        print 'postvalidate'

    def execute(self, batch='slurm'):
        """
            Perform the actual work
        """
        run_dir = self.config.get('run_directory')
        if not os.path.exists(run_dir):
            os.makedirs(run_dir)
        # render the csh script
        template_out = os.path.join(run_dir, 'amwg.csh')
        render(
            variables=self.config,
            input_path=self.config.get('template_path'),
            output_path=template_out)
        # get the list of climo files for this diagnostic
        file_list = get_climo_output_files(
            input_path=self.config.get('regrided_climo_path'),
            set_start_year=self.config.get('start_year'),
            set_end_year=self.config.get('end_year'))
        # create the directory of symlinks
        create_symlink_dir(
            src_dir=self.config.get('regrided_climo_path'),
            src_list=file_list,
            dst=self.config.get('test_path_climo'))<commit_msg>Adds pre-compute check for amwg<commit_after># system level modules
import logging
import os
import re
import json
# system module functions
from uuid import uuid4
from subprocess import Popen, PIPE
from pprint import pformat
from time import sleep
import time
# job modules
from JobStatus import JobStatus
# output_viewer modules
from output_viewer.index import OutputPage
from output_viewer.index import OutputIndex
from output_viewer.index import OutputRow
from output_viewer.index import OutputGroup
from output_viewer.index import OutputFile
# lib.util modules
from lib.util import print_debug
from lib.util import print_message
from lib.util import check_slurm_job_submission
from lib.util import create_symlink_dir
from lib.util import push_event
from lib.util import render
from lib.util import get_climo_output_files


class AMWGDiagnostic(object):
    """
    A job class to perform the NCAR AMWG Diagnostic
    """
    def __init__(self, config, event_list):
        """
            Setup class attributes

            inputs:
                test_casename: the name of the test case e.g. b40.20th.track1.2deg.001
                test_filetype: the filetype of the history files, either monthly_history or time_series
                test_path_history: path to the directory holding your history files
                test_path_climo: path to directory holding climo files
                test_path_diag: the output path for the diagnostics to go
                control: what type to use for the control set, either OBS for observations, or USER for another model
                    the following are only set with control==USER
                    cntl_casename: the case_name of the control case
                    cntl_filetype: either monthly_history or time_series
                    cntl_path_history: path to the control history file
                    cntl_path_climo: path to the control climo files
        """
        self.event_list = event_list
        self.inputs = {
            'diag_home': '',
            'test_casename': '',
            'test_path': '',
            'test_filetype': 'monthly_history',
            'test_path_history': '',
            'test_path_climo': '',
            'test_path_diag': '',
            'regrided_climo_path': '',
            'start_year': '',
            'end_year': '',
            'set_number': '',
            'run_directory': '',
            'template_path': '',
            'dataset_name': '',
        }
        self.type = 'amwg_diagnostic'
        self.outputs = {}
        self.config = {}
        self.uuid = uuid4().hex
        self.job_id = 0
        self.depends_on = []
        self.slurm_args = {
            'num_cores': '-n 16', # 16 cores
            'run_time': '-t 0-02:00', # 2 hours run time
            'num_machines': '-N 1', # run on one machine
            'oversubscribe': '--oversubscribe'
        }
        self.prevalidate(config)

    def get_type(self):
        return self.type

    def set_status(self, status):
        self.status = status

    def prevalidate(self, config):
        """
            Iterate over given config dictionary making sure all the inputs are set
            and rejecting any inputs that arent in the input dict
        """
        for key, value in config.iteritems():
            if key in self.inputs:
                self.config[key] = value
        for key, value in self.inputs.iteritems():
            if key not in self.config:
                self.config[key] = value

        for d in config.get('depends_on'):
            self.depends_on.append(d)
        self.status = JobStatus.VALID

    def get_set(self, filename):
        for i in range(len(filename)):
            if filename[i].isdigit():
                s_index = i
                break
        set_number = filename[i:]
        return set_number

    def get_attrs(self, filename):
        filesplit = filename.split('_')
        set_id = self.get_set(filesplit[0])
        seasons = ['DJF', 'MAM', 'JJA', 'SON', 'ANN']
        if filesplit[1] in seasons:
            col = filesplit[1]
            row = filesplit[2]
            group = '_'.join(filesplit[3:])[:-4]
        else:
            s_index = 2
            for i in range(s_index, len(filesplit)):
                if filesplit[i] in seasons:
                    s_index = i
                    break
            col = filesplit[s_index]
            row = '_'.join(filesplit[1: s_index])
            group = '_'.join(filesplit[s_index:])[:-4]
        return set_id, group, row, col

    def generateIndex(self):
        self.event_list = push_event(self.event_list, 'Starting index generataion for AMWG diagnostic')
        contents = [s for s in os.listdir(self.config.get('run_directory')) if s.endswith('png')]
        dataset_name = self.config.get('dataset_name')
        index = OutputIndex('AMWG Diagnostic', version=dataset_name)

        pages = {}
        for item in contents:
            page, group, row, col = self.get_attrs(item)
            if not pages.get(page):
                pages[page] = {}
            if not pages.get(page).get(group):
                pages[page][group] = {}
            if not pages.get(page).get(group).get(row):
                pages[page][group][row] = {}
            if not pages.get(page).get(group).get(row).get(col):
                pages[page][group][row][col] = []
            pages[page][group][row][col].append(OutputFile(path=item, title=item))
            # pages[page][group][row][col].append(item)

        # with open('amwg_index.json', 'w') as f:
        #     json.dump(pages, f)

        for pi, page in pages.items():
            outpage = OutputPage(pi)
            for gi, group in page.items():
                outgroup = OutputGroup(gi)
                group_ind = len(outpage.groups)
                outpage.addGroup(outgroup)
                for ri, row in group.items():
                    tmp_row_list = []
                    for key, val in row.items():
                        tmp_row_list.append(val)
                    outrow = OutputRow(ri, tmp_row_list)
                    outpage.addRow(outrow, group_ind)
            index.addPage(outpage)
        try:
            index.toJSON(os.path.join(self.config.get('run_directory'), 'index.json'))
        except:
            self.event_list = push_event(self.event_list, 'Index generation failed')
        else:
            self.event_list = push_event(self.event_list, 'Index generataion complete')

    def postvalidate(self):
        """
            Check that what the job was supposed to do actually happened
        """
        print 'postvalidate'

    def execute(self, batch='slurm'):
        """
            Perform the actual work
        """

        # First check if the job has already been completed
        if os.path.exists(self.config.get('run_directory')):
            contents = os.listdir(self.config.get('run_directory'))
        if len(contents) > 2300:
            self.event_list = push_event(self.event_list, 'AMWG already computed, skipping job')
            self.status = JobStatus.COMPLETED
            return 

        run_dir = self.config.get('run_directory')
        if not os.path.exists(run_dir):
            os.makedirs(run_dir)
        # render the csh script
        template_out = os.path.join(run_dir, 'amwg.csh')
        render(
            variables=self.config,
            input_path=self.config.get('template_path'),
            output_path=template_out)
        # get the list of climo files for this diagnostic
        file_list = get_climo_output_files(
            input_path=self.config.get('regrided_climo_path'),
            set_start_year=self.config.get('start_year'),
            set_end_year=self.config.get('end_year'))
        # create the directory of symlinks
        create_symlink_dir(
            src_dir=self.config.get('regrided_climo_path'),
            src_list=file_list,
            dst=self.config.get('test_path_climo'))
{'fraction_non_alphanumeric': 0.09792831305491614, 'fraction_numerical': 0.004735284445905952, 'mean_word_length': 1.9909520062942565, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6306791', 'n_tokens_mistral': 4375, 'n_tokens_neox': 4235, 'n_words': 1127}