<commit_before>#!/usr/bin/env python

import os
import sys
import logging
import ece2cmor
import ifs2cmor
import cmor_source
import jsonloader
import optparse
import datetime
from dateutil.relativedelta import relativedelta

# This example script performs cmorization of one month of ifs data, starting at \
# januari 1st 1990. It requires an output directory and an experiment name/prefix \
# to determine the output data files and configure \
# cmor3 correctly. The processed variables are listed in the "variables" dictionary

def is3hrtask(task):
   if(not isinstance(task.source,cmor_source.ifs_source)): return False
   if(task.target.variable in ["ua850","va850"]): return True
   return (task.source.spatial_dims == 2)

logging.basicConfig(level=logging.DEBUG)

startdate = datetime.date(1990,1,1)
interval = relativedelta(months=1)
srcdir = os.path.dirname(os.path.abspath(ece2cmor.__file__))
curdir = os.path.join(srcdir,"examples","primavera")
datdir = os.path.join(srcdir,"test","test_data","ifsdata","3hr")
tmpdir = os.path.join(curdir,"tmp")
varfile = os.path.join(curdir,"varlist.json")
conffile = os.path.join(curdir,"primavera.json")

def main(args):

    parser = optparse.OptionParser()
    parser.add_option("-d","--dir", dest = "dir",  help = "IFS output directory (optional)",       default = datdir)
    parser.add_option("-c","--conf",dest = "conf", help = "Input variable list (optional)",        default = conffile)
    parser.add_option("-e","--exp", dest = "exp",  help = "Experiment prefix (optional)",          default = "ECE3")
    parser.add_option("-t","--tmp", dest = "temp", help = "Temporary working directory (optional)",default = tmpdir)
    parser.add_option("-v","--var", dest = "vars", help = "Input variable list (optional)",        default = varfile)

    (opt,args) = parser.parse_args()

    # Initialize ece2cmor with experiment prefix:
    ece2cmor.initialize(opt.conf,opt.exp)

    odir = os.path.abspath(opt.dir)
    if(not os.path.isdir(odir)): raise Exception("Nonexistent output directory given:",odir)

    # Set directory and time interval for cmorization step:
    ece2cmor.ifsdir = odir
    ece2cmor.startdate = startdate
    ece2cmor.interval = interval

    # Load the variables as task targets:
    jsonloader.load_targets(opt.vars)

    # Remove targets that are constructed from six-hourly data:
    ece2cmor.tasks = [t for t in ece2cmor.tasks if is3hrtask(t)]

    if(not os.path.isdir(opt.temp)): os.makedirs(opt.temp)

    # Execute the cmorization:
    ece2cmor.perform_ifs_tasks(outputfreq = 3,tempdir = opt.temp,cleanup=False)

if __name__ == "__main__":
    main(sys.argv[1:])<commit_msg>Allow cleanup for 3-hourly atmosphere cmor<commit_after>#!/usr/bin/env python

import os
import sys
import logging
import ece2cmor
import ifs2cmor
import cmor_source
import jsonloader
import optparse
import datetime
from dateutil.relativedelta import relativedelta

# This example script performs cmorization of one month of ifs data, starting at \
# januari 1st 1990. It requires an output directory and an experiment name/prefix \
# to determine the output data files and configure \
# cmor3 correctly. The processed variables are listed in the "variables" dictionary

def is3hrtask(task):
   if(not isinstance(task.source,cmor_source.ifs_source)): return False
   if(task.target.variable in ["ua850","va850"]): return True
   return (task.source.spatial_dims == 2)

logging.basicConfig(level=logging.DEBUG)

startdate = datetime.date(1990,1,1)
interval = relativedelta(months=1)
srcdir = os.path.dirname(os.path.abspath(ece2cmor.__file__))
curdir = os.path.join(srcdir,"examples","primavera")
datdir = os.path.join(srcdir,"test","test_data","ifsdata","3hr")
tmpdir = os.path.join(curdir,"tmp")
varfile = os.path.join(curdir,"varlist.json")
conffile = os.path.join(curdir,"primavera.json")

def main(args):

    parser = optparse.OptionParser()
    parser.add_option("-d","--dir", dest = "dir",  help = "IFS output directory (optional)",       default = datdir)
    parser.add_option("-c","--conf",dest = "conf", help = "Input variable list (optional)",        default = conffile)
    parser.add_option("-e","--exp", dest = "exp",  help = "Experiment prefix (optional)",          default = "ECE3")
    parser.add_option("-t","--tmp", dest = "temp", help = "Temporary working directory (optional)",default = tmpdir)
    parser.add_option("-v","--var", dest = "vars", help = "Input variable list (optional)",        default = varfile)

    (opt,args) = parser.parse_args()

    # Initialize ece2cmor with experiment prefix:
    ece2cmor.initialize(opt.conf,opt.exp)

    odir = os.path.abspath(opt.dir)
    if(not os.path.isdir(odir)): raise Exception("Nonexistent output directory given:",odir)

    # Set directory and time interval for cmorization step:
    ece2cmor.ifsdir = odir
    ece2cmor.startdate = startdate
    ece2cmor.interval = interval

    # Load the variables as task targets:
    jsonloader.load_targets(opt.vars)

    # Remove targets that are constructed from six-hourly data:
    ece2cmor.tasks = [t for t in ece2cmor.tasks if is3hrtask(t)]

    if(not os.path.isdir(opt.temp)): os.makedirs(opt.temp)

    # Execute the cmorization:
    ece2cmor.perform_ifs_tasks(outputfreq = 3,tempdir = opt.temp)

if __name__ == "__main__":
    main(sys.argv[1:])
{'fraction_non_alphanumeric': 0.12971653838933733, 'fraction_numerical': 0.014079219072648771, 'mean_word_length': 4.376387487386478, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3932854', 'n_tokens_mistral': 1779, 'n_tokens_neox': 1735, 'n_words': 438}